Merge pull request #3336 from PascalWithopf/omclickhouse

new output module for DBMS Clickhouse
2025-12-18 20:20:41 +01:00 · 2018-12-20 15:11:00 +01:00 · 2018-12-20 15:11:00 +01:00 · ebc6a56250
commit ebc6a56250
parent 3ef7dbfe88 687f5c1ee1
23 changed files with 1456 additions and 0 deletions
--- a/Makefile.am
+++ b/Makefile.am
@ -179,6 +179,10 @@ if ENABLE_ELASTICSEARCH
 SUBDIRS += plugins/omelasticsearch
 endif

+if ENABLE_CLICKHOUSE
+SUBDIRS += plugins/omclickhouse
+endif
+
 if ENABLE_OMHTTP
 SUBDIRS += contrib/omhttp
 endif
@ -442,6 +446,13 @@ if ENABLE_ELASTICSEARCH_TESTS
 DISTCHECK_CONFIGURE_FLAGS+= --enable-elasticsearch-tests
 endif

+if ENABLE_CLICKHOUSE
+DISTCHECK_CONFIGURE_FLAGS+= --enable-clickhouse
+endif
+if ENABLE_CLICKHOUSE_TESTS
+DISTCHECK_CONFIGURE_FLAGS+= --enable-clickhouse-tests
+endif
+
 if ENABLE_MMGROK
 DISTCHECK_CONFIGURE_FLAGS+= --enable-mmgrok
 endif
--- a/configure.ac
+++ b/configure.ac
@ -918,6 +918,23 @@ fi
 AM_CONDITIONAL(ENABLE_ELASTICSEARCH, test x$enable_elasticsearch = xyes)


+# clickhouse support
+AC_ARG_ENABLE(clickhouse,
+        [AS_HELP_STRING([--enable-clickhouse],[Enable clickhouse output module @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_clickhouse="yes" ;;
+          no) enable_clickhouse="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-clickhouse) ;;
+         esac],
+        [enable_clickhouse=no]
+)
+if test "x$enable_clickhouse" = "xyes"; then
+	PKG_CHECK_MODULES([CURL], [libcurl])
+	LT_LIB_M
+fi
+AM_CONDITIONAL(ENABLE_CLICKHOUSE, test x$enable_clickhouse = xyes)
+
+
 # omhttp support
 AC_ARG_ENABLE(omhttp,
        [AS_HELP_STRING([--enable-omhttp],[Enable http output module @<:@default=no@:>@])],
@ -953,6 +970,20 @@ AM_CONDITIONAL(ENABLE_ELASTICSEARCH_TESTS, test x$enable_elasticsearch_tests = x
 AM_CONDITIONAL(ENABLE_ELASTICSEARCH_TESTS_MINIMAL, test x$enable_elasticsearch_tests = xminimal -o x$enable_elasticsearch_tests = xyes)


+# capability to enable clickhouse testbench tests. This requries that a test
+# environment is present on the local (127.0.0.1) machine. 
+AC_ARG_ENABLE(clickhouse_tests,
+        [AS_HELP_STRING([--enable-clickhouse-tests],[enable Elasticsearch specific tests in testbench @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_clickhouse_tests="yes" ;;
+          no) enable_clickhouse_tests="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-clickhouse-tests) ;;
+         esac],
+        [enable_clickhouse_tests=no]
+)
+AM_CONDITIONAL(ENABLE_CLICKHOUSE_TESTS, test x$enable_clickhouse_tests = xyes)
+
+
 # openssl support
 AC_ARG_ENABLE(openssl,
        [AS_HELP_STRING([--enable-openssl],[Enable openssl support @<:@default=no@:>@])],
@ -2344,6 +2375,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/mmfields/Makefile \
 		plugins/mmpstrucdata/Makefile \
 		plugins/omelasticsearch/Makefile \
+		plugins/omclickhouse/Makefile \
 		plugins/mmsnmptrapd/Makefile \
 		plugins/pmlastmsg/Makefile \
 		plugins/mmdblookup/Makefile \
@ -2414,6 +2446,7 @@ echo "    omstdout module will be compiled:         $enable_omstdout"
 echo "    omjournal module will be compiled:        $enable_omjournal"
 echo "    omhdfs module will be compiled:           $enable_omhdfs"
 echo "    omelasticsearch module will be compiled:  $enable_elasticsearch"
+echo "    omclickhouse module will be compiled:     $enable_clickhouse"
 echo "    omhttp module will be compiled:           $enable_omhttp"
 echo "    omruleset module will be compiled:        $enable_omruleset"
 echo "    omudpspoof module will be compiled:       $enable_omudpspoof"
@ -2478,6 +2511,7 @@ echo "    Testbench libfaketime tests enabled:      $enable_libfaketime"
 echo "    Extended Testbench enabled:               $enable_extended_tests"
 echo "    MySQL Tests enabled:                      $enable_mysql_tests"
 echo "    Elasticsearch Tests:                      $enable_elasticsearch_tests"
+echo "    ClickHouse Tests:                         $enable_clickhouse_tests"
 echo "    PostgreSQL Tests enabled:                 $enable_pgsql_tests"
 echo "    Kafka Tests enabled:                      $enable_kafka_tests"
 echo "    systemd journal tests enabled:            $enable_journal_tests"
--- a/devtools/prepare_clickhouse.sh
+++ b/devtools/prepare_clickhouse.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+# this script prepares a clickhouse instance for use by the rsyslog testbench
+
+
+clickhouse-client --query="CREATE DATABASE rsyslog"
+
+# At the moment only the database is created for preperation.
+# Every test creates a table for itself and drops it afterwards.
+# This could look something like this:
+
+#clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.test ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+#clickhouse-client --query="DROP TABLE rsyslog.test"
--- a/plugins/omclickhouse/Makefile.am
+++ b/plugins/omclickhouse/Makefile.am
@ -0,0 +1,8 @@
+pkglib_LTLIBRARIES = omclickhouse.la
+
+omclickhouse_la_SOURCES = omclickhouse.c
+omclickhouse_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(CURL_CFLAGS)
+omclickhouse_la_LDFLAGS = -module -avoid-version
+omclickhouse_la_LIBADD =  $(CURL_LIBS) $(LIBM)
+
+EXTRA_DIST = 
--- a/plugins/omclickhouse/omclickhouse.c
+++ b/plugins/omclickhouse/omclickhouse.c
@ -0,0 +1,940 @@
+/* omclickhouse.c
+ * This is the https://clickhouse.yandex/ output module.
+ *
+ * Copyright 2018 Pascal Withopf and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *       -or-
+ *       see COPYING.ASL20 in the source distribution
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <memory.h>
+#include <string.h>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <assert.h>
+#include <signal.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#if defined(__FreeBSD__)
+#include <unistd.h>
+#endif
+#include <json.h>
+#include "conf.h"
+#include "syslogd-types.h"
+#include "srUtils.h"
+#include "template.h"
+#include "module-template.h"
+#include "errmsg.h"
+#include "statsobj.h"
+#include "cfsysline.h"
+#include "unicode-helper.h"
+#include "obj-types.h"
+#include "ratelimit.h"
+#include "ruleset.h"
+
+#ifndef O_LARGEFILE
+#  define O_LARGEFILE 0
+#endif
+
+MODULE_TYPE_OUTPUT
+MODULE_TYPE_NOKEEP
+MODULE_CNFNAME("omclickhouse")
+
+/* internal structures */
+DEF_OMOD_STATIC_DATA
+DEFobjCurrIf(statsobj)
+DEFobjCurrIf(prop)
+DEFobjCurrIf(ruleset)
+
+statsobj_t *indexStats;
+STATSCOUNTER_DEF(indexSubmit, mutIndexSubmit)
+STATSCOUNTER_DEF(indexHTTPFail, mutIndexHTTPFail)
+STATSCOUNTER_DEF(indexHTTPReqFail, mutIndexHTTPReqFail)
+STATSCOUNTER_DEF(indexFail, mutIndexFail)
+STATSCOUNTER_DEF(indexSuccess, mutIndexSuccess)
+
+
+
+typedef struct curl_slist HEADER;
+typedef struct instanceConf_s {
+	uchar *serverBaseUrl;
+	int port;
+	uchar *user;
+	uchar *pwd;
+	uchar *authBuf;
+	uchar *tplName;
+	sbool useHttps;
+	int fdErrFile;
+	uchar *errorFile;
+	sbool bulkmode;
+	size_t maxbytes;
+	uchar *caCertFile;
+	uchar *myCertFile;
+	uchar *myPrivKeyFile;
+	struct instanceConf_s *next;
+} instanceData;
+
+struct modConfData_s {
+	rsconf_t *pConf;		/* our overall config object */
+	instanceConf_t *root, *tail;
+};
+static modConfData_t *loadModConf = NULL;	/* modConf ptr to use for the current load process */
+
+typedef struct wrkrInstanceData {
+	PTR_ASSERT_DEF
+	instanceData *pData;
+	CURL	*curlPostHandle;	/* libcurl session handle for posting data to the server */
+	HEADER	*curlHeader;	/* json POST request info */
+	CURL	*curlCheckConnHandle;	/* libcurl session handle for checking the server connection */
+	int replyLen;
+	char *reply;
+	uchar *restURL;
+	struct {
+		es_str_t *data;
+		int nmemb;	/* number of messages in batch (for statistics counting) */
+	} batch;
+	sbool insertErrorSent;  /* needed for insert error message */
+} wrkrInstanceData_t;
+
+/* tables for interfacing with the v6 config system */
+/* action (instance) parameters */
+static struct cnfparamdescr actpdescr[] = {
+	{ "server", eCmdHdlrGetWord, 0 },
+	{ "port", eCmdHdlrInt, 0 },
+	{ "user", eCmdHdlrGetWord, 0 },
+	{ "pwd", eCmdHdlrGetWord, 0 },
+	{ "template", eCmdHdlrGetWord, 0 },
+	{ "usehttps", eCmdHdlrBinary, 0 },
+	{ "errorfile", eCmdHdlrGetWord, 0 },
+	{ "bulkmode", eCmdHdlrBinary, 0 },
+	{ "maxbytes", eCmdHdlrSize, 0 },
+	{ "tls.cacert", eCmdHdlrString, 0 },
+	{ "tls.mycert", eCmdHdlrString, 0 },
+	{ "tls.myprivkey", eCmdHdlrString, 0 }
+};
+static struct cnfparamblk actpblk =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(actpdescr)/sizeof(struct cnfparamdescr),
+	  actpdescr
+	};
+
+static rsRetVal curlSetup(wrkrInstanceData_t *pWrkrData);
+
+BEGINcreateInstance
+CODESTARTcreateInstance
+	pData->fdErrFile = -1;
+	pData->caCertFile = NULL;
+	pData->myCertFile = NULL;
+	pData->myPrivKeyFile = NULL;
+ENDcreateInstance
+
+BEGINcreateWrkrInstance
+CODESTARTcreateWrkrInstance
+	pWrkrData->curlHeader = NULL;
+	pWrkrData->curlPostHandle = NULL;
+	pWrkrData->curlCheckConnHandle = NULL;
+	pWrkrData->restURL = NULL;
+	if(pData->bulkmode) {
+		if((pWrkrData->batch.data = es_newStr(1024)) == NULL) {
+			LogError(0, RS_RET_OUT_OF_MEMORY,
+				"omclickhouse: error creating batch string "
+			        "turned off bulk mode\n");
+			pData->bulkmode = 0; /* at least it works */
+		}
+	}
+	pWrkrData->insertErrorSent = 0;
+
+	iRet = curlSetup(pWrkrData);
+ENDcreateWrkrInstance
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+	if(eFeat == sFEATURERepeatedMsgReduction)
+		iRet = RS_RET_OK;
+ENDisCompatibleWithFeature
+
+BEGINfreeInstance
+CODESTARTfreeInstance
+	free(pData->serverBaseUrl);
+	free(pData->user);
+	free(pData->pwd);
+	free(pData->authBuf);
+	if(pData->fdErrFile != -1)
+		close(pData->fdErrFile);
+	free(pData->errorFile);
+	free(pData->tplName);
+	free(pData->caCertFile);
+	free(pData->myCertFile);
+	free(pData->myPrivKeyFile);
+ENDfreeInstance
+
+BEGINfreeWrkrInstance
+CODESTARTfreeWrkrInstance
+	if(pWrkrData->curlHeader != NULL) {
+		curl_slist_free_all(pWrkrData->curlHeader);
+		pWrkrData->curlHeader = NULL;
+	}
+	if(pWrkrData->curlCheckConnHandle != NULL) {
+		curl_easy_cleanup(pWrkrData->curlCheckConnHandle);
+		pWrkrData->curlCheckConnHandle = NULL;
+	}
+	if(pWrkrData->curlPostHandle != NULL) {
+		curl_easy_cleanup(pWrkrData->curlPostHandle);
+		pWrkrData->curlPostHandle = NULL;
+	}
+	if (pWrkrData->restURL != NULL) {
+		free(pWrkrData->restURL);
+		pWrkrData->restURL = NULL;
+	}
+	es_deleteStr(pWrkrData->batch.data);
+ENDfreeWrkrInstance
+
+BEGINdbgPrintInstInfo
+CODESTARTdbgPrintInstInfo
+	dbgprintf("omclickhouse\n");
+	dbgprintf("\tserver='%s'\n", pData->serverBaseUrl);
+	dbgprintf("\tport='%d'\n", pData->port);
+	dbgprintf("\tuser='%s'\n", pData->user);
+	dbgprintf("\tpwd='%s'\n", pData->pwd);
+	dbgprintf("\ttemplate='%s'\n", pData->tplName);
+	dbgprintf("\tusehttps='%d'\n", pData->useHttps);
+	dbgprintf("\terrorFile='%s'\n", pData->errorFile);
+	dbgprintf("\tbulkmode='%d'\n", pData->bulkmode);
+	dbgprintf("\tmaxbytes='%zu'\n", pData->maxbytes);
+	dbgprintf("\ttls.cacert='%s'\n", pData->caCertFile);
+	dbgprintf("\ttls.mycert='%s'\n", pData->myCertFile);
+	dbgprintf("\ttls.myprivkey='%s'\n", pData->myPrivKeyFile);
+ENDdbgPrintInstInfo
+
+
+BEGINtryResume
+CODESTARTtryResume
+	dbgprintf("omclickhouse: tryResume called\n");
+ENDtryResume
+
+
+/*
+ * Dumps entire bulk request and response in error log
+ */
+static rsRetVal
+getDataErrorDefault(wrkrInstanceData_t *pWrkrData, char *reply, uchar *reqmsg, char **rendered)
+{
+	DEFiRet;
+	fjson_object *req=NULL;
+	fjson_object *errRoot=NULL;
+
+	if((req=fjson_object_new_object()) == NULL) ABORT_FINALIZE(RS_RET_ERR);
+	fjson_object_object_add(req, "url", fjson_object_new_string((char*)pWrkrData->restURL));
+	fjson_object_object_add(req, "postdata", fjson_object_new_string((char*)reqmsg));
+
+	if((errRoot=fjson_object_new_object()) == NULL) ABORT_FINALIZE(RS_RET_ERR);
+	fjson_object_object_add(errRoot, "request", req);
+	fjson_object_object_add(errRoot, "reply", fjson_object_new_string(reply));
+	*rendered = strdup((char*)fjson_object_to_json_string(errRoot));
+
+	req=NULL;
+	fjson_object_put(errRoot);
+
+	finalize_it:
+		fjson_object_put(req);
+		RETiRet;
+}
+
+
+/* write data error request/replies to separate error file
+ * Note: we open the file but never close it before exit. If it
+ * needs to be closed, HUP must be sent.
+ */
+static rsRetVal ATTR_NONNULL()
+writeDataError(wrkrInstanceData_t *const pWrkrData, uchar *const reqmsg)
+{
+	DEFiRet;
+	instanceData *pData = pWrkrData->pData;
+	char *rendered = pWrkrData->reply;
+	size_t toWrite;
+	ssize_t wrRet;
+
+	if(pData->errorFile == NULL) {
+		dbgprintf("omclickhouse: no local error logger defined - "
+		          "ignoring ClickHouse error information\n");
+		FINALIZE;
+	}
+
+
+	if(pData->fdErrFile == -1) {
+		pData->fdErrFile = open((char*)pData->errorFile,
+					O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE|O_CLOEXEC,
+					S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP);
+		if(pData->fdErrFile == -1) {
+			LogError(errno, RS_RET_ERR, "omclickhouse: error opening error file %s",
+				pData->errorFile);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	}
+
+	if(getDataErrorDefault(pWrkrData, pWrkrData->reply, reqmsg, &rendered) != RS_RET_OK) {
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	/* we do not do real error-handling on the err file, as this finally complicates
+	 * things way to much.
+	 */
+	dbgprintf("omclickhouse: message sent: '%s'\n", reqmsg);
+	dbgprintf("omclickhouse: error record: '%s'\n", rendered);
+	toWrite = strlen(rendered) + 1;
+	/* Note: we overwrite the '\0' terminator with '\n' -- so we avoid
+	 * caling malloc() -- write() does NOT need '\0'!
+	 */
+	rendered[toWrite-1] = '\n'; /* NO LONGER A STRING! */
+	wrRet = write(pData->fdErrFile, rendered, toWrite);
+	if(wrRet != (ssize_t) toWrite) {
+		LogError(errno, RS_RET_IO_ERROR,
+			"omclickhouse: error writing error file %s, write returned %lld",
+			pData->errorFile, (long long) wrRet);
+	}
+
+finalize_it:
+	RETiRet;
+}
+
+
+static rsRetVal
+checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg)
+{
+	DEFiRet;
+
+	if(strstr(pWrkrData->reply, "e.what() = DB::Exception") != NULL) {
+		dbgprintf("omclickhouse: action failed with error: %s\n", pWrkrData->reply);
+		iRet = RS_RET_DATAFAIL;
+	}
+
+
+	if(iRet == RS_RET_DATAFAIL) {
+		STATSCOUNTER_INC(indexFail, mutIndexFail);
+		writeDataError(pWrkrData, reqmsg);
+		iRet = RS_RET_OK; /* we have handled the problem! */
+	}
+
+
+	if(iRet != RS_RET_OK) {
+		STATSCOUNTER_INC(indexFail, mutIndexFail);
+	}
+	RETiRet;
+}
+
+static rsRetVal ATTR_NONNULL(1)
+setPostURL(wrkrInstanceData_t *const pWrkrData)
+{
+	char* baseUrl;
+	es_str_t *url;
+	DEFiRet;
+	instanceData *const pData = pWrkrData->pData;
+
+	baseUrl = (char*)pData->serverBaseUrl;
+	url = es_newStrFromCStr(baseUrl, strlen(baseUrl));
+	if (url == NULL) {
+		LogError(0, RS_RET_OUT_OF_MEMORY,
+			"omclickhouse: error allocating new estr for POST url.");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	if(pWrkrData->restURL != NULL)
+		free(pWrkrData->restURL);
+
+	pWrkrData->restURL = (uchar*)es_str2cstr(url, NULL);
+	curl_easy_setopt(pWrkrData->curlPostHandle, CURLOPT_URL, pWrkrData->restURL);
+	dbgprintf("omclickhouse: using REST URL: '%s'\n", pWrkrData->restURL);
+
+finalize_it:
+	if (url != NULL)
+		es_deleteStr(url);
+	RETiRet;
+}
+
+
+/* this method computes the next data set to be added to the batch
+ * returns the expected size of adding the next message into the
+ * batched request to clickhouse
+ */
+static size_t
+computeBulkMessage(const wrkrInstanceData_t *const pWrkrData,
+	const uchar *const message, char **newMessage)
+{
+	size_t r = 0;
+	if(pWrkrData->batch.nmemb != 0) {
+		*newMessage = strchr(strstr((const char*)message, "VALUES"), '(');
+		r = strlen(*newMessage);
+	} else {
+		*newMessage = (char*)message;
+		r = strlen(*newMessage);
+	}
+	dbgprintf("omclickhouse: computeBulkMessage: new message part: %s\n", *newMessage);
+
+	return r;
+}
+
+
+/* This method builds the batch, that will be submitted.
+ */
+static rsRetVal
+buildBatch(wrkrInstanceData_t *pWrkrData, char *message)
+{
+	DEFiRet;
+	int length = strlen(message);
+	int r;
+
+	r = es_addBuf(&pWrkrData->batch.data, message, length);
+	if(r != 0) {
+		LogError(0, RS_RET_ERR, "omclickhouse: growing batch failed with code %d", r);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	++pWrkrData->batch.nmemb;
+	iRet = RS_RET_OK;
+
+finalize_it:
+	RETiRet;
+}
+
+
+static void ATTR_NONNULL()
+initializeBatch(wrkrInstanceData_t *pWrkrData)
+{
+	es_emptyStr(pWrkrData->batch.data);
+	pWrkrData->batch.nmemb = 0;
+}
+
+
+static rsRetVal ATTR_NONNULL(1, 2)
+curlPost(wrkrInstanceData_t *pWrkrData, uchar *message, int msglen, const int nmsgs)
+{
+	CURLcode code;
+	CURL *const curl = pWrkrData->curlPostHandle;
+	char errbuf[CURL_ERROR_SIZE] = "";
+	DEFiRet;
+
+	if(!strstr((char*)message, "INSERT INTO") && !pWrkrData->insertErrorSent) {
+		indexHTTPFail += nmsgs;
+		LogError(0, RS_RET_ERR, "omclickhouse: Message is no Insert query: "
+				"Message suspended: %s", (char*)message);
+		pWrkrData->insertErrorSent = 1;
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	pWrkrData->reply = NULL;
+	pWrkrData->replyLen = 0;
+
+	CHKiRet(setPostURL(pWrkrData));
+
+	curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message);
+	curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, msglen);
+	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
+	code = curl_easy_perform(curl);
+	dbgprintf("curl returned %lld\n", (long long) code);
+	if (code != CURLE_OK && code != CURLE_HTTP_RETURNED_ERROR) {
+		STATSCOUNTER_INC(indexHTTPReqFail, mutIndexHTTPReqFail);
+		indexHTTPFail += nmsgs;
+		LogError(0, RS_RET_SUSPENDED,
+			"omclickhouse: we are suspending ourselfs due "
+			"to server failure %lld: %s", (long long) code, errbuf);
+		ABORT_FINALIZE(RS_RET_SUSPENDED);
+	}
+
+	if(pWrkrData->reply == NULL) {
+		dbgprintf("omclickhouse: pWrkrData reply==NULL, replyLen = '%d'\n",
+			pWrkrData->replyLen);
+		STATSCOUNTER_INC(indexSuccess, mutIndexSuccess);
+	} else {
+		dbgprintf("omclickhouse: pWrkrData replyLen = '%d'\n", pWrkrData->replyLen);
+		if(pWrkrData->replyLen > 0) {
+			pWrkrData->reply[pWrkrData->replyLen] = '\0';
+			/* Append 0 Byte if replyLen is above 0 - byte has been reserved in malloc */
+		}
+		dbgprintf("omclickhouse: pWrkrData reply: '%s'\n", pWrkrData->reply);
+		CHKiRet(checkResult(pWrkrData, message));
+	}
+
+finalize_it:
+	free(pWrkrData->reply);
+	pWrkrData->reply = NULL; /* don't leave dangling pointer */
+	RETiRet;
+}
+
+
+static rsRetVal
+submitBatch(wrkrInstanceData_t *pWrkrData)
+{
+	char *cstr = NULL;
+	DEFiRet;
+
+	cstr = es_str2cstr(pWrkrData->batch.data, NULL);
+	dbgprintf("omclickhouse: submitBatch, batch: '%s'\n", cstr);
+
+	CHKiRet(curlPost(pWrkrData, (uchar*) cstr, strlen(cstr), pWrkrData->batch.nmemb));
+
+finalize_it:
+	free(cstr);
+	RETiRet;
+}
+
+
+BEGINbeginTransaction
+CODESTARTbeginTransaction
+	if(!pWrkrData->pData->bulkmode) {
+		FINALIZE;
+	}
+
+	initializeBatch(pWrkrData);
+finalize_it:
+ENDbeginTransaction
+
+
+BEGINdoAction
+	char *batchPart = NULL;
+CODESTARTdoAction
+	dbgprintf("CODESTARTdoAction: entered\n");
+	STATSCOUNTER_INC(indexSubmit, mutIndexSubmit);
+
+	if(pWrkrData->pData->bulkmode) {
+		const size_t nBytes = computeBulkMessage(pWrkrData, ppString[0], &batchPart);
+		dbgprintf("pascal: doAction: message: %s\n", batchPart);
+
+		/* If max bytes is set and this next message will put us over the limit,
+		* submit the current buffer and reset */
+		if(pWrkrData->pData->maxbytes > 0
+			&& es_strlen(pWrkrData->batch.data) + nBytes > pWrkrData->pData->maxbytes) {
+
+			dbgprintf("omclickhouse: maxbytes limit reached, submitting partial "
+				"batch of %d elements.\n", pWrkrData->batch.nmemb);
+			CHKiRet(submitBatch(pWrkrData));
+			initializeBatch(pWrkrData);
+			batchPart = (char*)ppString[0];
+		}
+
+		CHKiRet(buildBatch(pWrkrData, batchPart));
+
+		iRet = pWrkrData->batch.nmemb == 1 ? RS_RET_PREVIOUS_COMMITTED : RS_RET_DEFER_COMMIT;
+	} else {
+		CHKiRet(curlPost(pWrkrData, ppString[0], strlen((char*)ppString[0]), 1));
+	}
+finalize_it:
+ENDdoAction
+
+
+BEGINendTransaction
+CODESTARTendTransaction
+/* End Transaction only if batch data is not empty */
+	if (pWrkrData->batch.data != NULL && pWrkrData->batch.nmemb > 0) {
+		CHKiRet(submitBatch(pWrkrData));
+	} else {
+		dbgprintf("omclickhouse: endTransaction, pWrkrData->batch.data is NULL, "
+			"nothing to send. \n");
+	}
+finalize_it:
+ENDendTransaction
+
+static void ATTR_NONNULL()
+setInstParamDefaults(instanceData *const pData)
+{
+	pData->serverBaseUrl = NULL;
+	pData->port = 8123;
+	pData->user = NULL;
+	pData->pwd = NULL;
+	pData->authBuf = NULL;
+	pData->tplName = NULL;
+	pData->useHttps = 1;
+	pData->errorFile = NULL;
+	pData->bulkmode = 1;
+	pData->maxbytes = 104857600; //100MB
+	pData->caCertFile = NULL;
+	pData->myCertFile = NULL;
+	pData->myPrivKeyFile = NULL;
+}
+
+/* POST result string ... useful for debugging */
+static size_t
+curlResult(void *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	char *p = (char *)ptr;
+	wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t*) userdata;
+	char *buf;
+	size_t newlen;
+	newlen = pWrkrData->replyLen + size*nmemb;
+	if((buf = realloc(pWrkrData->reply, newlen + 1)) == NULL) {
+		LogError(errno, RS_RET_ERR, "omclickhouse: realloc failed in curlResult");
+		return 0; /* abort due to failure */
+	}
+	memcpy(buf+pWrkrData->replyLen, p, size*nmemb);
+	pWrkrData->replyLen = newlen;
+	pWrkrData->reply = buf;
+	return size*nmemb;
+}
+
+static void ATTR_NONNULL()
+curlSetupCommon(wrkrInstanceData_t *const pWrkrData, CURL *const handle)
+{
+	curl_easy_setopt(handle, CURLOPT_HTTPHEADER, pWrkrData->curlHeader);
+	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, TRUE);
+	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult);
+	curl_easy_setopt(handle, CURLOPT_WRITEDATA, pWrkrData);
+	if(pWrkrData->pData->authBuf != NULL) {
+		curl_easy_setopt(handle, CURLOPT_USERPWD, pWrkrData->pData->authBuf);
+		curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
+	}
+
+	if(pWrkrData->pData->caCertFile) {
+		curl_easy_setopt(handle, CURLOPT_CAINFO, pWrkrData->pData->caCertFile);
+	}
+	if(pWrkrData->pData->myCertFile) {
+		curl_easy_setopt(handle, CURLOPT_SSLCERT, pWrkrData->pData->myCertFile);
+	}
+	if(pWrkrData->pData->myPrivKeyFile) {
+		curl_easy_setopt(handle, CURLOPT_SSLKEY, pWrkrData->pData->myPrivKeyFile);
+	}
+	/* uncomment for in-dept debuggung:
+	curl_easy_setopt(handle, CURLOPT_VERBOSE, TRUE); */
+}
+
+static void ATTR_NONNULL(1)
+curlPostSetup(wrkrInstanceData_t *const pWrkrData)
+{
+	curlSetupCommon(pWrkrData, pWrkrData->curlPostHandle);
+	curl_easy_setopt(pWrkrData->curlPostHandle, CURLOPT_POST, 1);
+}
+
+#define CONTENT_JSON "Content-Type: application/json; charset=utf-8"
+
+static rsRetVal ATTR_NONNULL()
+curlSetup(wrkrInstanceData_t *const pWrkrData)
+{
+	DEFiRet;
+	pWrkrData->curlHeader = curl_slist_append(NULL, CONTENT_JSON);
+	CHKmalloc(pWrkrData->curlPostHandle = curl_easy_init());
+	curlPostSetup(pWrkrData);
+
+finalize_it:
+	if(iRet != RS_RET_OK && pWrkrData->curlPostHandle != NULL) {
+		curl_easy_cleanup(pWrkrData->curlPostHandle);
+		pWrkrData->curlPostHandle = NULL;
+	}
+	RETiRet;
+}
+
+static rsRetVal
+computeAuthHeader(char* user, char* pwd, uchar** authBuf)
+{
+	DEFiRet;
+	int r;
+
+	es_str_t* auth = es_newStr(1024);
+	if (auth == NULL) {
+		LogError(0, RS_RET_OUT_OF_MEMORY,
+			"omclickhouse: failed to allocate es_str auth for auth header construction");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	r = es_addBuf(&auth, user, strlen(user));
+	if(r == 0)
+		r = es_addChar(&auth, ':');
+	if(r == 0 && pwd != NULL)
+		r = es_addBuf(&auth, pwd, strlen(pwd));
+	if(r == 0)
+		*authBuf = (uchar*) es_str2cstr(auth, NULL);
+
+	if (r != 0 || *authBuf == NULL) {
+		LogError(0, RS_RET_ERR, "omclickhouse: failed to build auth header\n");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+finalize_it:
+	if (auth != NULL)
+		es_deleteStr(auth);
+	RETiRet;
+}
+
+/* Build basic URL part, which includes hostname, port, user, pwd as follows:
+ * http://user:pwd@hostname:port/ based on a server param
+ * Newly creates a cstr for this purpose.
+ */
+static rsRetVal
+computeBaseUrl(const char* server, const int port, const sbool useHttps, instanceData *pData)
+{
+#	define SCHEME_HTTPS "https://"
+#	define SCHEME_HTTP "http://"
+
+	char portBuf[64];
+	int r = 0;
+	const char *host = server;
+	DEFiRet;
+
+	assert(server[strlen(server)-1] != '/');
+
+	es_str_t *urlBuf = es_newStr(256);
+	if (urlBuf == NULL) {
+		LogError(0, RS_RET_OUT_OF_MEMORY,
+		"omclickhouse: failed to allocate es_str urlBuf in computeBaseUrl");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	/* Find where the hostname/ip of the server starts. If the scheme is not specified
+	  in the uri, start the buffer with a scheme corresponding to the useHttps parameter.
+	*/
+	if(strcasestr(server, SCHEME_HTTP)) {
+		host = server + strlen(SCHEME_HTTP);
+	} else if(strcasestr(server, SCHEME_HTTPS)) {
+		host = server + strlen(SCHEME_HTTPS);
+	} else {
+		r = useHttps ? es_addBuf(&urlBuf, SCHEME_HTTPS, sizeof(SCHEME_HTTPS)-1) :
+			es_addBuf(&urlBuf, SCHEME_HTTP, sizeof(SCHEME_HTTP)-1);
+	}
+	if (r == 0)
+		r = es_addBuf(&urlBuf, (char *)server, strlen(server));
+	if (r == 0 && !strchr(host, ':')) {
+		snprintf(portBuf, sizeof(portBuf), ":%d", port);
+		r = es_addBuf(&urlBuf, portBuf, strlen(portBuf));
+	}
+	if (r == 0)
+		r = es_addChar(&urlBuf, '/');
+	if (r == 0)
+		pData->serverBaseUrl = (uchar*) es_str2cstr(urlBuf, NULL);
+
+	if (r != 0 || pData->serverBaseUrl == NULL) {
+		LogError(0, RS_RET_ERR, "omclickhouse: error occurred computing baseUrl from "
+			"server %s", server);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+finalize_it:
+	if (urlBuf) {
+		es_deleteStr(urlBuf);
+	}
+	RETiRet;
+}
+
+BEGINnewActInst
+	struct cnfparamvals *pvals;
+	uchar *server = NULL;
+	int i;
+	FILE *fp;
+	char errStr[1024];
+CODESTARTnewActInst
+	if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) {
+		ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
+	}
+
+	CHKiRet(createInstance(&pData));
+	setInstParamDefaults(pData);
+
+	for(i = 0 ; i < actpblk.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(actpblk.descr[i].name, "server")) {
+			server = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "port")) {
+			pData->port = (int) pvals[i].val.d.n;
+		} else if(!strcmp(actpblk.descr[i].name, "user")) {
+			pData->user = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "pwd")) {
+			pData->pwd = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "template")) {
+			pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "usehttps")) {
+			pData->useHttps = pvals[i].val.d.n;
+		} else if(!strcmp(actpblk.descr[i].name, "errorfile")) {
+			pData->errorFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(actpblk.descr[i].name, "bulkmode")) {
+			pData->bulkmode = pvals[i].val.d.n;
+		} else if(!strcmp(actpblk.descr[i].name, "maxbytes")) {
+			pData->maxbytes = (size_t) pvals[i].val.d.n;
+		} else if(!strcmp(actpblk.descr[i].name, "tls.cacert")) {
+			pData->caCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+			fp = fopen((const char*)pData->caCertFile, "r");
+			if(fp == NULL) {
+				rs_strerror_r(errno, errStr, sizeof(errStr));
+				LogError(0, RS_RET_NO_FILE_ACCESS,
+					"error: omclickhouse: 'tls.cacert' file %s couldn't be accessed: %s\n",
+						pData->caCertFile, errStr);
+			} else {
+				fclose(fp);
+			}
+		} else if(!strcmp(actpblk.descr[i].name, "tls.mycert")) {
+			pData->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+			fp = fopen((const char*)pData->myCertFile, "r");
+			if(fp == NULL) {
+				rs_strerror_r(errno, errStr, sizeof(errStr));
+				LogError(0, RS_RET_NO_FILE_ACCESS,
+					"error: omclickhouse: 'tls.mycert' file %s couldn't be accessed: %s\n",
+						pData->myCertFile, errStr);
+			} else {
+				fclose(fp);
+			}
+		} else if(!strcmp(actpblk.descr[i].name, "tls.myprivkey")) {
+			pData->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
+			fp = fopen((const char*)pData->myPrivKeyFile, "r");
+			if(fp == NULL) {
+				rs_strerror_r(errno, errStr, sizeof(errStr));
+				LogError(0, RS_RET_NO_FILE_ACCESS,
+					"error: omclickhouse: 'tls.myprivkey' file %s couldn't be accessed: %s\n",
+						pData->myPrivKeyFile, errStr);
+			} else {
+				fclose(fp);
+			}
+		} else {
+			LogError(0, RS_RET_INTERNAL_ERROR, "omclickhouse: program error, "
+				"non-handled param '%s'", actpblk.descr[i].name);
+		}
+	}
+
+
+	if(pData->user == NULL && pData->pwd != NULL) {
+		LogMsg(0, RS_RET_OK, LOG_WARNING, "omclickhouse: No user was specified "
+				"but a password was given.");
+	}
+
+	if(pData->user != NULL)
+		CHKiRet(computeAuthHeader((char*) pData->user, (char*) pData->pwd, &pData->authBuf));
+
+	CODE_STD_STRING_REQUESTnewActInst(1)
+	CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*)strdup((pData->tplName == NULL) ?
+		" StdClickHouseFmt" : (char*)pData->tplName), OMSR_RQD_TPL_OPT_SQL));
+
+	if(server != NULL) {
+		CHKiRet(computeBaseUrl((const char*)server, pData->port, pData->useHttps, pData));
+	} else {
+		LogMsg(0, RS_RET_OK, LOG_WARNING,
+			"omclickhouse: No servers specified, using localhost");
+		CHKiRet(computeBaseUrl("localhost", pData->port, pData->useHttps,
+					pData));
+	}
+
+	/* node created, let's add to list of instance configs for the module */
+	if(loadModConf->tail == NULL) {
+		loadModConf->tail = loadModConf->root = pData;
+	} else {
+		loadModConf->tail->next = pData;
+		loadModConf->tail = pData;
+	}
+
+CODE_STD_FINALIZERnewActInst
+	free(server);
+	cnfparamvalsDestruct(pvals, &actpblk);
+ENDnewActInst
+
+
+BEGINbeginCnfLoad
+CODESTARTbeginCnfLoad
+	loadModConf = pModConf;
+	pModConf->pConf = pConf;
+	pModConf->root = pModConf->tail = NULL;
+ENDbeginCnfLoad
+
+
+BEGINendCnfLoad
+CODESTARTendCnfLoad
+	loadModConf = NULL; /* done loading */
+ENDendCnfLoad
+
+
+BEGINcheckCnf
+CODESTARTcheckCnf
+ENDcheckCnf
+
+
+BEGINactivateCnf
+CODESTARTactivateCnf
+ENDactivateCnf
+
+
+BEGINfreeCnf
+CODESTARTfreeCnf
+ENDfreeCnf
+
+
+BEGINdoHUP
+CODESTARTdoHUP
+	if(pData->fdErrFile != -1) {
+		close(pData->fdErrFile);
+		pData->fdErrFile = -1;
+	}
+ENDdoHUP
+
+
+BEGINmodExit
+CODESTARTmodExit
+	curl_global_cleanup();
+	statsobj.Destruct(&indexStats);
+	objRelease(statsobj, CORE_COMPONENT);
+	objRelease(prop, CORE_COMPONENT);
+	objRelease(ruleset, CORE_COMPONENT);
+ENDmodExit
+
+NO_LEGACY_CONF_parseSelectorAct
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_OMOD_QUERIES
+CODEqueryEtryPt_STD_OMOD8_QUERIES
+CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES
+CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES
+CODEqueryEtryPt_doHUP
+CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */
+CODEqueryEtryPt_STD_CONF2_QUERIES
+ENDqueryEtryPt
+
+
+BEGINmodInit()
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(statsobj, CORE_COMPONENT));
+	CHKiRet(objUse(prop, CORE_COMPONENT));
+	CHKiRet(objUse(ruleset, CORE_COMPONENT));
+
+	if (curl_global_init(CURL_GLOBAL_ALL) != 0) {
+		LogError(0, RS_RET_OBJ_CREATION_FAILED, "CURL fail. -indexing disabled");
+		ABORT_FINALIZE(RS_RET_OBJ_CREATION_FAILED);
+	}
+
+	/* support statistics gathering */
+	CHKiRet(statsobj.Construct(&indexStats));
+	CHKiRet(statsobj.SetName(indexStats, (uchar *)"omclickhouse"));
+	CHKiRet(statsobj.SetOrigin(indexStats, (uchar *)"omclickhouse"));
+	STATSCOUNTER_INIT(indexSubmit, mutIndexSubmit);
+	CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"submitted",
+		ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexSubmit));
+	STATSCOUNTER_INIT(indexHTTPFail, mutIndexHTTPFail);
+	CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.http",
+		ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexHTTPFail));
+	STATSCOUNTER_INIT(indexHTTPReqFail, mutIndexHTTPReqFail);
+	CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.httprequests",
+		ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexHTTPReqFail));
+	STATSCOUNTER_INIT(indexFail, mutIndexFail);
+	CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"failed.clickhouse",
+		ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexFail));
+	STATSCOUNTER_INIT(indexSuccess, mutIndexSuccess);
+	CHKiRet(statsobj.AddCounter(indexStats, (uchar *)"response.success",
+		ctrType_IntCtr, CTR_FLAG_RESETTABLE, &indexSuccess));
+	CHKiRet(statsobj.ConstructFinalize(indexStats));
+
+ENDmodInit
+
+/* vi:set ai:
+ */
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@ -111,6 +111,9 @@ static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg:::json%\\\",\\
 "%HOSTNAME:::json%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\""
 "%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\""
 "%timegenerated:::date-rfc3339%\\\"}\"";
+static uchar template_StdClickHouseFmt[] = "\"INSERT INTO rsyslog.SystemEvents (severity, facility, "
+"timestamp, hostname, tag, message) VALUES (%syslogseverity%, %syslogfacility%, "
+"'%timereported:::date-unixtimestamp%', '%hostname%', '%syslogtag%', '%msg%')\"";
 /* end templates */

 /* tables for interfacing with the v6 config system (as far as we need to) */
@ -1247,6 +1250,8 @@ initLegacyConf(void)
 	tplAddLine(ourConf, " StdPgSQLFmt", &pTmp);
 	pTmp = template_StdJSONFmt;
 	tplAddLine(ourConf, " StdJSONFmt", &pTmp);
+	pTmp = template_StdClickHouseFmt;
+	tplAddLine(ourConf, " StdClickHouseFmt", &pTmp);
 	pTmp = template_spoofadr;
 	tplLastStaticInit(ourConf, tplAddLine(ourConf, "RSYSLOG_omudpspoofDfltSourceTpl", &pTmp));

--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@ -405,6 +405,49 @@ TESTS +=  \
 	mmanon_random_cons_128_ipembedded.sh
 endif # if ENABLE_MMANON

+if ENABLE_CLICKHOUSE_TESTS
+TESTS +=  \
+	clickhouse-start.sh \
+	clickhouse-basic.sh \
+	clickhouse-load.sh \
+	clickhouse-bulk.sh \
+	clickhouse-bulk-load.sh \
+	clickhouse-limited-batch.sh \
+	clickhouse-select.sh \
+	clickhouse-errorfile.sh \
+	clickhouse-wrong-quotation-marks.sh \
+	clickhouse-wrong-template-option.sh \
+	clickhouse-wrong-insert-syntax.sh
+
+clickhouse-basic.log: clickhouse-start.log
+clickhouse-load.log: clickhouse-basic.log
+clickhouse-bulk.log: clickhouse-load.log
+clickhouse-bulk-load.log: clickhouse-bulk.log
+clickhouse-limited-batch.log: clickhouse-bulk-load.log
+clickhouse-select.log: clickhouse-limited-batch.log
+clickhouse-errorfile.log: clickhouse-select.log
+clickhouse-wrong-quotation-marks.log: clickhouse-errorfile.log
+clickhouse-wrong-template-option.log: clickhouse-wrong-quotation-marks.log
+clickhouse-wrong-insert-syntax.log: clickhouse-wrong-template-option.log
+
+if HAVE_VALGRIND
+TESTS +=  \
+	clickhouse-basic-vg.sh \
+	clickhouse-load-vg.sh \
+	clickhouse-bulk-vg.sh \
+	clickhouse-bulk-load-vg.sh
+
+clickhouse-basic-vg.log: clickhouse-wrong-insert-syntax.log
+clickhouse-load-vg.log: clickhouse-basic-vg.log
+clickhouse-bulk-vg.log: clickhouse-load-vg.log
+clickhouse-bulk-load-vg.log: clickhouse-bulk-vg.log
+endif # VALGRIND
+
+TESTS += clickhouse-stop.sh
+endif # CLICKHOUSE_TESTS
+
+
+
 if ENABLE_ELASTICSEARCH_TESTS
 TESTS +=  \
 	es-basic-server.sh \
@ -1402,6 +1445,22 @@ EXTRA_DIST= \
 	incltest_dir_empty_wildcard.sh \
 	incltest_dir_wildcard.sh \
 	testsuites/es.yml \
+	clickhouse-start.sh \
+	clickhouse-stop.sh \
+	clickhouse-basic.sh \
+	clickhouse-load.sh \
+	clickhouse-bulk.sh \
+	clickhouse-bulk-load.sh \
+	clickhouse-limited-batch.sh \
+	clickhouse-select.sh \
+	clickhouse-wrong-quotation-marks.sh \
+	clickhouse-wrong-template-option.sh \
+	clickhouse-errorfile.sh \
+	clickhouse-wrong-insert-syntax.sh \
+	clickhouse-basic-vg.sh \
+	clickhouse-load-vg.sh \
+	clickhouse-bulk-vg.sh \
+	clickhouse-bulk-load-vg.sh \
 	es_response_get_msgnum.py \
 	elasticsearch-error-format-check.py \
 	es-basic-es6.0.sh \
@ -1919,6 +1978,7 @@ EXTRA_DIST= \
 	pgsql-template-cnf6-vg.sh \
 	pgsql-actq-mt-withpause-vg.sh \
 	../devtools/prep-mysql-db.sh \
+	../devtools/prepare_clickhouse.sh \
 	mmkubernetes-basic.sh \
 	mmkubernetes-basic-vg.sh \
 	mmkubernetes_test_server.py \
--- a/tests/clickhouse-basic-vg.sh
+++ b/tests/clickhouse-basic-vg.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/clickhouse-basic.sh
--- a/tests/clickhouse-basic.sh
+++ b/tests/clickhouse-basic.sh
@ -0,0 +1,32 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.basic (id, severity, facility, timestamp, ipaddress, tag, message) VALUES (%msg:F,58:2%, %syslogseverity%, %syslogfacility%, '
+add_conf "'%timereported:::date-unixtimestamp%', '%fromhost-ip%', '%syslogtag%', '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off" bulkmode="off"
+					user="default" pwd="" template="outfmt")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.basic ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<130>Mar 10 01:00:00 172.20.245.8 tag: msgnum:00000001\""
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT id, severity, facility, ipaddress, tag, message FROM rsyslog.basic" > $RSYSLOG_OUT_LOG
+
+export EXPECTED='1	2	16	127.0.0.1	tag:	 msgnum:00000001'
+cmp_exact $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.basic"
+exit_test
--- a/tests/clickhouse-bulk-load-vg.sh
+++ b/tests/clickhouse-bulk-load-vg.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/clickhouse-bulk-load.sh
--- a/tests/clickhouse-bulk-load.sh
+++ b/tests/clickhouse-bulk-load.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+export NUMMESSAGES=100000
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.bulkLoad (id, ipaddress, message) VALUES (%msg:F,58:2%, '
+add_conf "'%fromhost-ip%', '%msg:F,58:2%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.bulkLoad ( id Int32, ipaddress String, message String ) ENGINE = MergeTree() PARTITION BY ipaddress Order By id"
+
+startup
+tcpflood -m $NUMMESSAGES
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT message FROM rsyslog.bulkLoad ORDER BY id" > $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.bulkLoad"
+seq_check  0 $(( NUMMESSAGES - 1 ))
+
+exit_test
--- a/tests/clickhouse-bulk-vg.sh
+++ b/tests/clickhouse-bulk-vg.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/clickhouse-bulk.sh
--- a/tests/clickhouse-bulk.sh
+++ b/tests/clickhouse-bulk.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.bulk (id, severity, facility, timestamp, ipaddress, tag, message) VALUES (%msg:F,58:2%, %syslogseverity%, %syslogfacility%, '
+add_conf "'%timereported:::date-unixtimestamp%', '%fromhost-ip%', '%syslogtag%', '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.bulk ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m10
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT id, severity, facility, ipaddress, tag, message FROM rsyslog.bulk ORDER BY id" > $RSYSLOG_OUT_LOG
+
+export EXPECTED='0	7	20	127.0.0.1	tag	 msgnum:00000000:
+1	7	20	127.0.0.1	tag	 msgnum:00000001:
+2	7	20	127.0.0.1	tag	 msgnum:00000002:
+3	7	20	127.0.0.1	tag	 msgnum:00000003:
+4	7	20	127.0.0.1	tag	 msgnum:00000004:
+5	7	20	127.0.0.1	tag	 msgnum:00000005:
+6	7	20	127.0.0.1	tag	 msgnum:00000006:
+7	7	20	127.0.0.1	tag	 msgnum:00000007:
+8	7	20	127.0.0.1	tag	 msgnum:00000008:
+9	7	20	127.0.0.1	tag	 msgnum:00000009:'
+cmp_exact $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.bulk"
+exit_test
--- a/tests/clickhouse-errorfile.sh
+++ b/tests/clickhouse-errorfile.sh
@ -0,0 +1,31 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.errorfile (id, severity, facility, timestamp, ipaddress, tag, message) VALUES (%msg:F,58:2%, %syslogseverity%, %syslogfacility%, '
+add_conf "'%timereported:::date-unixtimestamp%', '%fromhost-ip%', '%syslogtag%', '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt"
+					bulkmode="off" errorfile="'$RSYSLOG_OUT_LOG'")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.errorfile ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag: msgnum:NoInteger\""
+shutdown_when_empty
+wait_shutdown
+
+export EXPECTED="{ \"request\": { \"url\": \"http:\\/\\/localhost:8123\\/\", \"postdata\": \"INSERT INTO rsyslog.errorfile (id, severity, facility, timestamp, ipaddress, tag, message) VALUES (NoInteger, 1, 16, '1520643600', '127.0.0.1', 'tag:', ' msgnum:NoInteger')\" }, \"reply\": \"Code: 47, e.displayText() = DB::Exception: Unknown identifier: NoInteger, e.what() = DB::Exception\\n\" }"
+cmp_exact $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.errorfile"
+exit_test
--- a/tests/clickhouse-limited-batch.sh
+++ b/tests/clickhouse-limited-batch.sh
@ -0,0 +1,34 @@
+#!/bin/bash
+# add 2018-12-20 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+export NUMMESSAGES=100000
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.limited (id, ipaddress, message) VALUES (%msg:F,58:2%, '
+add_conf "'%fromhost-ip%', '%msg:F,58:2%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt"
+					maxbytes="1k")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.limited ( id Int32, ipaddress String, message String ) ENGINE = MergeTree() PARTITION BY ipaddress Order By id"
+
+startup
+tcpflood -m $NUMMESSAGES
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT message FROM rsyslog.limited ORDER BY id" > $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.limited"
+seq_check  0 $(( NUMMESSAGES - 1 ))
+
+exit_test
--- a/tests/clickhouse-load-vg.sh
+++ b/tests/clickhouse-load-vg.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/clickhouse-load.sh
--- a/tests/clickhouse-load.sh
+++ b/tests/clickhouse-load.sh
@ -0,0 +1,34 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+export NUMMESSAGES=1000
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.load (id, ipaddress, message) VALUES (%msg:F,58:2%, '
+add_conf "'%fromhost-ip%', '%msg:F,58:2%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt"
+					bulkmode="off")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.load ( id Int32, ipaddress String, message String ) ENGINE = MergeTree() PARTITION BY ipaddress Order By id"
+
+startup
+tcpflood -m $NUMMESSAGES
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT message FROM rsyslog.load ORDER BY id" > $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.load"
+seq_check  0 $(( NUMMESSAGES - 1 ))
+
+exit_test
--- a/tests/clickhouse-select.sh
+++ b/tests/clickhouse-select.sh
@ -0,0 +1,29 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+template(name="outfmt" option.stdsql="on" type="string" string="SELECT * FROM rsyslog.select")
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					bulkmode="off" user="default" pwd=""
+					template="outfmt")
+
+action(type="omfile" file="'$RSYSLOG_OUT_LOG'")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.select ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag: msgnum:00000001\""
+shutdown_when_empty
+wait_shutdown
+
+content_check "omclickhouse: Message is no Insert query: Message suspended: SELECT * FROM rsyslog.select"
+
+clickhouse-client --query="DROP TABLE rsyslog.select"
+exit_test
--- a/tests/clickhouse-start.sh
+++ b/tests/clickhouse-start.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+# This is not a real test, but a script to start clickhouse. It is
+# implemented as test so that we can start clickhouse at the time we need
+# it (do so via Makefile.am).
+# Copyright (C) 2018 Pascal Withopf and Adiscon GmbH
+# Released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+if [ "$CLICKHOUSE_START_CMD" == "" ]; then
+	exit_test # no start needed
+fi
+
+test_error_exit_handler() {
+	printf 'clickhouse startup failed, log is:\n'
+	$SUDO cat /var/log/clickhouse-server/clickhouse-server.err.log
+}
+
+printf 'starting clickhouse...\n'
+$CLICKHOUSE_START_CMD &
+sleep 10
+#wait_startup_pid /var/run/clickhouse-server/clickhouse-server.pid
+printf 'preparing clickhouse for testbench use...\n'
+$SUDO ${srcdir}/../devtools/prepare_clickhouse.sh
+printf 'done, clickhouse ready for testbench\n'
+exit_test
--- a/tests/clickhouse-stop.sh
+++ b/tests/clickhouse-stop.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+# This is not a real test, but a script to stop mysql. It is
+# implemented as test so that we can stop mysql at the time we need
+# it (do so via Makefile.am).
+# Copyright (C) 2018 Pascal Withopf and Adiscon GmbH
+# Released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+if [ "$CLICKHOUSE_STOP_CMD" == "" ]; then
+	exit_test
+fi
+
+clickhouse-client --query="DROP DATABASE rsyslog"
+sleep 1
+printf 'stopping clickhouse...\n'
+#$SUDO sed -n -r 's/PID: ([0-9]+\.*)/\1/p' /var/lib/clickhouse/status > /tmp/clickhouse-server.pid
+#$SUDO kill $($SUDO sed -n -r 's/PID: ([0-9]+\.*)/\1/p' /var/lib/clickhouse/status)
+eval $CLICKHOUSE_STOP_CMD
+sleep 1 # cosmetic: give clickhouse a chance to emit shutdown message
+exit_test
--- a/tests/clickhouse-wrong-insert-syntax.sh
+++ b/tests/clickhouse-wrong-insert-syntax.sh
@ -0,0 +1,31 @@
+#!/bin/bash
+# add 2018-12-07 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.wrongInsertSyntax (id, severity, facility, timestamp, ipaddress, tag, message) VLUES (%msg:F,58:2%, %syslogseverity%, %syslogfacility%, '
+add_conf "'%timereported:::date-unixtimestamp%', '%fromhost-ip%', '%syslogtag%', '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost" usehttps="off"
+					user="default" pwd="" template="outfmt"
+					bulkmode="off" errorfile="'$RSYSLOG_OUT_LOG'")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.wrongInsertSyntax ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag: msgnum:00000001\""
+shutdown_when_empty
+wait_shutdown
+
+export EXPECTED="{ \"request\": { \"url\": \"http:\\/\\/localhost:8123\\/\", \"postdata\": \"INSERT INTO rsyslog.wrongInsertSyntax (id, severity, facility, timestamp, ipaddress, tag, message) VLUES (00000001, 1, 16, '1520643600', '127.0.0.1', 'tag:', ' msgnum:00000001')\" }, \"reply\": \"Code: 62, e.displayText() = DB::Exception: Syntax error: failed at position 100: VLUES (00000001, 1, 16, '1520643600', '127.0.0.1', 'tag:', ' msgnum:00000001'). Expected one of: FORMAT, VALUES, SELECT, WITH, e.what() = DB::Exception\\n\" }"
+cmp_exact $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.wrongInsertSyntax"
+exit_test
--- a/tests/clickhouse-wrong-quotation-marks.sh
+++ b/tests/clickhouse-wrong-quotation-marks.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+# add 2018-12-19 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" option.stdsql="on" type="string" string="INSERT INTO rsyslog.quotation (id, severity, message) VALUES ( 1, '
+add_conf "%syslogseverity%, '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost"
+					usehttps="off" bulkmode="off"
+					user="default" pwd="" template="outfmt")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.quotation ( id Int32, severity Int8, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<130>Mar 10 01:00:00 172.20.245.8 tag:it's a test\""
+shutdown_when_empty
+wait_shutdown
+clickhouse-client --query="SELECT id, severity, message FROM rsyslog.quotation" > $RSYSLOG_OUT_LOG
+
+export EXPECTED="1	2	it\\'s a test"
+cmp_exact $RSYSLOG_OUT_LOG
+
+clickhouse-client --query="DROP TABLE rsyslog.quotation"
+exit_test
--- a/tests/clickhouse-wrong-template-option.sh
+++ b/tests/clickhouse-wrong-template-option.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+# add 2018-12-19 by Pascal Withopf, released under ASL 2.0
+. ${srcdir:=.}/diag.sh init
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/omclickhouse/.libs/omclickhouse")
+input(type="imtcp" port="'$TCPFLOOD_PORT'")
+
+
+template(name="outfmt" type="string" string="INSERT INTO rsyslog.template (id, severity, facility, timestamp, ipaddress, tag, message) VALUES (%msg:F,58:2%, %syslogseverity%, %syslogfacility%, '
+add_conf "'%timereported:::date-unixtimestamp%', '%fromhost-ip%', '%syslogtag%', '%msg%')"
+add_conf '")
+
+
+:syslogtag, contains, "tag" action(type="omclickhouse" server="localhost"
+					usehttps="off" bulkmode="off"
+					user="default" pwd="" template="outfmt")
+
+action(type="omfile" file="'$RSYSLOG_OUT_LOG'")
+'
+
+clickhouse-client --query="CREATE TABLE IF NOT EXISTS rsyslog.template ( id Int32, severity Int8, facility Int8, timestamp DateTime, ipaddress String, tag String, message String ) ENGINE = MergeTree() PARTITION BY severity Order By id"
+
+startup
+tcpflood -m1 -M "\"<130>Mar 10 01:00:00 172.20.245.8 tag: msgnum:00000001\""
+shutdown_when_empty
+wait_shutdown
+
+content_check "you have to specify the SQL or stdSQL option in your template!"
+
+clickhouse-client --query="DROP TABLE rsyslog.template"
+exit_test