Merge branch 'v8-stable' into v8-stable-es-fix

Conflicts:
	ChangeLog

ChangeLog

@@ -1,11 +1,24 @@
 ---------------------------------------------------------------------------
 Version 8.4.1 [v8-stable] 2014-08-??
+- bugfix: ompgsql: don't loose uncomitted data on retry
+  Thanks to Jared Johnson and Axel Rau for the patch.
+- bugfix: imfile: if a state file for a different file name was set,
+  that different file (name) was monitored instead of the configured
+  one. Now, the state file is deleted and the correct file monitored.
+  closes: https://github.com/rsyslog/rsyslog/issues/103
+- bugfix: omudpspoof: source port was invalid
+  Thanks to Pavel Levshin for the patch
 - bugfix: build failure on systems which don't have json_tokener_errors
   Older versions of json-c need to use a different API (which don't exists
   on newer versions, unfortunately...)
   Thanks to Thomas D. for reporting this problem.
 - bugfix: omelasticsearch does not work with broken/changed ES 1.0+ API
   closes: https://github.com/rsyslog/rsyslog/issues/104
+- bugfix: mmanon did not properly anonymize IP addresses starting with '9'
+  Thanks to defa-at-so36.net for reporting this problem.
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=529
+- permit at-sign in variable names
+  closes: https://github.com/rsyslog/rsyslog/issues/110
 ---------------------------------------------------------------------------
 Version 8.4.0 [v8-stable] 2014-08-18
 - this is the new stable branch, which incorporates all enhancements of
@@ -425,6 +438,10 @@ Version 7.6.4 [v7.6-stable] 2014-03-??
   * dirgroupnum
   Thanks to Karol Jurak for the patch.
 - bugfix: memory leak in TCP TLS mode
+- bugfix: imfile: if a state file for a different file name was set,
+  that different file (name) was monitored instead of the configured
+  one. Now, the state file is deleted and the correct file monitored.
+  closes: https://github.com/rsyslog/rsyslog/issues/103
 - bugfix: using UUID property could cause segfault
 - bugfix: mmutf8fix did not detect two invalid sequences
   Thanks to Axel Rau for the patch.
@@ -462,6 +479,9 @@ Version 7.6.4 [v7.6-stable] 2014-03-??
   This was a regression introduced some time in the past in order to
   support API changes in json-c. Now we check for the version and use
   proper code.
+- bugfix: mmanon did not properly anonymize IP addresses starting with '9'
+  Thanks to defa-at-so36.net for reporting this problem.
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=529
 ---------------------------------------------------------------------------
 Version 7.6.3 [v7.6-stable] 2014-03-27
 - add capability to override GnuTLS path in build process

grammar/lexer.l

@@ -133,7 +133,7 @@ int fileno(FILE *stream);
 <EXPR>0[0-7]+ |			/* octal number */
 <EXPR>0x[0-7a-f] |		/* hex number, following rule is dec; strtoll handles all! */
 <EXPR>([1-9][0-9]*|0)		{ yylval.n = strtoll(yytext, NULL, 0); return NUMBER; }
-<EXPR>\$[$!./]{0,1}[a-z][!a-z0-9\-_\.]*	{ yylval.s = strdup(yytext+1); return VAR; }
+<EXPR>\$[$!./]{0,1}[@a-z][!@a-z0-9\-_\.]*	{ yylval.s = strdup(yytext+1); return VAR; }
 <EXPR>\'([^'\\]|\\['"\\$bntr]|\\x[0-9a-f][0-9a-f]|\\[0-7][0-7][0-7])*\'	 {
 				   yytext[yyleng-1] = '\0';
 				   unescapeStr((uchar*)yytext+1, yyleng-2);

plugins/imfile/imfile.c

@@ -422,6 +422,18 @@ openFile(fileInfo_t *pThis)
 
 	/* read back in the object */
 	CHKiRet(obj.Deserialize(&pThis->pStrm, (uchar*) "strm", psSF, NULL, pThis));
+	DBGPRINTF("imfile: deserialized state file, state file base name '%s', "
+		  "configured base name '%s'\n", pThis->pStrm->pszFName,
+		  pThis->pszFileName);
+	if(ustrcmp(pThis->pStrm->pszFName, pThis->pszFileName)) {
+		errmsg.LogError(0, RS_RET_STATEFILE_WRONG_FNAME, "imfile: state file '%s' "
+				"contains file name '%s', but is used for file '%s'. State "
+				"file deleted, starting from begin of file.",
+				pszSFNam, pThis->pStrm->pszFName, pThis->pszFileName);
+
+		unlink((char*)pszSFNam);
+		ABORT_FINALIZE(RS_RET_STATEFILE_WRONG_FNAME);
+	}
 
 	strm.CheckFileChange(pThis->pStrm);
 	CHKiRet(strm.SeekCurrOffs(pThis->pStrm));

plugins/imkmsg/kmsg.c

@@ -34,6 +34,7 @@
 #include <ctype.h>
 #include <sys/klog.h>
 #include <sys/sysinfo.h>
+#include <sys/time.h>
 #include <json.h>
 
 #include "rsyslog.h"

plugins/mmanon/mmanon.c

@@ -295,7 +295,7 @@ anonip(instanceData *pData, uchar *msg, int *pLenMsg, int *idx)
 	int endpos;
 	int lenMsg = *pLenMsg;
 
-	while(i < lenMsg && (msg[i] <= '0' || msg[i] >= '9')) {
+	while(i < lenMsg && (msg[i] <= '0' || msg[i] > '9')) {
 		++i; /* skip to first number */
 	}
 	if(i >= lenMsg)

plugins/ompgsql/ompgsql.c

@@ -235,9 +235,18 @@ writePgSQL(uchar *psz, instanceData *pData)
 
 	if(bHadError || (PQstatus(pData->f_hpgsql) != CONNECTION_OK)) {
 		/* error occured, try to re-init connection and retry */
-		closePgSQL(pData); /* close the current handle */
-		CHKiRet(initPgSQL(pData, 0)); /* try to re-open */
-		bHadError = tryExec(psz, pData); /* retry */
+		int inTransaction = 0;
+		if(pData->f_hpgsql != NULL) {
+			PGTransactionStatusType xactStatus = PQtransactionStatus(pData->f_hpgsql);
+			if((xactStatus == PQTRANS_INTRANS) || (xactStatus == PQTRANS_ACTIVE)) {
+				inTransaction = 1;
+			}
+		}
+		if ( inTransaction == 0 ) {
+			closePgSQL(pData); /* close the current handle */
+			CHKiRet(initPgSQL(pData, 0)); /* try to re-open */
+			bHadError = tryExec(psz, pData); /* retry */
+		}
 		if(bHadError || (PQstatus(pData->f_hpgsql) != CONNECTION_OK)) {
 			/* we failed, giving up for now */
 			reportDBError(pData, 0);

plugins/omudpspoof/omudpspoof.c

@@ -383,7 +383,7 @@ UDPSend(wrkrInstanceData_t *pWrkrData, uchar *pszSourcename, char *msg, size_t l
 
 	if(len > 65528) {
 		DBGPRINTF("omudpspoof: msg with length %d truncated to 64k: '%.768s'\n",
-			  len, msg);
+			  (int) len, msg);
 		len = 65528;
 	}
 
@@ -417,7 +417,7 @@ UDPSend(wrkrInstanceData_t *pWrkrData, uchar *pszSourcename, char *msg, size_t l
 		libnet_clear_packet(pWrkrData->libnet_handle);
 		/* note: libnet does need ports in host order NOT in network byte order! -- rgerhards, 2009-11-12 */
 		udp = libnet_build_udp(
-			ntohs(pWrkrData->sourcePort),/* source port */
+			pWrkrData->sourcePort,	/* source port */
 			ntohs(tempaddr->sin_port),/* destination port */
 			pktLen+LIBNET_UDP_H,	/* packet length */
 			0,			/* checksum */
@@ -455,7 +455,7 @@ UDPSend(wrkrInstanceData_t *pWrkrData, uchar *pszSourcename, char *msg, size_t l
 			 * it is useful for consolidating with strace output.
 			 */
 			DBGPRINTF("omudpspoof: write error (total len %d): pktLen %d, sent %d, fd %d: %s\n",
-				  len, LIBNET_IPV4_H+LIBNET_UDP_H+pktLen, lsent, pWrkrData->libnet_handle->fd,
+				  (int) len, LIBNET_IPV4_H+LIBNET_UDP_H+pktLen, lsent, pWrkrData->libnet_handle->fd,
 				  libnet_geterror(pWrkrData->libnet_handle));
 			if(lsent != -1) {
 				bSendSuccess = RSTRUE;
@@ -504,7 +504,7 @@ UDPSend(wrkrInstanceData_t *pWrkrData, uchar *pszSourcename, char *msg, size_t l
 			lsent = libnet_write(pWrkrData->libnet_handle);
 			if(lsent != (int) (LIBNET_IPV4_H+pktLen)) {
 				DBGPRINTF("omudpspoof: fragment write error len %d, sent %d: %s\n",
-					  LIBNET_IPV4_H+LIBNET_UDP_H+len, lsent, libnet_geterror(pWrkrData->libnet_handle));
+					  (int) (LIBNET_IPV4_H+LIBNET_UDP_H+len), lsent, libnet_geterror(pWrkrData->libnet_handle));
 				bSendSuccess = RSFALSE;
 				continue;
 			}

rsyslog.service.in

@@ -1,11 +1,14 @@
 [Unit]
 Description=System Logging Service
 Requires=syslog.socket
+Documentation=man:rsyslogd(8)
+Documentation=http://www.rsyslog.com/doc/
 
 [Service]
 Type=notify
 ExecStart=@sbindir@/rsyslogd -n
 StandardOutput=null
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

runtime/rsyslog.h

@@ -3,7 +3,7 @@
  *
  * Begun 2005-09-15 RGerhards
  *
- * Copyright (C) 2005-2013 by Rainer Gerhards and Adiscon GmbH
+ * Copyright (C) 2005-2014 by Rainer Gerhards and Adiscon GmbH
  *
  * This file is part of the rsyslog runtime library.
  *
@@ -386,6 +386,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_CONF_WRN_FULLDLY_BELOW_HIGHWTR = -2358,/**< warning queue full delay mark below high wtr mark */
 	RS_RET_RESUMED = -2359,/**< status: action was resumed (used for reporting) */
 	RS_RET_RELP_NO_TLS = -2360,/**< librel does not support TLS (but TLS requested) */
+	RS_RET_STATEFILE_WRONG_FNAME  = -2361,/**< state file is for wrong file */
 
 	/* up to 2400 reserved for 7.5 & 7.6 */
 	RS_RET_INVLD_OMOD = -2400, /**< invalid output module, does not provide proper interfaces */

tests/Makefile.am

@@ -128,6 +128,11 @@ TESTS +=  \
 	imptcp_conndrop.sh 
 endif
 
+if ENABLE_ELASTICSEARCH
+TESTS +=  \
+	elasticsearch-basic.sh
+endif
+
 if ENABLE_MMPSTRUCDATA
 TESTS +=  \
 	mmpstrucdata.sh
@@ -362,6 +367,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   incltest_dir_wildcard.sh \
 	   testsuites/incltest_dir_wildcard.conf \
 	   testsuites/incltest.d/include.conf \
+	   elasticsearch-basic.sh \
+	   testsuites/elasticsearch-basic.conf \
 	   linkedlistqueue.sh \
 	   testsuites/linkedlistqueue.conf \
 	   da-mainmsg-q.sh \

tests/diag.sh

@@ -37,6 +37,17 @@ case $1 in
 		rm -f rsyslog.input rsyslog.conf.tlscert stat-file1 rsyslog.empty
 		echo  -------------------------------------------------------------------------------
 		;;
+   'es-init')   # initialize local Elasticsearch *testbench* instance for the next
+                # test. NOTE: do NOT put anything useful on that instance!
+		curl -XDELETE localhost:9200/rsyslog_testbench
+		;;
+   'es-getdata') # read data from ES to a local file so that we can process
+   		# it with out regular tooling.
+		# Note: param 2 MUST be number of records to read (ES does
+		# not return the full set unless you tell it explicitely).
+		curl localhost:9200/rsyslog_testbench/_search?size=$2 > work
+		python $srcdir/es_response_get_msgnum.py > rsyslog.out.log
+		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
    		# returns only after successful startup, $3 is the instance (blank or 2!)
 		$valgrind ../tools/rsyslogd -u2 -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 &

tests/elasticsearch-basic.sh

@@ -0,0 +1,12 @@
+# This file is part of the rsyslog project, released under ASL 2.0
+echo ===============================================================================
+echo \[elasticsearch-basic.sh\]: basic test for elasticsearch functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh es-init
+source $srcdir/diag.sh startup elasticsearch-basic.conf
+source $srcdir/diag.sh injectmsg  0 10000
+source $srcdir/diag.sh shutdown-when-empty
+source $srcdir/diag.sh wait-shutdown 
+source $srcdir/diag.sh es-getdata 10000
+source $srcdir/diag.sh seq-check  0 9999
+source $srcdir/diag.sh exit

tests/es_response_get_msgnum.py

@@ -0,0 +1,7 @@
+import json
+
+with open("work") as json_file:
+    json_data = json.load(json_file)
+    json_data = json_data["hits"]
+    for item in json_data["hits"]:
+	print item["_source"]["msgnum"]

tests/testsuites/elasticsearch-basic.conf

@@ -0,0 +1,9 @@
+$IncludeConfig diag-common.conf
+
+template(name="tpl" type="string"
+	 string="{\"msgnum\":\"%msg:F,58:2%\"}")
+
+module(load="../plugins/omelasticsearch/.libs/omelasticsearch")
+:msg, contains, "msgnum:" action(type="omelasticsearch"
+				 template="tpl"
+				 searchIndex="rsyslog_testbench")