Merge branch 'v7-stable-libnet-frag'

Conflicts:
	ChangeLog
	configure.ac
	tools/omfile.c

ChangeLog

@@ -1,4 +1,9 @@
 ---------------------------------------------------------------------------
+Version 7.3.6  [devel] 2012-12-??
+- impstats: added ability to write stats records to local file
+  and avoid going through the syslog log stream. syslog logging can now
+  also be turned off (see doc for details).
+---------------------------------------------------------------------------
 Version 7.3.5  [devel] 2012-12-19
 - ommysql: addded batching/transaction support
 - enhanced script optimizer to optimize common PRI-based comparisons

Makefile.am

@@ -70,7 +70,7 @@ EXTRA_DIST = \
 	contrib/gnutls/key.pem \
 	rsyslog.service.in
 
-SUBDIRS = doc runtime grammar . plugins/immark plugins/imuxsock plugins/imtcp plugins/imudp plugins/omtesting
+SUBDIRS = doc runtime grammar compat . plugins/immark plugins/imuxsock plugins/imtcp plugins/imudp plugins/omtesting
 
 if ENABLE_RSYSLOGD
 SUBDIRS += tools

doc/impstats.html

@@ -16,26 +16,66 @@ availabilty and format of counters may change and is not yet stable (so be
 prepared to change your trending scripts when you upgrade to a newer rsyslog version).
 <p>The set of available counters will be output as a set of syslog messages. This
 output is periodic, with the interval being configurable (default is 5 minutes).
-Be sure that your configuration records the counter messages (default is syslog.info).
+Be sure that your configuration records the counter messages (default is syslog.=info).
+Besides logging to the regular syslog stream, the module can also be configured to
+write statistics data into a (local) file.
 <p>Note that loading this module has impact on rsyslog performance. Depending on
 settings, this impact may be noticable (for high-load environments).
 <p>The rsyslog website has an updated overview of available
 <a href="http://rsyslog.com/rsyslog-statistic-counter/">rsyslog statistic counters</a>.
 </p>
-<p><b>Configuration Directives</b>:</p>
+<p><b>Module Confguration Parameters</b>:</p>
+<p>This module supports module parameters, only.
 <ul>
-<li>$PStatInterval &lt;Seconds&gt;<br>
+	<li><strong>interval </strong>[seconds] (default 300 [5minutes])<br>
-Sets the interval, in <b>seconds</b> at which messages are generated. Please note that the
+	Sets the interval, in <b>seconds</b> at which messages are generated. Please note that the
-actual interval may be a bit longer. We do not try to be precise and so the interval is
+	actual interval may be a bit longer. We do not try to be precise and so the interval is
-actually a sleep period which is entered after generating all messages. So the actual 
+	actually a sleep period which is entered after generating all messages. So the actual 
-interval is what is configured here plus the actual time required to generate messages.
+	interval is what is configured here plus the actual time required to generate messages.
-In general, the difference should not really matter.
+	In general, the difference should not really matter.
-<li>$PStatFacility &lt;numerical facility&gt;<br>
+	<br></li>
-The numerical syslog facility code to be used for generated messages. Default
+	<li><strong>facility </strong>[templateName]<br>
-is 5 (syslog).This is useful for filtering messages.</li>
+	The numerical syslog facility code to be used for generated messages. Default
-<li>$PStatSeverity &lt;numerical severity&gt;<br>
+	is 5 (syslog). This is useful for filtering messages.
-The numerical syslog severity code to be used for generated messages. Default
+	<br></li>
-is 6 (info).This is useful for filtering messages.</li>
+	<li><strong>severity </strong>[templateName]<br>
+	The numerical syslog severity code to be used for generated messages. Default
+	is 6 (info).This is useful for filtering messages.
+	<br></li>
+	<li><strong>format </strong>[json/cee/<b>legacy</b>](rsyslog v6.3.8+ only)<br>
+	Specifies the format of emitted stats messages. The default of "legacy" is
+	compatible with pre v6-rsyslog. The other options provide support for
+	structured formats (note the "cee" is actually "project lumberack" logging).
+	<br></li>
+	<li><strong>log.syslog </strong>[<b>on</b>/off] - available since 7.3.6<br>
+	This is a boolean setting specifying if data should be sent
+	to the usual syslog stream. This is useful if custom formatting
+	or more elaborate processing is desired. However, output is placed
+	under the same restrictions as regular syslog data, especially in
+	regard to the queue position (stats data may sit for an extended
+	period of time in queues if they are full).<br></li>
+	<li><strong>log.file </strong>[file name] - available since 7.3.6<br>
+	If specified, statistics data is written the specified file. For
+	robustness, this should be a local file. The file format cannot be
+	customized, it consists of a date header, followed by a colon, 
+	followed by the actual statistics record, all on one line. Only
+	very limited error handling is done, so if things go wrong stats
+	records will probably be lost. Logging to file an be a useful
+	alternative if for some reasons (e.g. full queues) the regular
+	syslog stream method shall not be used solely. Note that turning
+	on file logging does NOT turn of syslog logging. If that is desired
+	log.syslog="off" must be explicitely set.
+	<br></li>
+	
+</ul>
+<p><b>Legacx Configuration Directives</b>:</p>
+A limited set of parameters can also be set via the legacy configuration
+syntax. Note that this is intended as an upward compatibilit layer, so
+newer features are intentionally <b>not</b> available via legacy directives.
+<ul>
+<li>$PStatInterval &lt;Seconds&gt; - same as the "interval" parameter.
+<li>$PStatFacility &lt;numerical facility&gt; - same as the "facility" parameter.
+<li>$PStatSeverity &lt;numerical severity&gt; - same as the "severity" parameter.
 <li>$PStatJSON &lt;on/<b>off</b>&gt; (rsyslog v6.3.8+ only)<br>
 If set to on, stats messages are emitted as structured cee-enhanced syslog. If
 set to off, legacy format is used (which is compatible with pre v6-rsyslog).
@@ -45,23 +85,45 @@ set to off, legacy format is used (which is compatible with pre v6-rsyslog).
 <ul>
 <li>This module MUST be loaded right at the top of rsyslog.conf, otherwise 
 stats may not get turned on in all places.</li>
-<li>experimental code</li>
 </ul>
-<p><b>Sample:</b></p>
+<p><b>Samples:</b></p>
 <p>This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:<br>
 </p>
-<textarea rows="8" cols="60">$ModLoad impstats
+<textarea rows="5" cols="60">module(load="impstats" interval="600" severity="7")
+
+# to actually gather the data:
+syslog.=debug  /var/log/rsyslog-stats
+</textarea>
+<p><b>Legacy Sample:</b></p>
+<p>This activates the module and records messages to /var/log/rsyslog-stats in 10 minute intervals:</p>
+<textarea rows="6" cols="60">$ModLoad impstats
 $PStatInterval 600
 $PStatSeverity 7
 
-syslog.debug  /var/log/rsyslog-stats
+syslog.=debug  /var/log/rsyslog-stats
 </textarea>
+<p>In the next sample, the default interval of 5 minutes is used. However, this time
+stats data is NOT emitted to the syslog stream but to a local file instead.
+<p>
+<textarea rows="3" cols="70">module(load="impstats" interval="600" severity="7"
+       log.syslog="off" /* need to turn log stream logging off! */
+       log.file="/path/to/local/stats.log")
+</textarea>
+<p>And finally, we log to both the regular syslog log stream as well as a file.
+Within the log stream, we forward the data records to another server:
+<p>
+<textarea rows="4" cols="70">module(load="impstats" interval="600" severity="7"
+       log.file="/path/to/local/stats.log")
+
+syslog.=debug  @central.example.net
+</textarea>
+
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
 [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
 <a href="http://www.rsyslog.com/">rsyslog</a>
 project.<br>
-Copyright &copy; 2010 by <a href="http://www.gerhards.net/rainer">Rainer
+Copyright &copy; 2013 by <a href="http://www.gerhards.net/rainer">Rainer
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>

doc/omfile.html

@@ -13,14 +13,14 @@
 <p>The omfile plug-in provides the core functionality of writing messages to files residing inside the local file system (which may actually be remote if methods like NFS are used). Both files named with static names as well files with names based on message content are supported by this module. It is a built-in module that does not need to be loaded. </p>
 <p>&nbsp;</p>
 
-<p><b>Global Configuration Directives</b>:</p>
+<p><b>Module Confguration Parameters</b>:</p>
 <ul>
 	<li><strong>Template </strong>[templateName]<br>
 	sets a new default template for file actions.<br></li>
 	
 </ul>
 <p>&nbsp;</p>
-<p><b>Action specific Configuration Directives</b>:</p>
+<p><b>Action Confguration Parameters</b>:</p>
 <ul>
 	<li><strong>DynaFileCacheSize </strong>(not mandatory, default will be used)<br>
 	Defines a template to be used for the output. <br></li><br>

plugins/impstats/impstats.c

@@ -27,6 +27,8 @@
 #include <signal.h>
 #include <string.h>
 #include <pthread.h>
+#include <fcntl.h>
+#include <sys/uio.h>
 #include "dirty.h"
 #include "cfsysline.h"
 #include "module-template.h"
@@ -63,11 +65,14 @@ typedef struct configSettings_s {
 } configSettings_t;
 
 struct modConfData_s {
-	rsconf_t *pConf;		/* our overall config object */
+	rsconf_t *pConf; /* our overall config object */
 	int iStatsInterval;
 	int iFacility;
 	int iSeverity;
+	int logfd; /* fd if logging to file, or -1 if closed */
 	statsFmtType_t statsFmt;
+	sbool bLogToSyslog;
+	char *logfile;
 	sbool configSetViaV2Method;
 };
 static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */
@@ -82,6 +87,8 @@ static struct cnfparamdescr modpdescr[] = {
 	{ "interval", eCmdHdlrInt, 0 },
 	{ "facility", eCmdHdlrInt, 0 },
 	{ "severity", eCmdHdlrInt, 0 },
+	{ "log.syslog", eCmdHdlrBinary, 0 },
+	{ "log.file", eCmdHdlrGetWord, 0 },
 	{ "format", eCmdHdlrGetWord, 0 }
 };
 static struct cnfparamblk modpblk =
@@ -120,7 +127,7 @@ initConfigSettings(void)
 
 /* actually submit a message to the rsyslog core
  */
-static inline rsRetVal
+static inline void
 doSubmitMsg(uchar *line)
 {
 	msg_t *pMsg;
@@ -144,8 +151,49 @@ doSubmitMsg(uchar *line)
 	          runModConf->iSeverity, line);
 
 finalize_it:
-	RETiRet;
+	return;
+}
 
+
+/* log stats message to file; limited error handling done */
+static inline void
+doLogToFile(cstr_t *cstr)
+{
+	struct iovec iov[4];
+	ssize_t nwritten;
+	ssize_t nexpect;
+	time_t t;
+	char timebuf[32];
+
+	if(cstrLen(cstr) == 0)
+		goto done;
+	if(runModConf->logfd == -1) {
+		runModConf->logfd = open(runModConf->logfile, O_WRONLY|O_CREAT|O_APPEND|O_CLOEXEC, S_IRUSR|S_IWUSR);
+		if(runModConf->logfd == -1) {
+			dbgprintf("error opening stats file %s\n", runModConf->logfile);
+			goto done;
+		}
+	}
+
+	time(&t);
+	iov[0].iov_base = ctime_r(&t, timebuf);
+	iov[0].iov_len = nexpect = strlen(iov[0].iov_base) - 1; /* -1: strip \n */
+	iov[1].iov_base = ": ";
+	iov[1].iov_len = 2;
+	nexpect += 2;
+	iov[2].iov_base = rsCStrGetSzStrNoNULL(cstr);
+	iov[2].iov_len = (size_t) cstrLen(cstr);
+	nexpect += cstrLen(cstr);
+	iov[3].iov_base = "\n";
+	iov[3].iov_len = 1;
+	nexpect++;
+	nwritten = writev(runModConf->logfd, iov, 4);
+
+	if(nwritten != nexpect) {
+			dbgprintf("error writing stats file %s, nwritten %lld, expected %lld\n",
+				  runModConf->logfile, (long long) nwritten, (long long) nexpect);
+	}
+done:	return;
 }
 
 
@@ -156,7 +204,10 @@ static rsRetVal
 doStatsLine(void __attribute__((unused)) *usrptr, cstr_t *cstr)
 {
 	DEFiRet;
-	doSubmitMsg(rsCStrGetSzStrNoNULL(cstr));
+	if(runModConf->bLogToSyslog)
+		doSubmitMsg(rsCStrGetSzStrNoNULL(cstr));
+	if(runModConf->logfile != NULL)
+		doLogToFile(cstr);
 	RETiRet;
 }
 
@@ -181,6 +232,9 @@ CODESTARTbeginCnfLoad
 	loadModConf->iFacility = DEFAULT_FACILITY;
 	loadModConf->iSeverity = DEFAULT_SEVERITY;
 	loadModConf->statsFmt = statsFmt_Legacy;
+	loadModConf->logfd = -1;
+	loadModConf->logfile = NULL;
+	loadModConf->bLogToSyslog = 1;
 	bLegacyCnfModGlobalsPermitted = 1;
 	/* init legacy config vars */
 	initConfigSettings();
@@ -213,6 +267,10 @@ CODESTARTsetModCnf
 			loadModConf->iFacility = (int) pvals[i].val.d.n;
 		} else if(!strcmp(modpblk.descr[i].name, "severity")) {
 			loadModConf->iSeverity = (int) pvals[i].val.d.n;
+		} else if(!strcmp(modpblk.descr[i].name, "log.syslog")) {
+			loadModConf->bLogToSyslog = (sbool) pvals[i].val.d.n;
+		} else if(!strcmp(modpblk.descr[i].name, "log.file")) {
+			loadModConf->logfile = es_str2cstr(pvals[i].val.d.estr, NULL);
 		} else if(!strcmp(modpblk.descr[i].name, "format")) {
 			mode = es_str2cstr(pvals[i].val.d.estr, NULL);
 			if(!strcasecmp(mode, "json")) {
@@ -273,7 +331,9 @@ BEGINactivateCnf
 	rsRetVal localRet;
 CODESTARTactivateCnf
 	runModConf = pModConf;
-	DBGPRINTF("impstats: stats interval %d seconds\n", runModConf->iStatsInterval);
+	DBGPRINTF("impstats: stats interval %d seconds, logToSyslog %d, logFile %s\n",
+		  runModConf->iStatsInterval, runModConf->bLogToSyslog,
+		  runModConf->logfile == NULL ? "deactivated" : (char*)runModConf->logfile);
 	localRet = statsobj.EnableStats();
 	if(localRet != RS_RET_OK) {
 		errmsg.LogError(0, localRet, "impstats: error enabling statistics gathering");
@@ -285,6 +345,9 @@ ENDactivateCnf
 
 BEGINfreeCnf
 CODESTARTfreeCnf
+	if(runModConf->logfd != -1)
+		close(runModConf->logfd);
+	free(runModConf->logfile);
 ENDfreeCnf
 
 
@@ -300,6 +363,7 @@ CODESTARTrunInput
 		if(glbl.GetGlobalInputTermState() == 1)
 			break; /* terminate input! */
 
+		DBGPRINTF("impstats: woke up, generating messages\n");
 		generateStatsMsgs();
 	}
 ENDrunInput

plugins/imudp/imudp.c

@@ -381,7 +381,7 @@ processSocket(thrdInfo_t *pThrd, struct lstn_s *lstn, struct sockaddr_storage *f
 			*pbIsPermitted = 1; /* no check -> everything permitted */
 		}
 
-		DBGPRINTF("recv(%d,%d),acl:%d,msg:%s\n", lstn->sock, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf);
+		DBGPRINTF("imudp:recv(%d,%d),acl:%d,msg:%s\n", lstn->sock, (int) lenRcvBuf, *pbIsPermitted, pRcvBuf);
 
 		if(*pbIsPermitted != 0)  {
 			if((runModConf->iTimeRequery == 0) || (iNbrTimeUsed++ % runModConf->iTimeRequery) == 0) {

tests/tcpflood.c

@@ -258,7 +258,7 @@ int openConnections(void)
 		return setupUDP();
 
 	if(bShowProgress)
-		write(1, "      open connections", sizeof("      open connections")-1);
+		if(write(1, "      open connections", sizeof("      open connections")-1)){}
 #	ifdef ENABLE_GNUTLS
 	sessArray = calloc(numConnections, sizeof(gnutls_session_t));
 #	endif
@@ -278,7 +278,7 @@ int openConnections(void)
 	}
 	if(bShowProgress) {
 		lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i);
-		write(1, msgBuf, lenMsg);
+		if(write(1, msgBuf, lenMsg)) {}
 	}
 
 	return 0;
@@ -303,12 +303,12 @@ void closeConnections(void)
 		return;
 
 	if(bShowProgress)
-		write(1, "      close connections", sizeof("      close connections")-1);
+		if(write(1, "      close connections", sizeof("      close connections")-1)){}
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
 			if(bShowProgress) {
 				lenMsg = sprintf(msgBuf, "\r%5.5d", i);
-				write(1, msgBuf, lenMsg);
+				if(write(1, msgBuf, lenMsg)){}
 			}
 		}
 		if(sockArray[i] != -1) {
@@ -325,7 +325,7 @@ void closeConnections(void)
 	}
 	if(bShowProgress) {
 		lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i);
-		write(1, msgBuf, lenMsg);
+		if(write(1, msgBuf, lenMsg)){}
 	}
 
 }

tools/omfile.c

@@ -857,6 +857,7 @@ BEGINdoAction
 CODESTARTdoAction
 	DBGPRINTF("file to log to: %s\n",
 		  (pData->bDynamicName) ? ppString[1] : pData->f_fname);
+	DBGPRINTF("omfile: start of data: '%.128s'\n", ppString[0]);
 	CHKiRet(writeFile(ppString, iMsgOpts, pData));
 	if(!bCoreSupportsBatching && pData->bFlushOnTXEnd) {
 		CHKiRet(strm.Flush(pData->pStrm));