config: add include() script object

This permits to include files (like legacy $IncludeConfig) via a
script object. Needless to say, the script object offers more
features:

- include files can now be
  - required, with rsyslog aborting when not present
  - required, with rsyslog emitting an error message but otherwise
    continuing when not present
  - optional, which means non-present include files will be
    skipped without notice
  This is controlled by the "mode" parameter.
- text can be included form e.g. an environment variable
  --> ex: include(text=`echo $ENVVAR`)

This finally really obsoletes $IncludeConfig.

closes https://github.com/rsyslog/rsyslog/issues/2151

grammar/grammar.y

@@ -63,6 +63,7 @@ extern int yyerror(const char*);
 %token <estr> FUNC
 %token <objType> BEGINOBJ
 %token ENDOBJ
+%token BEGIN_INCLUDE
 %token BEGIN_ACTION
 %token BEGIN_PROPERTY
 %token BEGIN_CONSTANT
@@ -135,6 +136,7 @@ conf:	/* empty (to end recursion) */
 	| conf LEGACY_RULESET		{ cnfDoCfsysline($2); }
 	| conf BSD_TAG_SELECTOR		{ cnfDoBSDTag($2); }
 	| conf BSD_HOST_SELECTOR	{ cnfDoBSDHost($2); }
+include:  BEGIN_INCLUDE nvlst ENDOBJ	{ includeProcessCnf($2); }
 obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
         | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
@@ -179,6 +181,7 @@ stmt:	  actlst			{ $$ = $1; }
 	| PRIFILT block			{ $$ = cnfstmtNewPRIFILT($1, $2); }
 	| PROPFILT block		{ $$ = cnfstmtNewPROPFILT($1, $2); }
 	| RELOAD_LOOKUP_TABLE_PROCEDURE '(' fparams ')' { $$ = cnfstmtNewReloadLookupTable($3);}
+	| include			{ $$ = NULL; }
 	| BEGINOBJ			{ $$ = NULL; parser_errmsg("declarative object '%s' not permitted in action block [stmt]", yytext);}
 block:    stmt				{ $$ = $1; }
 	| '{' script '}'		{ $$ = $2; }

grammar/lexer.l

@@ -251,7 +251,7 @@ int fileno(FILE *stream);
   * always the longest match :-(
   */
 <INCL>.|\n
-<INCL>[^ \t\n]+			{ if(cnfDoInclude(yytext) != 0)
+<INCL>[^ \t\n]+			{ if(cnfDoInclude(yytext, 0) != 0)
 					yyterminate();
 				  BEGIN INITIAL; }
 "main_queue"[ \n\t]*"("		{ yylval.objType = CNFOBJ_MAINQ;
@@ -278,6 +278,7 @@ int fileno(FILE *stream);
 				  BEGIN INOBJ; return BEGINOBJ; }
 "dyn_stats"[ \n\t]*"("		{ yylval.objType = CNFOBJ_DYN_STATS;
 				  BEGIN INOBJ; return BEGINOBJ; }
+"include"[ \n\t]*"("		{ BEGIN INOBJ; return BEGIN_INCLUDE; }
 "action"[ \n\t]*"("		{ BEGIN INOBJ; return BEGIN_ACTION; }
 ^[ \t]*:\$?[a-z\-]+[ ]*,[ ]*!?[a-z]+[ ]*,[ ]*\"(\\\"|[^\"])*\"	{
 				  yylval.s = strdup(rmLeadingSpace(yytext));
@@ -369,6 +370,47 @@ cnfParseBuffer(char *buf, unsigned lenBuf)
 done:	return r;
 }
 
+
+/* add config file or text the the stack of config objects to be
+ * processed.
+ * cnfobjname is either the file name or "text" if generated from
+ * text ("text" can also be replaced by something more intelligent
+ * by the caller.
+ * The provided string is freed.
+ */
+int ATTR_NONNULL()
+cnfAddConfigBuffer(es_str_t *const str, const char *const cnfobj_name)
+{
+	struct bufstack *bs;
+	int r = 0;
+	assert(str != NULL);
+	assert(cnfobj_name != NULL);
+
+	if((bs = malloc(sizeof(struct bufstack))) == NULL) {
+		r = 1;
+		goto done;
+	}
+
+	if(currbs != NULL)
+		currbs->lineno = yylineno;
+	bs->prev = currbs;
+	bs->fn = strdup(cnfobj_name);
+	yy_size_t lll = es_strlen(str);
+	bs->bs = yy_scan_buffer((char*)es_getBufAddr(str), lll);
+	bs->estr = str; /* needed so we can free it later */
+	currbs = bs;
+	cnfcurrfn = bs->fn;
+	yylineno = 1;
+	dbgprintf("config parser: pushed config fragment on top of stack: %s\n", cnfobj_name);
+
+done:
+	if(r != 0) {
+		es_deleteStr(str);
+	}
+	return r;
+}
+
+
 /* set a new buffers. Returns 0 on success, 1 on error, 2 on file not exists.
  * note: in case of error, errno must be kept valid!
  */
@@ -376,9 +418,9 @@ int
 cnfSetLexFile(char *fname)
 {
 	es_str_t *str = NULL;
+	struct bufstack *bs;
 	FILE *fp;
 	int r = 0;
-	struct bufstack *bs;
 
 	/* check for invalid recursive include */
 	for(bs = currbs ; bs != NULL ; bs = bs->prev) {
@@ -402,30 +444,9 @@ cnfSetLexFile(char *fname)
 	if(fp != stdin)
 		fclose(fp);
 	
-	/* maintain stack */
+	r = cnfAddConfigBuffer(str, ((fname == NULL) ? "stdin" : fname));
-	if((bs = malloc(sizeof(struct bufstack))) == NULL) {
-		r = 1;
-		goto done;
-	}
-
-	if(currbs != NULL)
-		currbs->lineno = yylineno;
-	bs->prev = currbs;
-	bs->fn = strdup(fname == NULL ? "stdin" : fname);
-	yy_size_t lll = es_strlen(str);
-	//bs->bs = yy_scan_buffer((char*)es_getBufAddr(str), (yy_size_t) es_strlen(str));
-	bs->bs = yy_scan_buffer((char*)es_getBufAddr(str), lll);
-	bs->estr = str; /* needed so we can free it later */
-	currbs = bs;
-	cnfcurrfn = bs->fn;
-	yylineno = 1;
-	dbgprintf("config parser: pushed file %s on top of stack\n", fname);
 
 done:
-	if(r != 0) {
-		if(str != NULL)
-			es_deleteStr(str);
-	}
 	return r;
 }
 

grammar/parserif.h

@@ -39,4 +39,5 @@ void cnfDoScript(struct cnfstmt *script);
 void cnfDoCfsysline(char *ln);
 void cnfDoBSDTag(char *ln);
 void cnfDoBSDHost(char *ln);
+int cnfAddConfigBuffer(es_str_t *const str, const char *const cnfobj_name);
 #endif

grammar/rainerscript.c

@@ -69,6 +69,17 @@ static void cnfstmtOptimizePRIFilt(struct cnfstmt *stmt);
 static void cnfarrayPrint(struct cnfarray *ar, int indent);
 struct cnffunc * cnffuncNew_prifilt(int fac);
 
+static struct cnfparamdescr incpdescr[] = {
+	{ "file", eCmdHdlrString, 0 },
+	{ "text", eCmdHdlrString, 0 },
+	{ "mode", eCmdHdlrGetWord, 0 }
+};
+static struct cnfparamblk incpblk =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(incpdescr)/sizeof(struct cnfparamdescr),
+	  incpdescr
+	};
+
 struct curl_funcData {
 	const char *reply;
 	size_t replyLen;
@@ -5014,18 +5025,21 @@ cnffuncNew_prifilt(int fac)
 /* returns 0 if everything is OK and config parsing shall continue,
  * and 1 if things are so wrong that config parsing shall be aborted.
  */
-int
+int ATTR_NONNULL()
-cnfDoInclude(char *name)
+cnfDoInclude(const char *const name, const int optional)
 {
 	char *cfgFile;
-	char *finalName;
+	const char *finalName;
 	int i;
 	int result;
 	glob_t cfgFiles;
+	int ret = 0;
 	struct stat fileInfo;
+	char errStr[1024];
 	char nameBuf[MAXFNAME+1];
 	char cwdBuf[MAXFNAME+1];
 
+	DBGPRINTF("cnfDoInclude: file: '%s', optional: %d\n", name, optional);
 	finalName = name;
 	if(stat(name, &fileInfo) == 0) {
 		/* stat usually fails if we have a wildcard - so this does NOT indicate error! */
@@ -5038,25 +5052,27 @@ cnfDoInclude(char *name)
 
 	/* Use GLOB_MARK to append a trailing slash for directories. */
 	/* Use GLOB_NOMAGIC to detect wildcards that match nothing. */
-#ifdef HAVE_GLOB_NOMAGIC
+	#ifdef HAVE_GLOB_NOMAGIC
-	/* Silently ignore wildcards that match nothing */
+		/* Silently ignore wildcards that match nothing */
-	result = glob(finalName, GLOB_MARK | GLOB_NOMAGIC, NULL, &cfgFiles);
+		result = glob(finalName, GLOB_MARK | GLOB_NOMAGIC, NULL, &cfgFiles);
-	if(result == GLOB_NOMATCH) {
+		if(result == GLOB_NOMATCH) {
-#else
+	#else
-	result = glob(finalName, GLOB_MARK, NULL, &cfgFiles);
+		result = glob(finalName, GLOB_MARK, NULL, &cfgFiles);
-    if(result == GLOB_NOMATCH && containsGlobWildcard(finalName)) {
+		if(result == GLOB_NOMATCH && containsGlobWildcard(finalName)) {
-#endif /* HAVE_GLOB_NOMAGIC */
+	#endif /* HAVE_GLOB_NOMAGIC */
-        return 0;
+		goto done;
-    }
+	}
 
 	if(result == GLOB_NOSPACE || result == GLOB_ABORTED) {
-		char errStr[1024];
+		if(optional == 0) {
-		rs_strerror_r(errno, errStr, sizeof(errStr));
+			rs_strerror_r(errno, errStr, sizeof(errStr));
-		if(getcwd(cwdBuf, sizeof(cwdBuf)) == NULL)
+			if(getcwd(cwdBuf, sizeof(cwdBuf)) == NULL)
-			strcpy(cwdBuf, "??getcwd() failed??");
+				strcpy(cwdBuf, "??getcwd() failed??");
-		parser_errmsg("error accessing config file or directory '%s' [cwd:%s]: %s",
+			parser_errmsg("error accessing config file or directory '%s' "
-				finalName, cwdBuf, errStr);
+				"[cwd:%s]: %s", finalName, cwdBuf, errStr);
-		return 1;
+			ret = 1;
+		}
+		goto done;
 	}
 
 	/* note: bison "stacks" the files, so we need to submit them
@@ -5067,13 +5083,15 @@ cnfDoInclude(char *name)
 	for(i = cfgFiles.gl_pathc - 1; i >= 0 ; i--) {
 		cfgFile = cfgFiles.gl_pathv[i];
 		if(stat(cfgFile, &fileInfo) != 0) {
-			char errStr[1024];
+			if(optional == 0) {
-			rs_strerror_r(errno, errStr, sizeof(errStr));
+				rs_strerror_r(errno, errStr, sizeof(errStr));
-			if(getcwd(cwdBuf, sizeof(cwdBuf)) == NULL)
+				if(getcwd(cwdBuf, sizeof(cwdBuf)) == NULL)
-				strcpy(cwdBuf, "??getcwd() failed??");
+					strcpy(cwdBuf, "??getcwd() failed??");
-			parser_errmsg("error accessing config file or directory '%s' "
+				parser_errmsg("error accessing config file or directory '%s' "
 					"[cwd: %s]: %s", cfgFile, cwdBuf, errStr);
-			return 1;
+				ret = 1;
+			}
+			goto done;
 		}
 
 		if(S_ISREG(fileInfo.st_mode)) { /* config file */
@@ -5081,16 +5099,105 @@ cnfDoInclude(char *name)
 			cnfSetLexFile(cfgFile);
 		} else if(S_ISDIR(fileInfo.st_mode)) { /* config directory */
 			DBGPRINTF("requested to include directory '%s'\n", cfgFile);
-			cnfDoInclude(cfgFile);
+			cnfDoInclude(cfgFile, optional);
 		} else {
 			DBGPRINTF("warning: unable to process IncludeConfig directive '%s'\n", cfgFile);
 		}
 	}
 
+done:
 	globfree(&cfgFiles);
-	return 0;
+	return ret;
 }
 
+
+/* Process include() objects */
+void
+includeProcessCnf(struct nvlst *const lst)
+{
+	struct cnfparamvals *pvals = NULL;
+	const char *inc_file = NULL;
+	const char *text = NULL;
+	int optional = 0;
+	int abort_if_missing = 0;
+	int i;
+
+	if(lst == NULL) {
+		parser_errmsg("include() must have either 'file' or 'text' "
+			"parameter - ignored");
+		goto done;
+	}
+
+	pvals = nvlstGetParams(lst, &incpblk, NULL);
+	if(pvals == NULL) {
+		goto done;
+	}
+	DBGPRINTF("include param blk after includeProcessCnf:\n");
+	cnfparamsPrint(&incpblk, pvals);
+	for(i = 0 ; i < incpblk.nParams ; ++i) {
+		if(!pvals[i].bUsed) {
+			continue;
+		}
+
+		if(!strcmp(incpblk.descr[i].name, "file")) {
+			inc_file = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(incpblk.descr[i].name, "text")) {
+			text = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(incpblk.descr[i].name, "mode")) {
+			char *const md = es_str2cstr(pvals[i].val.d.estr, NULL);
+			if(!strcmp(md, "abort-if-missing")) {
+				optional = 0;
+				abort_if_missing = 1;
+			} else if(!strcmp(md, "required")) {
+				optional = 0;
+			} else if(!strcmp(md, "optional")) {
+				optional = 1;
+			} else {
+				parser_errmsg("invalid 'mode' paramter: '%s' - ignored", md);
+			}
+			free((void*)md);
+		} else {
+			LogError(0, RS_RET_INTERNAL_ERROR,
+				"rainerscript/include: program error, non-handled inclpblk "
+				"param '%s' in includeProcessCnf()", incpblk.descr[i].name);
+		}
+	}
+
+	if(text != NULL && inc_file != NULL) {
+		parser_errmsg("include() must have either 'file' or 'text' "
+			"parameter, but both are set - ignored");
+		goto done;
+	}
+
+	if(inc_file != NULL) {
+		if(cnfDoInclude(inc_file, optional) != 0 && abort_if_missing) {
+			fprintf(stderr, "include file '%s' mode is set to abort-if-missing "
+				"and the file is indeed missing - thus aborting rsyslog\n",
+				inc_file);
+			exit(1); /* "good exit" - during config processing, requested by user */
+		}
+	} else if(text != NULL) {
+		es_str_t *estr = es_newStrFromCStr((char*)text, strlen(text));
+		/* lex needs 2 \0 bytes as terminator indication (wtf ;-)) */
+		es_addChar(&estr, '\0');
+		es_addChar(&estr, '\0');
+		cnfAddConfigBuffer(estr, "text");
+	} else {
+		parser_errmsg("include must have either 'file' or 'text' "
+			"parameter - ignored");
+		goto done;
+	}
+
+done:
+	free((void*)text);
+	free((void*)inc_file);
+	nvlstDestruct(lst);
+	if(pvals != NULL)
+		cnfparamvalsDestruct(pvals, &incpblk);
+	return;
+}
+
+
 void
 varDelete(const struct svar *v)
 {
@@ -5316,6 +5423,7 @@ tokenval2str(const int tok)
 	case BEGIN_PROPERTY: return "BEGIN_PROPERTY";
 	case BEGIN_CONSTANT: return "BEGIN_CONSTANT";
 	case BEGIN_TPL: return "BEGIN_TPL";
+	case BEGIN_INCLUDE: return "BEGIN_INCLUDE";
 	case BEGIN_RULESET: return "BEGIN_RULESET";
 	case STOP: return "STOP";
 	case SET: return "SET";

grammar/rainerscript.h

@@ -1,6 +1,6 @@
 /* rsyslog rainerscript definitions
  *
- * Copyright 2011-2016 Rainer Gerhards
+ * Copyright 2011-2018 Rainer Gerhards
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -333,7 +333,7 @@ struct cnfstringval* cnfstringvalNew(es_str_t *estr);
 struct cnfvar* cnfvarNew(char *name);
 struct cnffunc * cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst);
 struct cnffparamlst * cnffparamlstNew(struct cnfexpr *expr, struct cnffparamlst *next);
-int cnfDoInclude(char *name);
+int cnfDoInclude(const char *name, const int optional);
 int cnfparamGetIdx(struct cnfparamblk *params, const char *name);
 struct cnfparamvals* nvlstGetParams(struct nvlst *lst, struct cnfparamblk *params,
 	       struct cnfparamvals *vals);
@@ -368,6 +368,7 @@ rsRetVal initRainerscript(void);
 void unescapeStr(uchar *s, int len);
 const char * tokenval2str(int tok);
 uchar* var2CString(struct svar *__restrict__ const r, int *__restrict__ const bMustFree);
+void includeProcessCnf(struct nvlst *const lst);
 
 /* debug helper */
 void cstrPrint(const char *text, es_str_t *estr);

runtime/rsconf.c

@@ -2,7 +2,7 @@
  *
  * Module begun 2011-04-19 by Rainer Gerhards
  *
- * Copyright 2011-2016 Adiscon GmbH.
+ * Copyright 2011-2018 Adiscon GmbH.
  *
  * This file is part of the rsyslog runtime library.
  *
@@ -426,10 +426,12 @@ yyerror(const char *s)
 	parser_errmsg("%s on token '%s'", s, yytext);
 	return 0;
 }
-void cnfDoObj(struct cnfobj *o)
+void ATTR_NONNULL()
+cnfDoObj(struct cnfobj *const o)
 {
 	int bDestructObj = 1;
 	int bChkUnuse = 1;
+	assert(o != NULL);
 
 	dbgprintf("cnf:global:obj: ");
 	cnfobjPrint(o);

runtime/ruleset.c

@@ -686,9 +686,11 @@ GetParserList(rsconf_t *conf, smsg_t *pMsg)
 
 
 /* Add a script block to the current ruleset */
-static void
+static void ATTR_NONNULL(1)
-addScript(ruleset_t *pThis, struct cnfstmt *script)
+addScript(ruleset_t *const pThis, struct cnfstmt *const script)
 {
+	if(script == NULL) /* happens for include() */
+		return;
 	if(pThis->last == NULL)
 		pThis->root = pThis->last = script;
 	else {

tests/Makefile.am

@@ -310,6 +310,9 @@ TESTS +=  \
 endif # ENABLE_LIBCURL
 if HAVE_VALGRIND
 TESTS +=  \
+	include-obj-outside-control-flow-vg.sh \
+	include-obj-in-if-vg.sh \
+	include-obj-text-vg.sh \
 	rscript_parse_json-vg.sh \
 	rscript_backticks-vg.sh \
 	prop-jsonmesg-vg.sh
@@ -928,6 +931,9 @@ EXTRA_DIST= \
 	testsuites/diskqueue.conf \
 	arrayqueue.sh \
 	testsuites/arrayqueue.conf \
+	include-obj-outside-control-flow-vg.sh \
+	include-obj-in-if-vg.sh \
+	include-obj-text-vg.sh \
 	rscript_http_request.sh \
 	rscript_http_request-vg.sh \
 	rscript_bare_var_root.sh \

tests/include-obj-in-if-vg.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# added 2018-01-22 by Rainer Gerhards; Released under ASL 2.0
+. $srcdir/diag.sh init
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+template(name="outfmt" type="string" string="%msg:F,58:2%\n")
+
+if $msg contains "msgnum:" then {
+	include(file="testsuites/include-std-omfile-actio*.conf")
+	continue
+}
+'
+. $srcdir/diag.sh startup-vg
+. $srcdir/diag.sh injectmsg 0 10
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+. $srcdir/diag.sh seq-check 0 9
+. $srcdir/diag.sh exit

tests/include-obj-outside-control-flow-vg.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# added 2018-01-22 by Rainer Gerhards; Released under ASL 2.0
+. $srcdir/diag.sh init
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+template(name="outfmt" type="string" string="%msg:F,58:2%\n")
+
+if not ($msg contains "msgnum:") then {
+	stop
+}
+
+# Note: the point of this test is to have this include outside of
+# a control flow construct -- this the "strange" if above.
+include(file="testsuites/include-std-omfile-actio*.conf")
+'
+. $srcdir/diag.sh startup-vg
+. $srcdir/diag.sh injectmsg 0 10
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+. $srcdir/diag.sh seq-check 0 9
+. $srcdir/diag.sh exit

tests/include-obj-text-vg.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# added 2018-01-22 by Rainer Gerhards; Released under ASL 2.0
+. $srcdir/diag.sh init
+. $srcdir/diag.sh generate-conf
+export CONF_SNIPPET=`cat testsuites/include-std-omfile-action.conf`
+printf "\nThis SNIPPET will be included via env var:\n$CONF_SNIPPET\n\nEND SNIPPET\n"
+. $srcdir/diag.sh add-conf '
+template(name="outfmt" type="string" string="%msg:F,58:2%\n")
+
+if $msg contains "msgnum:" then {
+	include(text=`echo $CONF_SNIPPET`)
+}
+'
+. $srcdir/diag.sh startup-vg
+. $srcdir/diag.sh injectmsg 0 10
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+. $srcdir/diag.sh seq-check 0 9
+. $srcdir/diag.sh exit

tests/testsuites/include-std-omfile-action.conf

@@ -0,0 +1,3 @@
+# this include provides our standard omfile action. It is primarily
+# used for include() tests, but may have other uses as well.
+action(type="omfile" template="outfmt" file="rsyslog.out.log")