Merge pull request #4551 from julthomas/dev/jth/fmunflatten

fmunflatten: rscript function to unflatten keys in a JSON object

Makefile.am

@@ -198,6 +198,10 @@ if ENABLE_FMHASH
 SUBDIRS += contrib/fmhash
 endif
 
+if ENABLE_FMUNFLATTEN
+SUBDIRS += contrib/fmunflatten
+endif
+
 if ENABLE_OMKAFKA
 SUBDIRS += plugins/omkafka
 endif

configure.ac

@@ -388,6 +388,19 @@ if test "$enable_fmhash_xxhash" = "yes"; then
 fi
 
 
+# rscript function unflatten
+AC_ARG_ENABLE(fmunflatten,
+	[AS_HELP_STRING([--enable-fmunflatten],[Enable fmunflatten support @<:@default=yes@:>@])],
+        [case "${enableval}" in
+         yes) enable_fmunflatten="yes" ;;
+          no) enable_fmunflatten="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-fmunflatten) ;;
+         esac],
+	[enable_fmunflatten=yes]
+)
+AM_CONDITIONAL(ENABLE_FMUNFLATTEN, test x$enable_fmunflatten = xyes)
+
+
 #gssapi
 AC_ARG_ENABLE(gssapi_krb5,
 	[AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])],
@@ -2639,6 +2652,7 @@ AC_CONFIG_FILES([Makefile \
     contrib/mmdarwin/Makefile \
 		contrib/omhttp/Makefile \
 		contrib/fmhash/Makefile \
+		contrib/fmunflatten/Makefile \
 		contrib/pmsnare/Makefile \
 		contrib/pmpanngfw/Makefile \
 		contrib/pmaixforwardedfrom/Makefile \
@@ -2784,6 +2798,7 @@ echo "---{ function modules }---"
 echo "    fmhttp enabled:                           $enable_fmhttp"
 echo "    fmhash enabled:                           $enable_fmhash"
 echo "    fmhash with xxhash enabled:               $enable_fmhash_xxhash"
+echo "    fmunflatten enabled:                      $enable_fmunflatten"
 echo
 echo "---{ debugging support }---"
 echo "    distcheck workaround enabled:             $enable_distcheck_workaround"

contrib/fmunflatten/Makefile.am

@@ -0,0 +1,5 @@
+pkglib_LTLIBRARIES = fmunflatten.la
+fmunflatten_la_SOURCES = fmunflatten.c
+fmunflatten_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(JSON_C_CFLAGS)
+fmunflatten_la_LDFLAGS = -module -avoid-version
+fmunflatten_la_LIBADD = $(JSON_C_LIBS)

contrib/fmunflatten/fmunflatten.c

@@ -0,0 +1,243 @@
+/*
+ * This is a function module providing ability to unflatten a JSON tree.
+ *
+ * Copyright 2020 Julien Thomas < jthomas @ zenetys.com >
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *       -or-
+ *       see COPYING.ASL20 in the source distribution
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+
+#include <stdint.h>
+#include <stddef.h>
+#ifndef _AIX
+#include <typedefs.h>
+#endif
+#include <sys/types.h>
+#include <string.h>
+
+#include "rsyslog.h"
+
+#include "errmsg.h"
+#include "msg.h"
+#include "parserif.h"
+#include "module-template.h"
+#include "rainerscript.h"
+#include "wti.h"
+
+#define FMUNFLATTEN_KBUFLEN 256
+#define _jso_type(x) json_type_to_name(json_object_get_type(x))
+
+MODULE_TYPE_FUNCTION
+MODULE_TYPE_NOKEEP
+DEF_FMOD_STATIC_DATA
+
+struct unflatten_ctx {
+	char *kbuf;
+	size_t kbuf_len;
+	char delim;
+};
+
+/* forward declarations */
+void unflatten_add(struct unflatten_ctx *ctx, struct json_object *dst, const char *key, struct json_object *value);
+void unflatten(struct unflatten_ctx *ctx, struct json_object *src, struct json_object *dst);
+
+void unflatten_add(struct unflatten_ctx *ctx,
+	struct json_object *dst,
+	const char *key,
+	struct json_object *value)
+{
+	const char *p = key;
+	const char *q = p;
+	int depth = 0;
+	size_t klen;
+	struct json_object *o;
+	json_bool exists_already;
+	int create;
+
+	while (1) {
+		while (*q != ctx->delim && *q != '\0')
+			q++;
+		klen = q - p;
+		if (klen + 1 > ctx->kbuf_len) {
+			DBGPRINTF("warning: flat key \"%.20s...\" truncated at depth #%d, buffer too "
+				"small (max %zd)\n", key, depth, ctx->kbuf_len);
+			klen = ctx->kbuf_len - 1;
+		}
+		memcpy(ctx->kbuf, p, klen);
+		ctx->kbuf[klen] = '\0';
+		exists_already = json_object_object_get_ex(dst, ctx->kbuf, &o);
+
+		if (*q == ctx->delim) {
+			if (!exists_already)
+				create = 1;
+			else if (json_object_is_type(o, json_type_object))
+				create = 0;
+			else {
+				DBGPRINTF("warning: while processing flat key \"%s\" at depth #%d (intermediate "
+					"node), overriding existing value of type %s by an object\n", key, depth,
+					_jso_type(o));
+				json_object_object_del(dst, ctx->kbuf);
+				create = 1;
+			}
+			if (create) {
+				o = json_object_new_object();
+				json_object_object_add(dst, ctx->kbuf, o);
+			}
+			dst = o;
+			p = q + 1;
+			q = p;
+			depth++;
+		}
+		else if (*q == '\0') {
+			if (json_object_is_type(value, json_type_object)) {
+				if (exists_already) {
+					if (!json_object_is_type(o, json_type_object)) {
+						DBGPRINTF("warning: while processing flat key \"%s\" at depth #%d "
+							"(final node), overriding existing value of type %s by an "
+							"object\n", key, depth, _jso_type(o));
+						json_object_object_del(dst, ctx->kbuf);
+						o = json_object_new_object();
+						json_object_object_add(dst, ctx->kbuf, o);
+					}
+				}
+				else {
+					o = json_object_new_object();
+					json_object_object_add(dst, ctx->kbuf, o);
+				}
+				unflatten(ctx, value, o);
+			}
+			else {
+				if (exists_already) {
+					DBGPRINTF("warning: while processing flat key \"%s\" at depth #%d "
+						"(final node), overriding existing value\n", key, depth);
+					json_object_object_del(dst, ctx->kbuf);
+				}
+				o = jsonDeepCopy(value);
+				json_object_object_add(dst, ctx->kbuf, o);
+			}
+			break;
+		}
+	}
+}
+
+void unflatten(struct unflatten_ctx *ctx,
+	struct json_object *src,
+	struct json_object *dst)
+{
+	struct json_object_iterator it = json_object_iter_begin(src);
+	struct json_object_iterator itEnd = json_object_iter_end(src);
+	const char *key;
+	struct json_object *value;
+
+	while (!json_object_iter_equal(&it, &itEnd)) {
+		key = json_object_iter_peek_name(&it);
+		value = json_object_iter_peek_value(&it);
+		unflatten_add(ctx, dst, key, value);
+		json_object_iter_next(&it);
+	}
+}
+
+static void ATTR_NONNULL()
+doFunc_unflatten(struct cnffunc *__restrict__ const func,
+	struct svar *__restrict__ const ret,
+	void *__restrict__ const usrptr,
+	wti_t *__restrict__ const pWti)
+{
+	struct svar src_var;
+	struct svar delim_var;
+
+	char kbuf[FMUNFLATTEN_KBUFLEN];
+	struct unflatten_ctx ctx = {
+		.kbuf = kbuf,
+		.kbuf_len = sizeof(kbuf),
+		.delim = 0
+	};
+
+	/* A dummy value of 0 (number) is returned by default. Callers should also
+	 * call script_error() to check if this script function succeeded before
+	 * using the value it returns. */
+	ret->datatype = 'N';
+	ret->d.n = 0;
+	wtiSetScriptErrno(pWti, RS_SCRIPT_EINVAL);
+
+	cnfexprEval(func->expr[0], &src_var, usrptr, pWti);
+	cnfexprEval(func->expr[1], &delim_var, usrptr, pWti);
+
+	/* Check argument 2 (delimiter character). */
+	if (delim_var.datatype == 'S' && es_strlen(delim_var.d.estr) == 1)
+		ctx.delim = *es_getBufAddr(delim_var.d.estr);
+	else if (delim_var.datatype == 'N')
+		ctx.delim = delim_var.d.n;
+	if (ctx.delim == 0) {
+		LogError(0, RS_RET_INVALID_PARAMS, "unflatten: invalid argument 2 (delim), single character "
+			"required (as string or decimal charcode)");
+		FINALIZE;
+	}
+
+	/* Check argument 1 (source). Not logging an error avoids emitting logs for
+	 * messages when $! was not touched. */
+	if (src_var.datatype != 'J') {
+		DBGPRINTF("unsupported argument 1 (src) datatype %c\n", src_var.datatype);
+		FINALIZE;
+	}
+
+	ret->datatype = 'J';
+	if (json_object_is_type(src_var.d.json, json_type_object)) {
+		ret->d.json = json_object_new_object();
+		unflatten(&ctx, src_var.d.json, ret->d.json);
+	}
+	else
+		ret->d.json = jsonDeepCopy(src_var.d.json);
+	wtiSetScriptErrno(pWti, RS_SCRIPT_EOK);
+
+finalize_it:
+	varFreeMembers(&src_var);
+	varFreeMembers(&delim_var);
+}
+
+static rsRetVal ATTR_NONNULL(1)
+initFunc_unflatten(struct cnffunc *const func)
+{
+	DEFiRet;
+	func->destructable_funcdata = 0;
+	RETiRet;
+}
+
+static struct scriptFunct functions[] = {
+	{ "unflatten", 2, 2, doFunc_unflatten, initFunc_unflatten, NULL },
+	{ NULL, 0, 0, NULL, NULL, NULL } /* last element to check end of array */
+};
+
+BEGINgetFunctArray
+CODESTARTgetFunctArray
+	*version = 1;
+	*functArray = functions;
+ENDgetFunctArray
+
+BEGINmodExit
+CODESTARTmodExit
+ENDmodExit
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_FMOD_QUERIES
+ENDqueryEtryPt
+
+BEGINmodInit()
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	dbgprintf("rsyslog fmunflatten init called, compiled with version %s\n", VERSION);
+ENDmodInit

runtime/msg.c

@@ -423,7 +423,7 @@ static int getAPPNAMELen(smsg_t * const pM, sbool bLockMutex);
 static rsRetVal jsonPathFindParent(struct json_object *jroot, uchar *name, uchar *leaf,
 	struct json_object **parent, int bCreate);
 static uchar * jsonPathGetLeaf(uchar *name, int lenName);
-static struct json_object *jsonDeepCopy(struct json_object *src);
+struct json_object *jsonDeepCopy(struct json_object *src);
 static json_bool jsonVarExtract(struct json_object* root, const char *key, struct json_object **value);
 void getRawMsgAfterPRI(smsg_t * const pM, uchar **pBuf, int *piLen);
 
@@ -5123,7 +5123,7 @@ finalize_it:
 	RETiRet;
 }
 
-static struct json_object *
+struct json_object *
 jsonDeepCopy(struct json_object *src)
 {
 	struct json_object *dst = NULL, *json;

runtime/msg.h

@@ -240,6 +240,7 @@ unsigned short *pbMustBeFreed);
 rsRetVal msgSetJSONFromVar(smsg_t *pMsg, uchar *varname, struct svar *var, int force_reset);
 rsRetVal msgDelJSON(smsg_t *pMsg, uchar *varname);
 rsRetVal jsonFind(smsg_t *const pMsg, msgPropDescr_t *pProp, struct json_object **jsonres);
+struct json_object *jsonDeepCopy(struct json_object *src);
 
 rsRetVal msgPropDescrFill(msgPropDescr_t *pProp, uchar *name, int nameLen);
 void msgPropDescrDestruct(msgPropDescr_t *pProp);

tests/Makefile.am

@@ -1055,6 +1055,30 @@ TESTS +=  \
 	rscript_hash64-vg.sh
 endif # ENABLE_FMHASH
 endif # HAVE_VALGRIND
+if ENABLE_FMUNFLATTEN
+TESTS +=  \
+	rscript_unflatten_arg1_unsuitable.sh \
+	rscript_unflatten_arg2_invalid.sh \
+	rscript_unflatten_conflict1.sh \
+	rscript_unflatten_conflict2.sh \
+	rscript_unflatten_conflict3.sh \
+	rscript_unflatten_key_truncated.sh \
+	rscript_unflatten_non_object.sh \
+	rscript_unflatten_object.sh \
+	rscript_unflatten_object_exclamation.sh
+if HAVE_VALGRIND
+TESTS +=  \
+	rscript_unflatten_arg1_unsuitable-vg.sh \
+	rscript_unflatten_arg2_invalid-vg.sh \
+	rscript_unflatten_conflict1-vg.sh \
+	rscript_unflatten_conflict2-vg.sh \
+	rscript_unflatten_conflict3-vg.sh \
+	rscript_unflatten_key_truncated-vg.sh \
+	rscript_unflatten_non_object-vg.sh \
+	rscript_unflatten_object-vg.sh \
+	rscript_unflatten_object_exclamation-vg.sh
+endif # HAVE_VALGRIND
+endif # ENABLE_FMUNFLATTEN
 endif
 
 if ENABLE_MMPSTRUCDATA
@@ -1743,6 +1767,24 @@ EXTRA_DIST= \
 	rscript_exists-not2.sh \
 	rscript_exists-not3.sh \
 	rscript_exists-not4.sh \
+	rscript_unflatten_arg1_unsuitable.sh \
+	rscript_unflatten_arg2_invalid.sh \
+	rscript_unflatten_conflict1.sh \
+	rscript_unflatten_conflict2.sh \
+	rscript_unflatten_conflict3.sh \
+	rscript_unflatten_key_truncated.sh \
+	rscript_unflatten_non_object.sh \
+	rscript_unflatten_object_exclamation.sh \
+	rscript_unflatten_object.sh \
+	rscript_unflatten_arg1_unsuitable-vg.sh \
+	rscript_unflatten_arg2_invalid-vg.sh \
+	rscript_unflatten_conflict1-vg.sh \
+	rscript_unflatten_conflict2-vg.sh \
+	rscript_unflatten_conflict3-vg.sh \
+	rscript_unflatten_key_truncated-vg.sh \
+	rscript_unflatten_non_object-vg.sh \
+	rscript_unflatten_object_exclamation-vg.sh \
+	rscript_unflatten_object-vg.sh \
 	rs-cnum.sh \
 	rs-int2hex.sh \
 	rs-substring.sh \

tests/rscript_unflatten_arg1_unsuitable-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_arg1_unsuitable.sh

tests/rscript_unflatten_arg1_unsuitable.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+EXPECTED=' msgnum:00000000: 1 0'
+cmp_exact "$RSYSLOG_OUT_LOG"
+exit_test

tests/rscript_unflatten_arg2_invalid-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_arg2_invalid.sh

tests/rscript_unflatten_arg2_invalid.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $!a.b.c = "foobar";
+set $.unflatten = unflatten($!, "too many chars");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+EXPECTED=' msgnum:00000000: 1 0'
+cmp_exact "$RSYSLOG_OUT_LOG"
+exit_test

tests/rscript_unflatten_conflict1-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_conflict1.sh

tests/rscript_unflatten_conflict1.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $!a!b = "foo";
+set $!a.b.c = "bar";
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "a": { "b": { "c": "bar" } } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+
+EXPECTED='fmunflatten.c: warning: while processing flat key "a.b.c" at depth #1 (intermediate node), overriding existing value of type string by an object'
+if ! grep -F "$EXPECTED" "$RSYSLOG_DEBUGLOG"; then
+    echo "GREP FAILED"
+    echo "  => FILE: $RSYSLOG_DEBUGLOG"
+    echo "  => EXPECTED: $EXPECTED"
+    error_exit 1
+fi
+
+exit_test

tests/rscript_unflatten_conflict2-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_conflict2.sh

tests/rscript_unflatten_conflict2.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $.x!a = 21;
+set $!a!b = "foo";
+set $!a.b = $.x;
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "a": { "b": { "a": 21 } } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+
+EXPECTED='fmunflatten.c: warning: while processing flat key "a.b" at depth #1 (final node), overriding existing value of type string by an object'
+if ! grep -F "$EXPECTED" "$RSYSLOG_DEBUGLOG"; then
+    echo "GREP FAILED"
+    echo "  => FILE: $RSYSLOG_DEBUGLOG"
+    echo "  => EXPECTED: $EXPECTED"
+    error_exit 1
+fi
+
+exit_test

tests/rscript_unflatten_conflict3-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_conflict3.sh

tests/rscript_unflatten_conflict3.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $!a!b = "foo";
+set $!a.b = "bar";
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "a": { "b": "bar" } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+
+EXPECTED='fmunflatten.c: warning: while processing flat key "a.b" at depth #1 (final node), overriding existing value'
+if ! grep -F "$EXPECTED" "$RSYSLOG_DEBUGLOG"; then
+    echo "GREP FAILED"
+    echo "  => FILE: $RSYSLOG_DEBUGLOG"
+    echo "  => EXPECTED: $EXPECTED"
+    error_exit 1
+fi
+
+exit_test

tests/rscript_unflatten_key_truncated-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_key_truncated.sh

tests/rscript_unflatten_key_truncated.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $!a.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb255ccccc.d = "bar";
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 1
+wait_file_lines "$RSYSLOG_OUT_LOG" 1 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "a": { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb255": { "d": "bar" } } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+
+EXPECTED='fmunflatten.c: warning: flat key "a.bbbbbbbbbbbbbbbbbb..." truncated at depth #1, buffer too small (max 256)'
+if ! grep -F "$EXPECTED" "$RSYSLOG_DEBUGLOG"; then
+    echo "GREP FAILED"
+    echo "  => FILE: $RSYSLOG_DEBUGLOG"
+    echo "  => EXPECTED: $EXPECTED"
+    error_exit 1
+fi
+
+exit_test

tests/rscript_unflatten_non_object-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_non_object.sh

tests/rscript_unflatten_non_object.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# added 2021-03-09 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $/cpt = $/cpt + 1;
+if ($/cpt == 1) then
+    set $! = "string";
+else
+    set $! = 42;
+
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 2
+wait_file_lines "$RSYSLOG_OUT_LOG" 2 60
+shutdown_when_empty
+wait_shutdown
+
+EXPECTED=' msgnum:00000000: 0 string
+ msgnum:00000001: 0 42'
+cmp_exact "$RSYSLOG_OUT_LOG"
+exit_test

tests/rscript_unflatten_object-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_object.sh

tests/rscript_unflatten_object.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# added 2020-08-16 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $!source.ip = "1.2.3.4";
+set $!source.bytes = 3258;
+set $!source.geo.country_iso_code = "FR";
+set $!destination.ip = "4.3.2.1";
+
+set $.unflatten = unflatten($!, ".");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 3
+wait_file_lines "$RSYSLOG_OUT_LOG" 3 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "source": { "ip": "1.2.3.4", "bytes": 3258, "geo": { "country_iso_code": "FR" } }, "destination": { "ip": "4.3.2.1" } }
+ msgnum:00000001: 0 { "source": { "ip": "1.2.3.4", "bytes": 3258, "geo": { "country_iso_code": "FR" } }, "destination": { "ip": "4.3.2.1" } }
+ msgnum:00000002: 0 { "source": { "ip": "1.2.3.4", "bytes": 3258, "geo": { "country_iso_code": "FR" } }, "destination": { "ip": "4.3.2.1" } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+exit_test

tests/rscript_unflatten_object_exclamation-vg.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export USE_VALGRIND="YES"
+source ${srcdir:-.}/rscript_unflatten_object_exclamation.sh

tests/rscript_unflatten_object_exclamation.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# added 2020-08-16 by Julien Thomas, released under ASL 2.0
+
+source "${srcdir:=.}/diag.sh" init
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debug"
+
+generate_conf
+add_conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../contrib/fmunflatten/.libs/fmunflatten")
+input(type="imtcp" port="0" listenPortFileName="'"$RSYSLOG_DYNNAME"'.tcpflood_port")
+template(name="outfmt" type="string" string="%msg% %$.ret% %$.unflatten%\n")
+
+if (not($msg contains "msgnum:")) then
+    stop
+
+set $.ret = parse_json("{\"source!ip\":\"1.2.3.4\",\"source!port\":53}", "\$!");
+set $.unflatten = unflatten($!, "!");
+set $.ret = script_error();
+action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt")
+'
+startup
+tcpflood -m 2
+wait_file_lines "$RSYSLOG_OUT_LOG" 2 60
+shutdown_when_empty
+wait_shutdown
+
+# this test may need changes to produce a more deterministic
+# output by sorting keys
+EXPECTED=' msgnum:00000000: 0 { "source": { "ip": "1.2.3.4", "port": 53 } }
+ msgnum:00000001: 0 { "source": { "ip": "1.2.3.4", "port": 53 } }'
+cmp_exact "$RSYSLOG_OUT_LOG"
+exit_test