Merge pull request #2014 from rgerhards/i-1822bis

core/msg: fix segfault in MsgSetPropsViaJSON

runtime/msg.c

@@ -4590,8 +4590,10 @@ MsgSetPropsViaJSON_Object(smsg_t *__restrict__ const pMsg, struct json_object *j
 	struct json_object_iterator it = json_object_iter_begin(json);
 	struct json_object_iterator itEnd = json_object_iter_end(json);
 	while (!json_object_iter_equal(&it, &itEnd)) {
+		struct json_object *child = json_object_iter_peek_value(&it);
+		json_object_get(child);
 		msgSetPropViaJSON(pMsg, json_object_iter_peek_name(&it),
-			json_object_iter_peek_value(&it), 0);
+			child, 0);
 		json_object_iter_next(&it);
 	}
 	json_object_put(json);

tests/Makefile.am

@@ -290,6 +290,8 @@ endif
 
 if HAVE_VALGRIND
 TESTS +=  \
+	mmexternal-SegFault-vg.sh \
+	mmexternal-SegFault-empty-jroot-vg.sh \
 	internal-errmsg-memleak.sh \
 	rscript_set_memleak-vg.sh \
 	no-parser-vg.sh \
@@ -724,6 +726,9 @@ EXTRA_DIST= \
 	glbl_setenv_err.sh \
 	glbl_setenv_err_too_long.sh \
 	glbl_setenv.sh \
+	mmexternal-SegFault-vg.sh \
+	mmexternal-SegFault-empty-jroot-vg.sh \
+	testsuites/mmexternal-SegFault-mm-python.py \
 	nested-call-shutdown.sh \
 	1.rstest 2.rstest 3.rstest err1.rstest \
 	invalid_nested_include.sh \

tests/mmexternal-SegFault-empty-jroot-vg.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# add 2017-11-06 by PascalWithopf, released under ASL 2.0
+uname
+if [ `uname` = "FreeBSD" ] ; then
+   echo "This test currently does not work on FreeBSD."
+   exit 77
+fi
+
+. $srcdir/diag.sh init
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/mmexternal/.libs/mmexternal")
+input(type="imtcp" port="13514")
+
+template(name="outfmt" type="string" string="-%$!%-\n")
+
+if $msg contains "msgnum:" then {
+	action(type="mmexternal" interface.input="fulljson"
+		binary="testsuites/mmexternal-SegFault-mm-python.py")
+	action(type="omfile" template="outfmt" file="rsyslog.out.log")
+}
+'
+. $srcdir/diag.sh startup-vg
+. $srcdir/diag.sh tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag:msgnum:1\""
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+
+echo '-{ "sometag": "somevalue" }-' | cmp rsyslog.out.log
+if [ ! $? -eq 0 ]; then
+  echo "invalid response generated, rsyslog.out.log is:"
+  cat rsyslog.out.log
+  . $srcdir/diag.sh error-exit  1
+fi;
+
+. $srcdir/diag.sh exit

tests/mmexternal-SegFault-vg.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# add 2017-11-06 by PascalWithopf, released under ASL 2.0
+uname
+if [ `uname` = "FreeBSD" ] ; then
+   echo "This test currently does not work on FreeBSD."
+   exit 77
+fi
+
+. $srcdir/diag.sh init
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imtcp/.libs/imtcp")
+module(load="../plugins/mmexternal/.libs/mmexternal")
+input(type="imtcp" port="13514")
+set $!x = "a";
+
+template(name="outfmt" type="string" string="-%$!%-\n")
+
+if $msg contains "msgnum:" then {
+	action(type="mmexternal" interface.input="fulljson"
+		binary="testsuites/mmexternal-SegFault-mm-python.py")
+	action(type="omfile" template="outfmt" file="rsyslog.out.log")
+}
+'
+. $srcdir/diag.sh startup-vg
+. $srcdir/diag.sh tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag:msgnum:1\""
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+
+echo '-{ "x": "a", "sometag": "somevalue" }-' | cmp rsyslog.out.log
+if [ ! $? -eq 0 ]; then
+  echo "invalid response generated, rsyslog.out.log is:"
+  cat rsyslog.out.log
+  . $srcdir/diag.sh error-exit  1
+fi;
+
+. $srcdir/diag.sh exit

tests/testsuites/mmexternal-SegFault-mm-python.py

@@ -0,0 +1,85 @@
+#! /usr/bin/python
+
+"""A skeleton for a python rsyslog message modification plugin
+   Copyright (C) 2014 by Adiscon GmbH
+
+   This file is part of rsyslog.
+  
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+   
+         http://www.apache.org/licenses/LICENSE-2.0
+         -or-
+         see COPYING.ASL20 in the source distribution
+   
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+"""
+
+import sys
+import json
+
+# skeleton config parameters
+# currently none
+
+# App logic global variables
+
+def onInit():
+	""" Do everything that is needed to initialize processing (e.g.
+	    open files, create handles, connect to systems...)
+	"""
+	# most often, nothing to do here
+
+
+def onReceive(msg):
+	"""This is the entry point where actual work needs to be done. It receives
+	   the messge from rsyslog and now needs to examine it, do any processing
+	   necessary. The to-be-modified properties (one or many) need to be pushed
+	   back to stdout, in JSON format, with no interim line breaks and a line
+	   break at the end of the JSON. If no field is to be modified, empty 
+	   json ("{}") needs to be emitted.
+	   Note that no batching takes place (contrary to the output module skeleton)
+	   and so each message needs to be fully processed (rsyslog will wait for the
+	   reply before the next message is pushed to this module).
+	"""
+	data = json.loads(msg)
+        print json.dumps({"$!": {"sometag": "somevalue"}})
+#	print json.dumps({'msg': msg + "-modified"})
+
+def onExit():
+	""" Do everything that is needed to finish processing (e.g.
+	    close files, handles, disconnect from systems...). This is
+	    being called immediately before exiting.
+	"""
+	# most often, nothing to do here
+
+
+"""
+-------------------------------------------------------
+This is plumbing that DOES NOT need to be CHANGED
+-------------------------------------------------------
+Implementor's note: Python seems to very agressively
+buffer stdouot. The end result was that rsyslog does not
+receive the script's messages in a timely manner (sometimes
+even never, probably due to races). To prevent this, we
+flush stdout after we have done processing. This is especially
+important once we get to the point where the plugin does
+two-way conversations with rsyslog. Do NOT change this!
+See also: https://github.com/rsyslog/rsyslog/issues/22
+"""
+onInit()
+keepRunning = 1
+while keepRunning == 1:
+	msg = sys.stdin.readline()
+	if msg:
+		msg = msg[:-1] # remove LF
+		onReceive(msg)
+		sys.stdout.flush() # very important, Python buffers far too much!
+	else: # an empty line means stdin has been closed
+		keepRunning = 0
+onExit()
+sys.stdout.flush() # very important, Python buffers far too much!