adding hash64 and hash64mod functions support in rainerscript

removing logmsg

fixing typo in Makefile.am

fixing typo

making default hash static

fixing cast issue

fixing test

configure.ac

@@ -300,7 +300,6 @@ if test "$enable_regexp" = "yes"; then
         AC_DEFINE(FEATURE_REGEXP, 1, [Regular expressions support enabled.])
 fi
 
-
 # zlib support
 PKG_CHECK_MODULES([ZLIB], [zlib], [found_zlib=yes], [found_zlib=no])
 AS_IF([test "x$found_zlib" = "xno"], [
@@ -313,6 +312,30 @@ AS_IF([test "x$found_zlib" = "xno"], [
 ])
 
 
+#hash implementations header checks
+AC_ARG_ENABLE(hash-xxhash,
+	[AS_HELP_STRING([--enable-hash-xxhash],[Enable xxhash support @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_hash_xxhash="yes" ;;
+          no) enable_hash_xxhash="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-hash-xxhash) ;;
+         esac],
+	[enable_hash_xxhash=no]
+)
+
+AM_CONDITIONAL(ENABLE_HASH_XXHASH, test x$enable_hash_xxhash = xyes)
+if test "$enable_hash_xxhash" = "yes"; then  
+	AC_CHECK_LIB([xxhash], [XXH64], [
+		AC_CHECK_HEADER([xxhash.h], [
+			AC_DEFINE(USE_HASH_XXHASH, 1,
+				  [Using XXHASH for hash64.])
+			HASH_XXHASH_LIBS="-lxxhash"
+			AC_SUBST(HASH_XXHASH_LIBS)], 
+			[AC_MSG_ERROR([Unable to add XXHASH support for hash64.])])
+	])
+fi
+
+
 #gssapi
 AC_ARG_ENABLE(gssapi_krb5,
 	[AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])],
@@ -2224,6 +2247,7 @@ echo
 echo "    Large file support enabled:               $enable_largefile"
 echo "    Networking support enabled:               $enable_inet"
 echo "    Regular expressions support enabled:      $enable_regexp"
+echo "    xxhash support enabled:                   $enable_hash_xxhash"
 echo "    rsyslog runtime will be built:            $enable_rsyslogrt"
 echo "    rsyslogd will be built:                   $enable_rsyslogd"
 echo "    have to generate man pages:               $have_to_generate_man_pages"

grammar/rainerscript.c

@@ -55,6 +55,7 @@
 #include "wti.h"
 #include "unicode-helper.h"
 #include "errmsg.h"
+#include "hash-impl.h"
 
 #if !defined(_AIX)
 #pragma GCC diagnostic ignored "-Wswitch-enum"
@@ -1832,6 +1833,46 @@ doRandomGen(struct svar *__restrict__ const sourceVal) {
 	return x % max;
 }
 
+static uint64_t
+doHash64(struct svar *__restrict__ const sourceVal, struct svar *__restrict__ const seedVal) {
+	int freeHashStr = 0, success = 0;
+    uint64_t seed = 0;
+    if(seedVal) {
+        seed = var2Number(seedVal, &success);
+        if (!success) {
+            DBGPRINTF("rainerscript: hash64(string, seed) didn't get a valid 'seed' limit, defaulting hash value to 0");
+            return 0;
+        }
+    }
+	
+    es_str_t *hashStr = var2String(sourceVal, &freeHashStr);
+    uchar *src = es_getBufAddr(hashStr);
+    size_t len = es_strlen(hashStr);
+	uint64_t xhash = hash64(src, len, seed);
+	if (freeHashStr) es_deleteStr(hashStr);
+    DBGPRINTF("rainerscript: hash64 generated hash %" PRIu64 " for string(%.*s)", xhash, (int)len, src);
+	return xhash;
+}
+
+static uint64_t
+doHash64Mod(struct svar *__restrict__ const sourceVal, struct svar *__restrict__ const modVal, struct svar *__restrict__ const seedVal) {
+    
+    int success = 0;
+	uint64_t mod = var2Number(modVal, &success);
+	if (! success) {
+		DBGPRINTF("rainerscript: hash64mod(string, mod)/hash64mod(string, mod, seed) didn't get a valid 'mod' limit, defaulting hash value to 0");
+		return 0;
+	}
+	if(mod == 0) {
+		DBGPRINTF("rainerscript: hash64mod(string, mod)/hash64mod(string, mod, seed) invalid, 'mod' is zero, , defaulting hash value to 0");
+		return 0;
+	}
+    
+    uint64_t xhash = doHash64(sourceVal, seedVal) % mod;
+    DBGPRINTF("rainerscript: hash64mod generated hash-mod %" PRIu64 ".", xhash);
+	return xhash % mod;
+}
+
 static es_str_t*
 lTrim(char *str)
 {
@@ -2244,7 +2285,25 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct svar *__restrict__ co
 		ret->datatype = 'N';
 		varFreeMembers(&r[0]);
 		break;
-	case CNFFUNC_NUM2IPV4:
+    case CNFFUNC_HASH64:
+		cnfexprEval(func->expr[0], &r[0], usrptr, pWti);
+        if(func->nParams == 2) cnfexprEval(func->expr[1], &r[1], usrptr, pWti);
+		ret->d.n = doHash64(&r[0], (func->nParams == 2 ? &r[1] : NULL));
+		ret->datatype = 'N';
+		varFreeMembers(&r[0]);
+        if(func->nParams == 2) varFreeMembers(&r[1]);
+		break;
+    case CNFFUNC_HASH64MOD:
+		cnfexprEval(func->expr[0], &r[0], usrptr, pWti);
+		cnfexprEval(func->expr[1], &r[1], usrptr, pWti);
+		if(func->nParams == 3) cnfexprEval(func->expr[2], &r[2], usrptr, pWti);
+		ret->d.n = doHash64Mod(&r[0], &r[1], func->nParams > 2 ? &r[2] : NULL);
+		ret->datatype = 'N';
+		varFreeMembers(&r[0]);
+		varFreeMembers(&r[1]);
+		if(func->nParams == 3) varFreeMembers(&r[2]);
+		break;
+    case CNFFUNC_NUM2IPV4:
 		cnfexprEval(func->expr[0], &r[0], usrptr, pWti);
 		ret->d.estr = num2ipv4(&r[0]);
 		ret->datatype = 'S';
@@ -4734,6 +4793,18 @@ funcName2ID(es_str_t *fname, unsigned short nParams)
 			"but is %d.");
 	} else if(FUNC_NAME("random")) {
 		GENERATE_FUNC("random", 1, CNFFUNC_RANDOM);
+	} else if(FUNC_NAME("hash64")) {
+        GENERATE_FUNC_WITH_NARG_RANGE("hash64", 1, 2, CNFFUNC_HASH64,
+			"number of parameters for %s() must either be "
+			"one (operand_string) or"
+			"two (operand_string, number_seed)"
+			"but is %d.");
+	} else if(FUNC_NAME("hash64mod")) {
+        GENERATE_FUNC_WITH_NARG_RANGE("hash64mod", 2, 3, CNFFUNC_HASH64MOD,
+			"number of parameters for %s() must either be "
+			"two (operand_string, number_mod_to) or"
+			"three (operand_string, number_mod_to, number_seed)"
+			"but is %d.");
 	} else if(FUNC_NAME("format_time")) {
 		GENERATE_FUNC("format_time", 2, CNFFUNC_FORMAT_TIME);
 	} else if(FUNC_NAME("parse_time")) {

grammar/rainerscript.h

@@ -233,6 +233,8 @@ enum cnffuncid {
 	CNFFUNC_REPLACE,
 	CNFFUNC_WRAP,
 	CNFFUNC_RANDOM,
+    CNFFUNC_HASH64,
+    CNFFUNC_HASH64MOD,
 	CNFFUNC_DYN_INC,
 	CNFFUNC_IPV42NUM,
 	CNFFUNC_NUM2IPV4,

runtime/Makefile.am

@@ -87,6 +87,8 @@ librsyslog_la_SOURCES = \
 	../parse.c \
 	../parse.h \
 	\
+	hash-impl.h \
+	hash-impl.c \
 	hashtable.c \
 	hashtable.h \
 	hashtable_itr.c \
@@ -133,6 +135,17 @@ lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(LIBLOGGING_STDLOG_CFLA
 lmzlibw_la_LDFLAGS = -module -avoid-version $(LIBLOGGING_STDLOG_LIBS)
 lmzlibw_la_LIBADD =
 
+#
+# xxhash support
+# 
+if ENABLE_HASH_XXHASH
+pkglib_LTLIBRARIES += lmxxhash.la
+lmxxhash_la_SOURCES = xxhash.h
+lmxxhash_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(LIBLOGGING_STDLOG_CFLAGS)
+lmxxhash_la_LDFLAGS = -module -avoid-version $(LIBLOGGING_STDLOG_LIBS)
+lmxxhash_la_LIBADD = $(HASH_XXHASH_LIBS)
+endif
+
 if ENABLE_INET
 pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la
 #

runtime/hash-impl.c

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2018-2020, Harshvardhan Shrivastava
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * 
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include "config.h"
+#include "hash-impl.h"
+#ifdef USE_HASH_XXHASH
+#  include <xxhash.h>
+#endif
+
+/*
+ * Modified Bernstein
+ * http://www.eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
+ */
+static uint64_t djb_hash(const void* input, size_t len, uint64_t seed) {
+    const char *p = input;
+    uint64_t hash = 5381;
+    uint64_t i;
+    for (i = 0; i < len; i++) {
+        hash = 33 * hash ^ p[i];
+    }
+
+    return hash + seed;
+}
+
+/*Get 64 bit hash for input*/
+uint64_t hash64(const void* input, size_t len, uint64_t seed) {
+    #ifdef USE_HASH_XXHASH
+        return XXH64(input, len, seed);
+    #else
+        return djb_hash(input, len, seed);
+    #endif
+}

runtime/hash-impl.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2018-2020, Harshvardhan Shrivastava
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of the original author; nor the names of any contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * 
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER
+ * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#ifndef __HASH_IMPL_H__
+#define __HASH_IMPL_H__
+
+#include <stdint.h>
+#include <stddef.h>
+
+/*Get 64 bit hash for input*/
+uint64_t hash64(const void* input, size_t len, uint64_t seed);
+
+#endif

tests/Makefile.am

@@ -533,6 +533,7 @@ TESTS +=  \
 	imptcp-NUL.sh \
 	imptcp-NUL-rawmsg.sh \
 	rscript_random.sh \
+	rscript_hash64.sh \
 	rscript_replace.sh
 if HAVE_VALGRIND
 TESTS +=  \
@@ -1650,6 +1651,8 @@ EXTRA_DIST= \
 	testsuites/tokenized_input \
 	rscript_random.sh \
 	testsuites/rscript_random.conf \
+	rscript_hash64.sh \
+	testsuites/rscript_hash64.conf \
 	rscript_replace.sh \
 	testsuites/rscript_replace.conf \
 	rscript_replace_complex.sh \

tests/rscript_hash64.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# added 2018-02-07 by Harshvardhan Shrivastava
+# This file is part of the rsyslog project, released under ASL 2.0
+echo ===============================================================================
+echo \rscript_hash64.sh\]: test for hash64 and hash64mod script-function
+. $srcdir/diag.sh init
+. $srcdir/diag.sh startup rscript_hash64.conf
+. $srcdir/diag.sh tcpflood -m 20
+echo doing shutdown
+. $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+. $srcdir/diag.sh wait-shutdown
+. $srcdir/diag.sh content-pattern-check "^-2574714428477944902 -  14\|-50452361579464591 -  25$"
+. $srcdir/diag.sh exit

tests/testsuites/rscript_hash64.conf

@@ -0,0 +1,10 @@
+$IncludeConfig diag-common.conf
+template(name="outfmt" type="string" string="%$.hash_no_1% -  %$.hash_no_2%\n")
+
+module(load="../plugins/imtcp/.libs/imtcp")
+input(type="imtcp" port="13514")
+
+set $.hash_no_1 = hash64("0f9a1d07-a8c9-43a7-a6f7-198dca3d932e");
+set $.hash_no_2 = hash64mod("0f9a1d07-a8c9-43a7-a6f7-198dca3d932e", 100);
+
+action(type="omfile" file="./rsyslog.out.log" template="outfmt")

tools/Makefile.am

@@ -41,7 +41,7 @@ rsyslogd_CPPFLAGS =  $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(LIBLOGGING_STDLOG_CFLAG
 # note: it looks like librsyslog.la must be explicitely given on LDDADD,
 # otherwise dependencies are not properly calculated (resulting in a
 # potentially incomplete build, a problem we had several times...)
-rsyslogd_LDADD = ../grammar/libgrammar.la ../runtime/librsyslog.la ../compat/compat.la $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBUUID_LIBS)
+rsyslogd_LDADD = ../grammar/libgrammar.la ../runtime/librsyslog.la ../compat/compat.la $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBUUID_LIBS) $(HASH_XXHASH_LIBS)
 
 # if you know how to do an "if AIX os OS_APPLE" or an elseif chain, let me now!
 #rsyslogd_LDFLAGS = -export-dynamic \