diff --git a/.travis.yml b/.travis.yml index e5c89b7e3..1a5bc2357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,73 @@ language: c +compiler: + - gcc + - clang + +# you can comment this in if a large number of known-good +# merges is to happen and only buildbot testing is desired +#branches: +# except: +# - master-candidate + +matrix: + exclude: + - compiler: "gcc" + - compiler: "clang" + include: + # note: -Werror makes ./configure.sh not properly detect functions + # like strndup() on Ubuntu 12.04. So we enable it on trusty builds + # only. The first build with gcc actually only has the purpose of + # checking for warnings (the testbench currently has some issues + # on travis Ubuntu 14.04, but not in the regular environments, we + # need to find out some time why this is). + - compiler: "gcc" + env: CFLAGS="-g -std=c99 -Werror -W -Wall -Wextra -Wformat-security -Wshadow -Wcast-align -Wpointer-arith -Wmissing-format-attribute" + dist: trusty + - compiler: "clang" + env: STAT_AN="YES", GROK="YES", CFLAGS="-g -O1 -std=c99 -Werror" + dist: trusty + - compiler: "gcc" + env: BUILD_FROM_TARBALL="YES", GROK="YES", CHECK="YES", CFLAGS="-g -O2 -W -Wall -Wformat-security -Wshadow -Wcast-align -Wpointer-arith -Wmissing-format-attribute", RS_TESTBENCH_VALGRIND_EXTRA_OPTS="--suppressions=travis/trusty.supp --gen-suppressions=all" + dist: trusty + - compiler: "clang" + env: CHECK="YES", CFLAGS="-g -O1 -fsanitize=address -fno-color-diagnostics" + before_install: - - travis_retry sudo apt-get install -qq python-software-properties - - travis_retry sudo add-apt-repository ppa:adiscon/v8-devel -y + #- travis_retry sudo apt-get install -qq python-software-properties + - travis_retry sudo add-apt-repository ppa:adiscon/v8-stable -y - travis_retry sudo apt-get update -qq install: - - travis_retry sudo apt-get install -qq build-essential automake pkg-config libtool autoconf autotools-dev gdb valgrind libdbi-dev libsnmp-dev - - travis_retry sudo apt-get install -qq libestr-dev librelp-dev libjson0-dev zlib1g-dev uuid-dev libgcrypt11-dev liblogging-stdlog-dev bison flex + - travis_retry sudo apt-get install -qq build-essential automake pkg-config libtool autoconf autotools-dev gdb valgrind libdbi-dev libsnmp-dev libmysqlclient-dev postgresql-client faketime + - travis_retry sudo apt-get install -qq libestr-dev librelp-dev libfastjson-dev zlib1g-dev uuid-dev libgcrypt11-dev liblogging-stdlog-dev bison flex libksi1 libksi1-dev libcurl4-gnutls-dev - travis_retry sudo apt-get install -qq python-docutils liblognorm1-dev + - if [ "$CC" == "clang" ]; then travis_retry sudo apt-get install -qq clang; fi + - if [ "x$GROK" == "xYES" ]; then travis_retry sudo apt-get install -qq libglib2.0-dev libgrok1 libgrok-dev libtokyocabinet-dev ; fi script: + - if [ "x$BUILD_FROM_TARBALL" == "xYES" ]; then autoreconf -fvi && ./configure && make dist && mv *.tar.gz rsyslog.tar.gz && mkdir unpack && cd unpack && tar xzf ../rsyslog.tar.gz && ls -ld rsyslog* && cd rsyslog* ; fi + - pwd - autoreconf --force --verbose --install - # add "CC=clang" to configure if clang is desired - - ./configure --prefix=/opt/rsyslog --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --disable-silent-rules --libdir=/usr/lib64 --docdir=/usr/share/doc/rsyslog --disable-generate-man-pages --enable-testbench --enable-imdiag --enable-imfile --enable-impstats --enable-imptcp --enable-mmanon --enable-mmaudit --enable-mmfields --enable-mmjsonparse --enable-mmpstrucdata --enable-mmsequence --enable-mmutf8fix --enable-mail --enable-omprog --enable-omruleset --enable-omstdout --enable-omuxsock --enable-pmaixforwardedfrom --enable-pmciscoios --enable-pmcisconames --enable-pmlastmsg --enable-pmsnare --enable-libgcrypt --enable-mmnormalize --disable-omudpspoof --enable-relp --disable-snmp --disable-mmsnmptrapd --enable-gnutls --enable-valgrind + # I don't know how to pass two env vars in the include matrix, so + # I set the second one here via an "if" + - if [ "x$GROK" == "xYES" ]; then export GROK="--enable-mmgrok"; fi + - if [ "$CC" == "clang" ]; then export NO_VALGRIND="--without-valgrind-testbench"; fi + - ./configure --prefix=/opt/rsyslog --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --enable-silent-rules --libdir=/usr/lib64 --docdir=/usr/share/doc/rsyslog --disable-generate-man-pages --enable-testbench --enable-imdiag --enable-imfile --enable-impstats --enable-imptcp --enable-mmanon --enable-mmaudit --enable-mmfields --enable-mmjsonparse --enable-mmpstrucdata --enable-mmsequence --enable-mmutf8fix --enable-mail --enable-omprog --enable-omruleset --enable-omstdout --enable-omuxsock --enable-pmaixforwardedfrom --enable-pmciscoios --enable-pmcisconames --enable-pmlastmsg --enable-pmsnare --enable-libgcrypt --enable-mmnormalize --disable-omudpspoof --enable-relp --disable-snmp --disable-mmsnmptrapd --enable-gnutls --enable-mysql --enable-gt-ksi --enable-libdbi --enable-pgsql --enable-omhttpfs --enable-elasticsearch --enable-valgrind $NO_VALGRIND $GROK - export USE_AUTO_DEBUG="off" # set to "on" to enable this for travis - - make && make check #make dist && make check && sudo make install - - make distcheck + - make + - if [ "x$CHECK" == "xYES" ] ; then make check ; fi + - if [ -f tests/test-suite.log ] ; then cat tests/test-suite.log; fi + - if [ "x$CHECK" == "xYES" ] ; then make distcheck ; fi + - if [ "x$STAT_AN" == "xYES" ] ; then make clean ; fi + - if [ "x$STAT_AN" == "xYES" ] ; then cd compat; scan-build --status-bugs make && cd .. ; fi + # we now build those components that we know to need some more work + # they will not be included in the later static analyzer run. But by + # explicitely listing the modules which do not work, we automatically + # get new modules/files covered. + - if [ "x$STAT_AN" == "xYES" ] ; then cd runtime; make lmnet_la-net.lo libgcry_la-libgcry.lo ; cd .. ; fi + - if [ "x$STAT_AN" == "xYES" ] ; then scan-build --status-bugs make ; fi + # if that all worked, we go the traditional gcc & valgrind way #- env TESTS="json_array_looping.sh" make -e check # only for newer autoconf tools you need to add: # - cat tests/test-suite.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c81edc49..d5227598d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,49 @@ These are many ways to contribute to the project: This list is not conclusive. There for sure are many more ways to contribute and if you find one, just let us know. We are very open to new suggestions and like to try out new things. + +Requirements for patches +------------------------ +In order to ensure good code quality, after applying the path the code must + +- compile cleanly without WARNING messages under both gcc and clang +- pass clang static analyzer without any report + +Note that both warning messages and static analyzer warnings may be false +positives. We have decided to accept that fate and work around it (e.g. by +re-arranging the code, etc). Otherwise, we cannot use these useful features. + +As a last resort, compiler warnings can be turned off via + #pragma diagnostic +directives. This should really only be done if there is no other known +way around it. If so, it should be applied to a single function, only and +not to full source file. Be sure to re-enable the warning after the function +in question. We have done this in some few cases ourselfs, and if someone +can fix the root cause, we would appreciate help. But, again, this is a +last resort which should normally not be used. + +For pull requests submitted via github, these two conditions are +verified automatically. See the PR for potential failueres. For patches +submitted otherwise, they will be verified semi-manually. + +Also, patches are requested to not break the testbench. Unfortunately, the +current testbench has some racy tests, which are still useful enough so that +we do not want to disable them until the root cause has been found. If your +PR runs into something that you think is not related to your code, just sit +back and relax. The rsyslog core developer team reviews PRs regularly and +restarts tests which we know to look racy. If the problem persists, we will +contact you. + +Once all this has passed, a final integration test will be done via buildbot +on a larger number of platforms. This is a semi-manual process. You will be +contacted if a problem shows up during it (very unlikely, but occasionally +happens). + Note to developers ------------------ -Please adress pull requests against the master branch. The changes will at first be merged into another branch (master-candidate). There, the changes will be tested and merged into the master branch, should the test succeed. +Please adress pull requests against the master branch. The changes will at +first be merged into another branch (master-candidate). There, the +changes will be tested and merged into the master branch, should the test succeed. More information is available at: http://www.rsyslog.com/how-to-contribute-to-rsyslog/ diff --git a/ChangeLog b/ChangeLog index d6beb7f4a..6e4f5c3a5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,234 @@ ------------------------------------------------------------------------------ -Version 8.14.0 [v8-stable] 2015-11-03 +Version 8.17.0 [v8-stable] 2016-03-08 +- NEW REQUIREMENT: libfastjson + see also: +- new feature: dynamic statistics counters + which may be changed during rule processing + Thanks to Janmejay Singh for suggesting and implementing this feature +- simplified locking in when accessing message and local variables + this simlifies the code and slightly increases performance if such + variables are heavily accessed. +- imuxsock: add "ruleset" input parameter +- imptcp: performance improvements + Thanks to Janmejay Singh for implementing this improvement +- made build compile (almost) without warnings + still some warnings are suppressed where this is currently required +- improve interface definition in some modules, e.g. mmanon, mmsequence + This is more an internal cleanup and should have no actual affect to + the end user. +- bugfix: potential loss of single message at queue shutdown + see also: https://github.com/rsyslog/rsyslog/issues/262 +- bugfix: potential deadlock with heavy variable access + When making havy use of global, local and message variables, a deadlock + could occur. While it is extremly unlikely to happen, we have at least + seen one incarnation of this problem in practice. +- bugfix ommysql: on some platforms, serverport parameter had no effect + This was caused by an invalid code sequence which's outcome depends on + compiler settings. +- bugfix omelasticsearch: invalid pointer dereference + The actual practical impact is not clear. This came up when working + on compiler warnings. + Thanks to David Lang for the patch. +- bugfix rsgtutil: -h command line option did not work + Thanks to Henri Lakk for the patch. +- bugfix lexer: hex numbers were not properly represented + see: https://github.com/rsyslog/rsyslog/pull/771 + Thanks to Sam Hanes for the patch. +------------------------------------------------------------------------------ +Version 8.16.0 [v8-stable] 2016-01-26 +- rsgtutil: Added extraction support including loglines and hash chains. + More details on how to extract loglines can be found in the rsgtutil + manpage. See also: https://github.com/rsyslog/rsyslog/issues/561 +- clean up doAction output module interface + We started with char * pointers, but used different types of pointers + over time. This lead to alignment warnings. In practice, I think this + should never cause any problems (at least there have been no reports + in the 7 or so years we do this), but it is not clean. The interface is + now cleaned up. We do this in a way that does not require modifications + to modules that just use string parameters. For those with message + parameters, have a look at e.g. mmutf8fix to see how easy the + required change is. +- new system properties for $NOW properties based on UTC + This permits to express current system time in UTC. + See also https://github.com/rsyslog/rsyslog/issues/729 +- impstats: support broken ElasticSearch JSON implementation + ES 2.0 no longer supports valid JSON and disallows dots inside names. + This adds a new "json-elasticsearch" format option which replaces + those dots by the bang ("!") character. So "discarded.full" becomes + "discarded!full". + This is a workaroud. A method that will provide more control over + replacements will be implemented some time in the future. For + details, see below-quoted issue tracker. + closes https://github.com/rsyslog/rsyslog/issues/713 +- omelasticsearch: craft better URLs + Elasticsearch is confused by url's ending in a bare '?' or '&'. While + this is valid, those are no longer produced. + Thanks to Benno Evers for the patch. +- imfile: add experimental "reopenOnTruncate" parameter + Thanks to Matthew Wang for the patch. +- bugfix imfile: proper handling of inotify initialization failure + Thanks to Zachary Zhao for the patch. +- bugfix imfile: potential segfault due to improper handling of ev var + This occurs in inotify mode, only. + Thanks to Zachary Zhao and Peter Portante for the patch. + closes https://github.com/rsyslog/rsyslog/issues/718 +- bugfix imfile: potential segfault under heavey load. + This occurs in inotify mode when using wildcards, only. + The root cause is dropped IN_IGNOPRED inotify events which be dropped + in circumstance of high input pressure and frequent rotation, and + according to wikipeida, they can also be dropped in other conditions. + Thanks to Zachary Zhao for the patch. + closes https://github.com/rsyslog/rsyslog/issues/723 +- bugfix ommail: invalid handling of server response + if that response was split into different read calls. Could lead to + error-termination of send operation. Problem is pretty unlikely to + occur in standard setups (requires slow connection to SMTP server). + Thank to github user haixingood for the patch. +- bugfix omelasticsearch: custom serverport was ignored on some platforms + Thanks to Benno Evers for the patch. +- bugfix: tarball did not include some testbench files + Thanks to Thomas D. (whissi) for the patch. +- bugfix: memory misadressing during config parsing string template + This occurred if an (invalid) template option larger than 63 characters + was given. + Thanks to git hub user c6226 for the patch. +- bugfix imzmq: memory leak + Thanks to Jeremy Liang for the patch. +- bugfix imzmq: memory leak + Thanks to github user xushengping for the patch. +- bugfix omzmq: memory leak + Thanks to Jack Lin for the patch. +- some code improvement and cleanup +------------------------------------------------------------------------------ +Version 8.15.0 [v8-stable] 2015-12-15 +- KSI Lib: Updated code to run with libksi 3.4.0.5 + Also libksi 3.4.0.x is required to build rsyslog if ksi support + is enabled. New libpackages have been build as well. +- KSI utilities: Added option to ser publication url. + Since libksi 3.4.0.x, there is no default publication url anymore. + The publication url has to be set using the --publications-server + Parameter, otherwise the ksi signature cannot be verified. UserID + and UserKey can also be set by parameter now. + Closes https://github.com/rsyslog/rsyslog/issues/581 +- KSI Lib: Fixed wrong TLV container for KSI signatures from 0905 to 0906. + closes https://github.com/rsyslog/rsyslog/issues/587 +- KSI/GT Lib: Fixed multiple issues found using static analyzer +- performance improvement for configs with heavy use of JSON variables + Depending on the config, this can be a very big gain in performance. +- added pmpanngfw: contributed module for translating Palo Alto Networks logs. + see also: https://github.com/rsyslog/rsyslog/pull/573 + Thanks to Luigi Mori for the contribution. +- testbench: Changed valgrind option for imtcp-tls-basic-vg.sh + For details see: https://github.com/rsyslog/rsyslog/pull/569 +- pmciscoios: support for asterisk before timestamp added + thanks to github user c0by for the patch + see also: https://github.com/rsyslog/rsyslog/pull/583 - solr external output plugin much enhanced see also: https://github.com/rsyslog/rsyslog/pull/529 Thanks to Radu Gheorghe for the patch. +- omrabbitmq: improvements + thanks to Luigi Mori for the patch + see also: https://github.com/rsyslog/rsyslog/pull/580 +- add support for libfastjson (as a replacement for json-c) +- KSI utilities: somewhat improved error messages + Thanks to Henri Lakk for the patch. + see also: https://github.com/rsyslog/rsyslog/pull/588 +- pmciscoios: support for some format variations + Thanks to github user c0by for the patch +- support grok via new contributed module mmgrok + Thanks to 饶琛琳 (github user chenryn) for the contribution. +- omkafka: new statistics counter "maxoutqsize" + Thanks to 饶琛琳 (github user chenryn) for the contribution. +- improvments for 0mq modules: + * omczmq - suspend / Retry handling - the output plugin can now recover + from some error states due to issues with plugin startup or message sending + * omczmq - refactored topic handling code for ZMQ_PUB output to be a little + more efficient + * omczmq - added ability to set a timeout for sends + * omczmq - set topics can be in separate frame (default) or part of message + frame (configurable) + * omcmzq - code cleanup + * imczmq - code cleanup + * imczmq - fixed a couple of cases where vars could be used uninitialized + * imczmq - ZMQ_ROUTER support + * imczmq - Fix small memory leak from not freeing sockets when done with them + * allow creation of on demand ephemeral CurveZMQ certs for encryption. + Clients may specify clientcertpath="*" to indicate they want an on + demand generated cert. + Thanks to Brian Knox for the contributions. +- cleanup on code to unset a variable + under extreme cases (very, very unlikely), the old code could also lead + to errornous processing +- omelasticsearch: build on FreeBSD + Thanks to github user c0by for the patch +- pmciscoios: fix some small issues clang static analyzer detected +- testbench: many improvements and some new tests + note that there still is a number of tests which are somewhat racy +- overall code improvements thanks to clang static analyzer +- gnutls fix: Added possible fix for gnutls issue #575 + see also: https://github.com/rsyslog/rsyslog/issues/575 + Thanks to Charles Southerland for the patch +- bugfix omkafka: restore ability to build on all platforms + Undo commit aea09800643343ab8b6aa205b0f10a4be676643b + because that lead to build failures on various important platforms. + This means it currently is not possible to configure the location + of librdkafka, but that will affect far fewer people. + closes: https://github.com/rsyslog/rsyslog/issues/596 +- bugfix omkafka: fix potentially negative partition number + Thanks to Tait Clarridge for providing a patch. +- bugfix: solve potential race in creation of additional action workers + Under extreme circumstances, this could lead to segfault. Note that we + detected this problem thanks to ASAN address sanitzier in combination + with a very exterme testbench test. We do not think that this issue + was ever reported in practice. +- bugfix: potential memory leak in config parsing + Thanks to github user linmujia for the patch +- bugfix: small memory leak in loading template config + This happened when a plugin was used inside the template. Then, the + memory for the template name was never freed. + Thanks to github user xushengping for the fix. +- bugfix: fix extra whitespace in property expansions + Address off-by-one issues introduced in f3bd7a2 resulting in extra + whitespace in property expansions + Thanks to Matthew Gabeler-Lee for the patch. +- bugfix: mmfields leaked memory if very large messages were processed + detected by clang static analyzer +- bugfix: mmfields could add garbagge data to field + this happened when very large fields were to be processed. + Thanks to Peter Portante for reporting this. +- bugfix: omhttpfs now also compiles with older json-c lib +- bugfix: memory leak in (contributed) module omhttpfs + Thanks to git hub user c6226 for the patch. +- bugfix: parameter mismatch in error message for wrap() function +- bugfix: parameter mismatch in error message for random() function +- bugfix: divide by zero if max() function was provided zero +- bugfix: invalid mutex handling in omfile async write mode + could lead to segfault, even though highly unlikely (caught by + testbench on a single platform) +- bugfix: fix inconsistent number processing + Unfortunately, previous versions of the rule engine tried to + support oct and hex, but that wasn't really the case. + Everything based on JSON was just dec-converted. As this was/is + the norm, we fix that inconsistency by always using dec. + Luckly, oct and hex support was never documented and could + probably only have been activated by constant numbers. +- bugfix: timezone() object: fix NULL pointer dereference + This happened during startup when the offset or id parameter was not + given. Could lead to a segfault at startup. + Detected by clang static analyzer. +- bugfix omfile: memory addressing error if very long outchannel name used + Thanks to github user c6226 for the patch. +------------------------------------------------------------------------------ +Version 8.14.0 [v8-stable] 2015-11-03 +- Added possibility to customize librdkafka location + see also: https://github.com/rsyslog/rsyslog/pull/502 + Thanks to Matthew Wang for the patch. +- add property "rawmsg-after-pri" +- bugfix: potential misadresseing in imfile + Could happen when wildcards were used. + see also https://github.com/rsyslog/rsyslog/issues/532 + see also https://github.com/rsyslog/rsyslog/issues/534 + Thanks to zhangdaoling for the bugfix. - bugfix: re_extract RainerScript function did not work Thanks to Janmejay Singh for the patch ------------------------------------------------------------------------------ @@ -188,6 +414,11 @@ Version 8.10.0 [v8-stable] 2015-05-19 source builds per default but enforced when building from git source. - mmpstrucdata: some code cleanup removed lots of early development debug outputs +- bugfix imuxsock: fix a memory leak that happened with large messages + ... when annotation was enabled. + Thanks to github user c6226 for the patch +- bugfix omhttpfs: memory leak + Thanks to github user c6226 for the patch - bugfix imuxsock: fix a crash when setting a hostname Setting a hostname via the legacy directive would lead to a crash during shutdown caused by a double-free. diff --git a/Makefile.am b/Makefile.am index a0c84ba59..b28d7c1a7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,10 @@ if ENABLE_PMSNARE SUBDIRS += contrib/pmsnare endif +if ENABLE_PMPANNGFW +SUBDIRS += contrib/pmpanngfw +endif + if ENABLE_PMLASTMSG SUBDIRS += plugins/pmlastmsg endif @@ -203,6 +207,10 @@ if ENABLE_MMJSONPARSE SUBDIRS += plugins/mmjsonparse endif +if ENABLE_MMGROK +SUBDIRS += contrib/mmgrok +endif + if ENABLE_MMAUDIT SUBDIRS += plugins/mmaudit endif @@ -254,7 +262,8 @@ SUBDIRS += tests # when some global update happened. DISTCHECK_CONFIGURE_FLAGS= \ --disable-silent-rules \ - --disable-testbench + --disable-testbench \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) THIS_IS_TEMPORARILY_DISABLED = \ --enable-distcheck-workaround \ @@ -270,6 +279,7 @@ THIS_IS_TEMPORARILY_DISABLED = \ --enable-mail \ --enable-klog \ --enable-diagtools \ + --enable-mmgrok \ --enable-gnutls \ --enable-omstdout \ --enable-pmlastmsg \ diff --git a/README.md b/README.md index 9f422eb89..bfec61e02 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,10 @@ Ubuntu Note: this list is (obviously) incomplete. We intend to complete it as we have more information and install additional systems. -sudo apt-get install libdbi-dev +sudo apt-get install libdbi-dev libmysqlclient-dev postgresql-client libpq-dev libnet-dev librdkafka-dev libgrok-dev libgrok1 libgrok-dev libpcre3-dev libtokyocabinet-dev libglib2.0-dev + +for KSI, from the Adiscon PPA: +sudo apt-get install libksi0 libksi-devel openSUSE 13 @@ -102,7 +105,7 @@ Contributions to rsyslog are very welcome. Fork and send us your Pull Requests. For more information about contributing, see the [CONTRIBUTING](CONTRIBUTING.md) file. -Note that it is esay to add output plugins using languages like Python or +Note that it is easy to add output plugins using languages like Python or Perl. So if you need to connect to a system which is not yet supported, you can easily do so via an external plugin. For more information see the [README](plugins/external/README.md) file in the external plugin directory. @@ -118,7 +121,7 @@ Project Philosophy ================== We are an open source project in all aspects and very open to outside feedback and contribution. We base our work on standards and try to solve all real-world -needs (of course, we occasionally fail tackeling actually all needs ;)). While +needs (of course, we occasionally fail tackling actually all needs ;)). While the project is primarily sponsored by Adiscon, technical development is independent from company goals and most decisions are solely based on mailing list discussion results. There is an active community around rsyslog. diff --git a/action.c b/action.c index 5685a29b3..e40ae99cd 100644 --- a/action.c +++ b/action.c @@ -63,7 +63,7 @@ * beast. * rgerhards, 2011-06-15 * - * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -306,6 +306,7 @@ rsRetVal actionDestruct(action_t * const pThis) pThis->pMod->freeInstance(pThis->pModData); pthread_mutex_destroy(&pThis->mutAction); + pthread_mutex_destroy(&pThis->mutWrkrDataTable); d_free(pThis->pszName); d_free(pThis->ppTpl); d_free(pThis->peParamPassing); @@ -360,6 +361,7 @@ rsRetVal actionConstruct(action_t **ppThis) pThis->tLastOccur = datetime.GetTime(NULL); /* done once per action on startup only */ pThis->iActionNbr = iActionNbr; pthread_mutex_init(&pThis->mutAction, NULL); + pthread_mutex_init(&pThis->mutWrkrDataTable, NULL); INIT_ATOMIC_HELPER_MUT(pThis->mutCAS); /* indicate we have a new action */ @@ -385,7 +387,7 @@ actionConstructFinalize(action_t *__restrict__ const pThis, struct nvlst *lst) } /* generate a friendly name for us action stats */ if(pThis->pszName == NULL) { - snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "action %d", pThis->iActionNbr); + snprintf((char*) pszAName, sizeof(pszAName), "action %d", pThis->iActionNbr); pThis->pszName = ustrdup(pszAName); } @@ -438,7 +440,7 @@ actionConstructFinalize(action_t *__restrict__ const pThis, struct nvlst *lst) /* create our queue */ /* generate a friendly name for the queue */ - snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "%s queue", + snprintf((char*) pszAName, sizeof(pszAName), "%s queue", pThis->pszName); /* now check if we can run the action in "firehose mode" during stage one of @@ -778,6 +780,7 @@ actionCheckAndCreateWrkrInstance(action_t * const pThis, wti_t * const pWti) /* maintain worker data table -- only needed if wrkrHUP is requested! */ + pthread_mutex_lock(&pThis->mutWrkrDataTable); int freeSpot; for(freeSpot = 0 ; freeSpot < pThis->wrkrDataTableSize ; ++freeSpot) if(pThis->wrkrDataTable[freeSpot] == NULL) @@ -788,9 +791,9 @@ actionCheckAndCreateWrkrInstance(action_t * const pThis, wti_t * const pWti) (pThis->wrkrDataTableSize + 1) * sizeof(void*)); pThis->wrkrDataTableSize++; } -dbgprintf("DDDD: writing data to table spot %d\n", freeSpot); pThis->wrkrDataTable[freeSpot] = pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData; pThis->nWrkr++; + pthread_mutex_unlock(&pThis->mutWrkrDataTable); DBGPRINTF("wti %p: created action worker instance %d for " "action %d\n", pWti, pThis->nWrkr, pThis->iActionNbr); } @@ -978,15 +981,20 @@ releaseDoActionParams(action_t *__restrict__ const pAction, wti_t *__restrict__ switch(pAction->peParamPassing[j]) { case ACT_ARRAY_PASSING: ppMsgs = (uchar***) pWrkrInfo->p.nontx.actParams[0].param; - if(((uchar**)ppMsgs)[j] != NULL) { - jArr = 0; - while(ppMsgs[j][jArr] != NULL) { - free(ppMsgs[j][jArr]); - ppMsgs[j][jArr] = NULL; - ++jArr; + /* if we every use array passing mode again, we need to check + * this code. It hasn't been used since refactoring for v7. + */ + if(ppMsgs != NULL) { + if(((uchar**)ppMsgs)[j] != NULL) { + jArr = 0; + while(ppMsgs[j][jArr] != NULL) { + free(ppMsgs[j][jArr]); + ppMsgs[j][jArr] = NULL; + ++jArr; + } + free(((uchar**)ppMsgs)[j]); + ((uchar**)ppMsgs)[j] = NULL; } - free(((uchar**)ppMsgs)[j]); - ((uchar**)ppMsgs)[j] = NULL; } break; case ACT_JSON_PASSING: @@ -1291,7 +1299,7 @@ processMsgMain(action_t *__restrict__ const pAction, { DEFiRet; - iRet = prepareDoActionParams(pAction, pWti, pMsg, ttNow); + CHKiRet(prepareDoActionParams(pAction, pWti, pMsg, ttNow)); if(pAction->isTransactional) { pWti->actWrkrInfo[pAction->iActionNbr].pAction = pAction; @@ -1332,7 +1340,10 @@ processBatchMain(void *__restrict__ const pVoid, for(i = 0 ; i < batchNumMsgs(pBatch) && !*pWti->pbShutdownImmediate ; ++i) { if(batchIsValidElem(pBatch, i)) { - iRet = processMsgMain(pAction, pWti, pBatch->pElem[i].pMsg, &ttNow); + /* we do not check error state below, because aborting would be + * more harmful than continuing. + */ + processMsgMain(pAction, pWti, pBatch->pElem[i].pMsg, &ttNow); batchSetElemState(pBatch, i, BATCH_STATE_COMM); } } @@ -1349,6 +1360,7 @@ void actionRemoveWorker(action_t *const __restrict__ pAction, void *const __restrict__ actWrkrData) { + pthread_mutex_lock(&pAction->mutWrkrDataTable); pAction->nWrkr--; for(int w = 0 ; w < pAction->wrkrDataTableSize ; ++w) { if(pAction->wrkrDataTable[w] == actWrkrData) { @@ -1356,6 +1368,7 @@ actionRemoveWorker(action_t *const __restrict__ pAction, break; /* done */ } } + pthread_mutex_unlock(&pAction->mutWrkrDataTable); } @@ -1377,14 +1390,21 @@ actionCallHUPHdlr(action_t * const pAction) } if(pAction->pMod->doHUPWrkr != NULL) { + pthread_mutex_lock(&pAction->mutWrkrDataTable); for(int i = 0 ; i < pAction->wrkrDataTableSize ; ++i) { dbgprintf("HUP: table entry %d: %p %s\n", i, pAction->wrkrDataTable[i], pAction->wrkrDataTable[i] == NULL ? "[unused]" : ""); if(pAction->wrkrDataTable[i] != NULL) { - CHKiRet(pAction->pMod->doHUPWrkr(pAction->wrkrDataTable[i])); + const rsRetVal localRet + = pAction->pMod->doHUPWrkr(pAction->wrkrDataTable[i]); + if(localRet != RS_RET_OK) { + DBGPRINTF("HUP handler returned error state %d - " + "ignored\n", localRet); + } } } + pthread_mutex_unlock(&pAction->mutWrkrDataTable); } finalize_it: @@ -1770,7 +1790,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, if(!(iTplOpts & OMSR_TPL_AS_MSG)) { if((pAction->ppTpl[i] = tplFind(ourConf, (char*)pTplName, strlen((char*)pTplName))) == NULL) { - snprintf(errMsg, sizeof(errMsg) / sizeof(char), + snprintf(errMsg, sizeof(errMsg), " Could not find template %d '%s' - action disabled", i, pTplName); errno = 0; diff --git a/action.h b/action.h index 0031dff9c..a38fdcf73 100644 --- a/action.h +++ b/action.h @@ -74,6 +74,7 @@ struct action_s { uchar *pszName; /* action name */ DEF_ATOMIC_HELPER_MUT(mutCAS) /* for per-worker HUP processing */ + pthread_mutex_t mutWrkrDataTable; /* protects table structures */ void **wrkrDataTable; int wrkrDataTableSize; int nWrkr; diff --git a/configure.ac b/configure.ac index aa8a968c0..b3ed3ca32 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.14.0.master],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[8.17.0.master],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE([subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -26,9 +26,7 @@ PKG_PROG_PKG_CONFIG # modules we require PKG_CHECK_MODULES(LIBESTR, libestr >= 0.1.9) -PKG_CHECK_MODULES([JSON_C], [json],, [ - PKG_CHECK_MODULES([JSON_C], [json-c],,) -]) +PKG_CHECK_MODULES([JSON_C], [libfastjson],,) save_CFLAGS="$CFLAGS" save_LIBS="$LIBS" @@ -294,6 +292,14 @@ AC_CHECK_HEADERS( [AC_MSG_FAILURE([pthread is missing])] ) +AC_CHECK_LIB( + [pthread], + [pthread_rwlockattr_setkind_np], + [AC_DEFINE( + [HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP], + [1], + [Set-kind available for rwlock attr.])]) + AC_CHECK_FUNCS( [pthread_setschedparam], [ @@ -955,6 +961,24 @@ AC_ARG_ENABLE(mmjsonparse, ) AM_CONDITIONAL(ENABLE_MMJSONPARSE, test x$enable_mmjsonparse = xyes) +# mmgrok +AC_ARG_ENABLE(mmgrok, + [AS_HELP_STRING([--enable-mmgrok],[Enable building mmgrok support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_mmgrok="yes" ;; + no) enable_mmgrok="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-mmgrok) ;; + esac], + [enable_mmgrok=no] +) +if test "x$enable_mmgrok" = "xyes"; then + GLIB_CFLAGS="$(pkg-config --cflags glib-2.0)" + GLIB_LIBS="$(pkg-config --libs glib-2.0)" +fi +AM_CONDITIONAL(ENABLE_MMGROK, test x$enable_mmgrok = xyes) +AC_SUBST(GLIB_CFLAGS) +AC_SUBST(GLIB_LIBS) + # mmaudit AC_ARG_ENABLE(mmaudit, [AS_HELP_STRING([--enable-mmaudit],[Enable building mmaudit support @<:@default=no@:>@])], @@ -1107,8 +1131,8 @@ AC_ARG_ENABLE(gt-ksi, [enable_gt_ksi=no] ) if test "x$enable_gt_ksi" = "xyes"; then - PKG_CHECK_MODULES(GT_KSI, libksi >= 3.2.2.0) - PKG_CHECK_MODULES(GT_KSI, libksi < 3.3.0.0) + PKG_CHECK_MODULES(GT_KSI, libksi >= 3.4.0.2) + PKG_CHECK_MODULES(GT_KSI, libksi < 3.5.0.0) fi AM_CONDITIONAL(ENABLE_GT_KSI, test x$enable_gt_ksi = xyes) @@ -1378,6 +1402,19 @@ AC_ARG_ENABLE(pmsnare, AM_CONDITIONAL(ENABLE_PMSNARE, test x$enable_pmsnare = xyes) +# settings for pmpanngfw +AC_ARG_ENABLE(pmpanngfw, + [AS_HELP_STRING([--enable-pmpanngfw],[Compiles Palo Alto Networks parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmpanngfw="yes" ;; + no) enable_pmpanngfw="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmpanngfw) ;; + esac], + [enable_pmpanngfw=no] +) +AM_CONDITIONAL(ENABLE_PMPANNGFW, test x$enable_pmpanngfw = xyes) + + # settings for omruleset AC_ARG_ENABLE(omruleset, [AS_HELP_STRING([--enable-omruleset],[Compiles ruleset forwarding module @<:@default=no@:>@])], @@ -1443,8 +1480,15 @@ AC_ARG_ENABLE(omkafka, esac], [enable_omkafka=no] ) -if test "x$enable_omkafka"; then - #PKG_CHECK_MODULES(LIBRDKAFKA, librdkafka) +if test "x$enable_omkafka" = "xyes"; then + PKG_CHECK_MODULES([LIBRDKAFKA], [librdkafka],, [ + AC_CHECK_LIB([rdkafka], [rd_kafka_produce], [ + AC_MSG_WARN([librdkafka is missing but library present, using -lrdkafka]) + LIBRDKAFKA_LIBS=-lrdkafka + ], [ + AC_MSG_ERROR([could not find rdkafka library]) + ]) + ]) AC_CHECK_HEADERS([librdkafka/rdkafka.h]) fi AM_CONDITIONAL(ENABLE_OMKAFKA, test x$enable_omkafka = xyes) @@ -1708,10 +1752,12 @@ AC_CONFIG_FILES([Makefile \ plugins/mmsnmptrapd/Makefile \ plugins/pmlastmsg/Makefile \ contrib/pmsnare/Makefile \ + contrib/pmpanngfw/Makefile \ contrib/pmaixforwardedfrom/Makefile \ contrib/omhiredis/Makefile \ contrib/omrabbitmq/Makefile \ contrib/imkmsg/Makefile \ + contrib/mmgrok/Makefile \ contrib/mmcount/Makefile \ contrib/omzmq3/Makefile \ contrib/omczmq/Makefile \ @@ -1777,10 +1823,12 @@ echo " pmcisconames module will be compiled: $enable_pmcisconames" echo " pmciscoios module will be compiled: $enable_pmciscoios" echo " pmaixforwardedfrom module w.be compiled: $enable_pmaixforwardedfrom" echo " pmsnare module will be compiled: $enable_pmsnare" +echo " pmpanngfw module will be compiled: $enable_pmpanngfw" echo echo "---{ message modification modules }---" echo " mmnormalize module will be compiled: $enable_mmnormalize" echo " mmjsonparse module will be compiled: $enable_mmjsonparse" +echo " mmgrok module will be compiled: $enable_mmgrok" echo " mmjaduit module will be compiled: $enable_mmaudit" echo " mmsnmptrapd module will be compiled: $enable_mmsnmptrapd" echo " mmutf8fix enabled: $enable_mmutf8fix" diff --git a/contrib/imczmq/imczmq.c b/contrib/imczmq/imczmq.c index 0280365b7..bf966f0d9 100644 --- a/contrib/imczmq/imczmq.c +++ b/contrib/imczmq/imczmq.c @@ -48,26 +48,29 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) - + +/* Arguments to pass to poll handler */ +/* --------------------------------- */ typedef struct _pollerData_t { ruleset_t *ruleset; - thrdInfo_t *thread; + thrdInfo_t *thread; + int sockType; } pollerData_t; struct instanceConf_s { - bool is_server; - char *beacon; - int beaconPort; - int sockType; - char *sockEndpoints; - char *topicList; - char *authType; - char *clientCertPath; - char *serverCertPath; - uchar *pszBindRuleset; - ruleset_t *pBindRuleset; - struct instanceConf_s *next; + bool serverish; /* whether socket will be a server or client */ + char *beacon; /* (optional) content of beacon */ + int beaconPort; /* (optional) port if beacon is set */ + int sockType; /* (required) type of zeromq socket */ + char *sockEndpoints; /* (required) comma delimited list of endpoints */ + char *topicList; /* (optional) topics to subscribe to if SUB */ + char *authType; /* (optional) type of authentication */ + char *clientCertPath; /* (optional) path to curve client cert(s) */ + char *serverCertPath; /* (optional) path to curve server cert */ + uchar *pszBindRuleset; /* (optional) ruleset to bind to */ + ruleset_t *pBindRuleset; /* holds the ruleset reference */ + struct instanceConf_s *next; /* pointer to next conf */ }; struct modConfData_s { @@ -78,12 +81,12 @@ struct modConfData_s { }; struct lstn_s { - zactor_t *beaconActor; - zsock_t *sock; - zcert_t *clientCert; - zcert_t *serverCert; - ruleset_t *pRuleset; - struct lstn_s *next; + zactor_t *beaconActor; /* CZMQ beacon actor */ + zsock_t *sock; /* zeromq socket */ + zcert_t *clientCert; /* client curve certificate */ + zcert_t *serverCert; /* server curve certificate */ + ruleset_t *pRuleset; /* ruleset */ + struct lstn_s *next; /* pointer to next input */ }; static modConfData_t *runModConf = NULL; @@ -103,6 +106,34 @@ static struct cnfparamblk modpblk = { modpdescr }; +/* ------------------------------------------------------------- + * Explanation of Options + * ---------------------- + * endpoints + * A comma delimited list of endpoints to connect to. Whether the + * endpoint connects or binds is determined by defaults for + * each socket type. + * socktype + * The type of zeromq socket to use; supports SUB and PULL. + * beacon + * The beacon is lit! If beacon is set to a value, that value + * will be used to advertise the existence of this endpoint + * over udp multicast beacons. + * beaconport + * The port to use for the beacon + * authtype + * The authentication type to use - currently only supports + * CURVESERVER and CURVECLIENT. + * clientcertpath + * The path to client certificate(s). Must be set if authtype is + * set to CURVE. + * servercertpath + * The path to server certificate. Must be set if authtype is + * set to CURVE + * topics + * A comma delimited list of topics to subscribe to. + * ------------------------------------------------------------- */ + static struct cnfparamdescr inppdescr[] = { { "beacon", eCmdHdlrGetWord, 0 }, { "beaconport", eCmdHdlrGetWord, 0 }, @@ -124,7 +155,7 @@ static struct cnfparamblk inppblk = { }; static void setDefaults(instanceConf_t* iconf) { - iconf->is_server = false; + iconf->serverish = true; iconf->beacon = NULL; iconf->beaconPort = -1; iconf->sockType = -1; @@ -169,46 +200,43 @@ static rsRetVal createListener(struct cnfparamvals* pvals) { continue; } - /* get the ruleset to bind to */ if(!strcmp(inppblk.descr[i].name, "ruleset")) { inst->pszBindRuleset = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get the socket endpoints to use */ else if(!strcmp(inppblk.descr[i].name, "endpoints")) { inst->sockEndpoints = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get beacon argument */ else if (!strcmp(inppblk.descr[i].name, "beacon")) { inst->beacon = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get beacon port */ else if (!strcmp(inppblk.descr[i].name, "beaconport")) { inst->beaconPort = atoi(es_str2cstr(pvals[i].val.d.estr, NULL)); } - - /* get the socket type */ else if(!strcmp(inppblk.descr[i].name, "socktype")){ char *stringType = es_str2cstr(pvals[i].val.d.estr, NULL); + if ( NULL == stringType ){ + errmsg.LogError(0, RS_RET_OUT_OF_MEMORY, + "imczmq: '%s' is invalid sockType", stringType); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } if (!strcmp("SUB", stringType)) { inst->sockType = ZMQ_SUB; } - else if (!strcmp("PULL", stringType)) { inst->sockType = ZMQ_PULL; } - - else { - errmsg.LogError(0, RS_RET_CONFIG_ERROR, - "imczmq: invalid sockType"); - ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + else if (!strcmp("ROUTER", stringType)) { + inst->sockType = ZMQ_ROUTER; } + else { + errmsg.LogError(0, RS_RET_CONFIG_ERROR, + "imczmq: '%s' is invalid sockType", stringType); + free(stringType); + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + } + free(stringType); } - - /* get the subscription topics */ else if(!strcmp(inppblk.descr[i].name, "topics")) { if (inst->sockType != ZMQ_SUB) { errmsg.LogError(0, RS_RET_CONFIG_ERROR, @@ -218,8 +246,6 @@ static rsRetVal createListener(struct cnfparamvals* pvals) { inst->topicList = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get the authentication type to use */ else if(!strcmp(inppblk.descr[i].name, "authtype")) { inst->authType = es_str2cstr(pvals[i].val.d.estr, NULL); @@ -234,18 +260,12 @@ static rsRetVal createListener(struct cnfparamvals* pvals) { ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } } - - /* get client cert argument */ else if(!strcmp(inppblk.descr[i].name, "clientcertpath")) { inst->clientCertPath = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get server cert argument */ else if(!strcmp(inppblk.descr[i].name, "servercertpath")) { inst->serverCertPath = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* the config has a bad option */ else { errmsg.LogError(0, NO_ERRCODE, "imczmq: program error, non-handled " @@ -265,20 +285,25 @@ static rsRetVal addListener(instanceConf_t* iconf){ pData->next = NULL; pData->pRuleset = iconf->pBindRuleset; - /* create our zeromq socket */ + /* Create the zeromq socket */ + /* ------------------------ */ + pData->sock = zsock_new(iconf->sockType); - if (!pData->sock) { + if (!pData->sock) { errmsg.LogError(0, RS_RET_NO_ERRCODE, "imczmq: new socket failed for endpoints: %s", iconf->sockEndpoints); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } - /* if a beacon is set start it */ + DBGPRINTF ("imczmq: created socket...\n"); + + /* Create the beacon actor if configured */ + /* ------------------------------------- */ + if((iconf->beacon != NULL) && (iconf->beaconPort > 0)) { DBGPRINTF ("imczmq: starting beacon actor...\n"); - /* create the beacon actor, if it fails abort */ pData->beaconActor = zactor_new(zbeacon, NULL); if (!pData->beaconActor) { errmsg.LogError(0, RS_RET_NO_ERRCODE, @@ -286,7 +311,6 @@ static rsRetVal addListener(instanceConf_t* iconf){ ABORT_FINALIZE (RS_RET_NO_ERRCODE); } - /* configure the beacon and abort if UDP broadcasting ont available */ zsock_send(pData->beaconActor, "si", "CONFIGURE", iconf->beaconPort); char *hostname = zstr_recv(pData->beaconActor); if (!*hostname) { @@ -295,73 +319,84 @@ static rsRetVal addListener(instanceConf_t* iconf){ ABORT_FINALIZE (RS_RET_NO_ERRCODE); } - /* start publishing the beacon */ - zsock_send(pData->beaconActor, "sbi", "PUBLISH", pData->beaconActor, strlen(iconf->beacon)); + zsock_send(pData->beaconActor, + "sbi", "PUBLISH", pData->beaconActor, strlen(iconf->beacon)); + + DBGPRINTF ("omczmq: beacon is lit: hostname: '%s', port: '%d'...\n", + hostname, iconf->beaconPort); } + + DBGPRINTF("imczmq: authtype is: %s\n", iconf->authType); - /* if we are a CURVE server */ - if ((iconf->authType != NULL) && (!strcmp(iconf->authType, "CURVESERVER"))) { + if (iconf->authType != NULL) { - iconf->is_server = true; + /* CURVESERVER */ + /* ----------- */ - /* set global auth domain */ - zsock_set_zap_domain(pData->sock, "global"); + if (!strcmp(iconf->authType, "CURVESERVER")) { - /* set that we are a curve server */ - zsock_set_curve_server(pData->sock, 1); + zsock_set_zap_domain(pData->sock, "global"); + zsock_set_curve_server(pData->sock, 1); - /* get and set our server cert */ - DBGPRINTF("imczmq: server cert is %s...\n", iconf->serverCertPath); - pData->serverCert = zcert_load(iconf->serverCertPath); - if (!pData->serverCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); - ABORT_FINALIZE(RS_RET_ERR); + pData->serverCert = zcert_load(iconf->serverCertPath); + + if (!pData->serverCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); + ABORT_FINALIZE(RS_RET_ERR); + } + + zcert_apply(pData->serverCert, pData->sock); + + zstr_sendx(authActor, "CURVE", iconf->clientCertPath, NULL); + zsock_wait(authActor); + + DBGPRINTF("imczmq: CURVESERVER: serverCertPath: '%s'\n", iconf->serverCertPath); + DBGPRINTF("mczmq: CURVESERVER: clientCertPath: '%s'\n", iconf->clientCertPath); } - zcert_apply(pData->serverCert, pData->sock); + /* CURVECLIENT */ + /* ----------- */ - /* set allowed clients */ - DBGPRINTF("imczmq: allowed clients are %s...\n", iconf->clientCertPath); - zstr_sendx(authActor, "CURVE", iconf->clientCertPath, NULL); - zsock_wait(authActor); + else if (!strcmp(iconf->authType, "CURVECLIENT")) { + if (!strcmp(iconf->clientCertPath, "*")) { + pData->clientCert = zcert_new(); + } + else { + pData->clientCert = zcert_load(iconf->clientCertPath); + } + + if (!pData->clientCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load client cert"); + ABORT_FINALIZE(RS_RET_ERR); + } + + zcert_apply(pData->clientCert, pData->sock); + pData->serverCert = zcert_load(iconf->serverCertPath); + + if (!pData->serverCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); + ABORT_FINALIZE(RS_RET_ERR); + } + + char *server_key = zcert_public_txt(pData->serverCert); + zsock_set_curve_serverkey (pData->sock, server_key); + + DBGPRINTF("imczmq: CURVECLIENT: serverCertPath: '%s'\n", iconf->serverCertPath); + DBGPRINTF("imczmq: CURVECLIENT: clientCertPath: '%s'\n", iconf->clientCertPath); + DBGPRINTF("imczmq: CURVECLIENT: server_key: '%s'\n", server_key); + } + else { + errmsg.LogError(0, NO_ERRCODE, "unrecognized auth type: '%s'", iconf->authType); + ABORT_FINALIZE(RS_RET_ERR); + } } - /* if we are a CURVE client */ - if ((iconf->authType != NULL) && (!strcmp(iconf->authType, "CURVECLIENT"))) { - DBGPRINTF("imczmq: we are a curve client...\n"); + /* subscribe to topics */ + /* ------------------- */ - iconf->is_server = false; - - /* get our client cert */ - pData->clientCert = zcert_load(iconf->clientCertPath); - if (!pData->clientCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load client cert"); - ABORT_FINALIZE(RS_RET_ERR); - } - - /* apply the client cert to the socket */ - zcert_apply(pData->clientCert, pData->sock); - - /* get the server cert */ - DBGPRINTF("omczmq: server cert is %s...\n", iconf->serverCertPath); - pData->serverCert = zcert_load(iconf->serverCertPath); - if (!pData->serverCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); - ABORT_FINALIZE(RS_RET_ERR); - } - - /* get the server public key and set it for the socket */ - char *server_key = zcert_public_txt(pData->serverCert); - DBGPRINTF("omczmq: server public key is %s...\n", server_key); - zsock_set_curve_serverkey (pData->sock, server_key); - } - - /* if we have a ZMQ_SUB sock, subscribe to topics */ if (iconf->sockType == ZMQ_SUB) { - iconf->is_server = false; - char topic[256], *list = iconf->topicList; while (list) { char *delimiter = strchr(list, ','); @@ -388,8 +423,19 @@ static rsRetVal addListener(instanceConf_t* iconf){ } } + switch (iconf->sockType) { + case ZMQ_SUB: + iconf->serverish = false; + break; + case ZMQ_PULL: + case ZMQ_ROUTER: + iconf->serverish = true; + break; + } + + int rc = zsock_attach(pData->sock, (const char*)iconf->sockEndpoints, - iconf->is_server); + iconf->serverish); if (rc == -1) { errmsg.LogError(0, NO_ERRCODE, "zsock_attach to %s", iconf->sockEndpoints); @@ -397,6 +443,8 @@ static rsRetVal addListener(instanceConf_t* iconf){ } /* add this struct to the global */ + /* ----------------------------- */ + if(lcnfRoot == NULL) { lcnfRoot = pData; } @@ -415,9 +463,11 @@ static int handlePoll(zloop_t __attribute__((unused)) *loop, zmq_pollitem_t *pol msg_t* pMsg; pollerData_t *pollerData = (pollerData_t *)args; - zmsg_t *msg = zmsg_recv(poller->socket); - zframe_t *payload = zmsg_last(msg); - char* buf = zframe_strdup(payload); + if (pollerData->sockType == ZMQ_ROUTER ) { + zframe_t *idFrame = zframe_recv (poller->socket); + zframe_destroy (&idFrame); + } + char *buf = zstr_recv(poller->socket); if (msgConstruct(&pMsg) == RS_RET_OK) { MsgSetRawMsg(pMsg, buf, strlen(buf)); @@ -432,8 +482,7 @@ static int handlePoll(zloop_t __attribute__((unused)) *loop, zmq_pollitem_t *pol submitMsg2(pMsg); } - free(buf); - zmsg_destroy(&msg); + zstr_free(&buf); if( pollerData->thread->bShallStop == TRUE) { return -1; @@ -445,14 +494,22 @@ static int handlePoll(zloop_t __attribute__((unused)) *loop, zmq_pollitem_t *pol static rsRetVal rcvData(thrdInfo_t* pThrd){ DEFiRet; - DBGPRINTF("imczmq: starting auth actor...\n"); + zmq_pollitem_t *items = NULL; + pollerData_t *pollerData = NULL; + + /* Create the authentication actor */ + /* ------------------------------ */ + authActor = zactor_new(zauth, NULL); + if (!authActor) { errmsg.LogError(0, RS_RET_NO_ERRCODE, "imczmq: could not create auth service"); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } + DBGPRINTF ("imczmq: auth actor started\n"); + instanceConf_t *inst; for (inst = runModConf->root; inst != NULL; inst=inst->next) { addListener(inst); @@ -470,12 +527,13 @@ static rsRetVal rcvData(thrdInfo_t* pThrd){ n_items++; } - zmq_pollitem_t *items = NULL; - CHKmalloc(items = (zmq_pollitem_t*)MALLOC(sizeof(zmq_pollitem_t)*n_items)); + /* create poller and set up zloop handler */ + /* TODO: replace zloop with zpoller */ + /* ------------------------------------- */ - pollerData_t *pollerData = NULL; CHKmalloc(pollerData = (pollerData_t *)MALLOC(sizeof(pollerData_t)*n_items)); - + CHKmalloc(items = (zmq_pollitem_t*)MALLOC(sizeof(zmq_pollitem_t)*n_items)); + size_t i; for(i=0, current = lcnfRoot; current != NULL; current = current->next, i++) { items[i].socket=zsock_resolve(current->sock); @@ -483,12 +541,12 @@ static rsRetVal rcvData(thrdInfo_t* pThrd){ pollerData[i].thread = pThrd; pollerData[i].ruleset = current->pRuleset; + pollerData[i].sockType = zsock_type(current->sock); } - zloopHandle = zloop_new(); int rc; + zloopHandle = zloop_new(); for(i=0; ipszBindRuleset, - inst->sockEndpoints); + errmsg.LogError(0, NO_ERRCODE, + "imczmq: ruleset '%s' for socket %s not found - " + "using default ruleset instead", inst->pszBindRuleset, + inst->sockEndpoints); } @@ -645,6 +714,7 @@ CODESTARTfreeCnf lstn_r = lstn; zcert_destroy(&lstn->clientCert); zcert_destroy(&lstn->serverCert); + zsock_destroy(&lstn->sock); lstn = lstn_r->next; free(lstn_r); } diff --git a/contrib/imkmsg/imkmsg.c b/contrib/imkmsg/imkmsg.c index 956aabee3..f44ae8819 100644 --- a/contrib/imkmsg/imkmsg.c +++ b/contrib/imkmsg/imkmsg.c @@ -99,7 +99,7 @@ enqMsg(uchar *msg, uchar* pszTag, syslog_pri_t pri, struct timeval *tp, struct j if(tp == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { - datetime.timeval2syslogTime(tp, &st); + datetime.timeval2syslogTime(tp, &st, TIME_IN_LOCALTIME); CHKiRet(msgConstructWithTime(&pMsg, &st, tp->tv_sec)); } MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -129,7 +129,7 @@ rsRetVal imkmsgLogIntMsg(syslog_pri_t priority, char *fmt, ...) uchar msgBuf[2048]; /* we use the same size as sysklogd to remain compatible */ va_start(ap, fmt); - vsnprintf((char*)msgBuf, sizeof(msgBuf) / sizeof(char), fmt, ap); + vsnprintf((char*)msgBuf, sizeof(msgBuf), fmt, ap); va_end(ap); logmsgInternal(NO_ERRCODE, priority, msgBuf, 0); diff --git a/contrib/imzmq3/imzmq3.c b/contrib/imzmq3/imzmq3.c index 08b1dbe4c..dd813d522 100644 --- a/contrib/imzmq3/imzmq3.c +++ b/contrib/imzmq3/imzmq3.c @@ -305,6 +305,7 @@ static rsRetVal parseSubscriptions(char* subscribes, sublist** subList){ DBGPRINTF("'%s'", currentSub->subscribe); } DBGPRINTF("\n"); + finalize_it: RETiRet; } @@ -474,8 +475,10 @@ static rsRetVal createListener(struct cnfparamvals* pvals) { } else if(!strcmp(inppblk.descr[i].name, "rcvHWM")) { inst->rcvHWM = (int) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "subscribe")) { - CHKiRet(parseSubscriptions(es_str2cstr(pvals[i].val.d.estr, NULL), - &inst->subscriptions)); + char *subscribes = es_str2cstr(pvals[i].val.d.estr, NULL); + rsRetVal ret = parseSubscriptions(subscribes, &inst->subscriptions); + free(subscribes); + CHKiRet(ret); } else if(!strcmp(inppblk.descr[i].name, "identity")){ inst->identity = es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(inppblk.descr[i].name, "sndBuf")) { diff --git a/contrib/mmgrok/Makefile.am b/contrib/mmgrok/Makefile.am new file mode 100644 index 000000000..af613ac11 --- /dev/null +++ b/contrib/mmgrok/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = mmgrok.la + +mmgrok_la_SOURCES = mmgrok.c +mmgrok_la_CPPFLAGS = $(GLIB_CFLAGS) $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +mmgrok_la_LDFLAGS = -module -avoid-version +mmgrok_la_LIBADD = $(GLIB_LIBS) -lgrok $(JSON_C_LIBS) + +EXTRA_DIST = diff --git a/contrib/mmgrok/README b/contrib/mmgrok/README new file mode 100644 index 000000000..10535453e --- /dev/null +++ b/contrib/mmgrok/README @@ -0,0 +1,32 @@ +Grok Message Modify Plugin + +Using hundreds of grok patterns from logstash-patterns-core. + +Build + +This plugin requires json-c, glib2, and grok packages. + +If you use RH/CentOS/Fedora, you'll have to build grok rpms by yourself as follow: + + sudo yum install -y yum-utils rpmdevtools + git clone git@github.com:jordansissel/grok.git + mkdir -p ~/rpmbuild/SPECS/; cp grok/grok.spec.template ~/rpmbuild/SPECS/grok.spec + (mkdir -p ~/rpmbuld/SOURCES/; cd ~/rpmbuild/SOURCES/; spectool -g ../SPECS/grok.spec) + sudo yum-builddep ~/rpmbuild/SPECS/grok.spec + rpmbuild -bb ~/rpmbuild/SPECS/grok.spec + # use yum command instead of rpm, because grok depends on libevent, pcre, tokyocabinet + sudo yum install -y libjson-c-devel glib2-devel ~/rpbuild/RPMS/x86_64/grok*.rpm + +Example + +module(load="mmgrok") +template(name="tmlp" type="string" string="%$!msg!test%\n") +action(type="mmgrok" patterndir="path/to/yourpatternsDir" match="%{WORD:test}" soure="msg" target="!msg") +action(type="omfile" file="path/to/file" template="tmlp") + +Descrption + +patterndir: path to grok patterns dir, default: /usr/share/grok/patterns/base +match：the pattern used to match message +source: the source message/variable to be matched +target: the root path to write the captured json tree diff --git a/contrib/mmgrok/mmgrok.c b/contrib/mmgrok/mmgrok.c new file mode 100644 index 000000000..ae376ef84 --- /dev/null +++ b/contrib/mmgrok/mmgrok.c @@ -0,0 +1,423 @@ +/* mmgrok.c + * Grok the message is parsed into a structured json data inside JSON. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" +#include "dirty.h" + +MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP +MODULE_CNFNAME("mmgrok"); + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); + +DEFobjCurrIf(errmsg); +DEF_OMOD_STATIC_DATA + +typedef struct result_s{ + char *key; + int key_len; + const char *value; + int value_len; + char *type; +}result_t; + +/* config variables */ +typedef struct _instanceData +{ + char *pszPatternDir; + char *pszMatch; + char *pszSource; + char *pszTarget;/* as a json root for store parse json data */ + msg_t *pmsg; /* store origin messages*/ +}instanceData; + +typedef struct wrkrInstanceData{ + instanceData *pData; +}wrkrInstanceData_t; + +struct modConfData_s{ + rsconf_t *pConf;/* our overall config object */ +}; + +static modConfData_t *loadModConf = NULL; +static modConfData_t *runModConf = NULL; + +/* action (instance) paramters */ +static struct cnfparamdescr actpdescr[]={ + {"patterndir",eCmdHdlrString,0}, + {"match",eCmdHdlrString,0}, + {"source",eCmdHdlrString,0}, + {"target",eCmdHdlrString,0}, +}; + +static struct cnfparamblk actpblk = +{ + CNFPARAMBLK_VERSION, + sizeof(actpdescr)/sizeof(struct cnfparamdescr), + actpdescr +}; + +BEGINbeginCnfLoad +CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; +ENDbeginCnfLoad + +BEGINendCnfLoad +CODESTARTendCnfLoad +ENDendCnfLoad + +BEGINcheckCnf +CODESTARTcheckCnf +ENDcheckCnf + +BEGINactivateCnf +CODESTARTactivateCnf + runModConf = pModConf; +ENDactivateCnf + +BEGINfreeCnf +CODESTARTfreeCnf +ENDfreeCnf + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +ENDisCompatibleWithFeature + +BEGINfreeInstance +CODESTARTfreeInstance + free(pData->pszPatternDir); + free(pData->pszMatch); + free(pData->pszSource); + free(pData->pszTarget); +ENDfreeInstance + +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + +static inline void setInstParamDefaults(instanceData *pData) +{ + pData->pszPatternDir= NULL; + pData->pszMatch = NULL; + pData->pszSource = NULL; + pData->pszTarget = NULL; + pData->pmsg = NULL; +} + + +BEGINnewActInst + struct cnfparamvals *pvals; + int i; +CODESTARTnewActInst + DBGPRINTF("newActInst (mmgrok)\n"); + if((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + CODE_STD_STRING_REQUESTnewActInst(1) + CHKiRet(OMSRsetEntry(*ppOMSR, 0, NULL, OMSR_TPL_AS_MSG)); + CHKiRet(createInstance(&pData)); + setInstParamDefaults(pData); + + for(i = 0 ; i < actpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(actpblk.descr[i].name, "patterndir")) { + pData->pszPatternDir= es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + else if(!strcmp(actpblk.descr[i].name, "match")) { + pData->pszMatch = es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + else if(!strcmp(actpblk.descr[i].name, "source")) { + pData->pszSource= es_str2cstr(pvals[i].val.d.estr, NULL); + continue; + } + else if(!strcmp(actpblk.descr[i].name,"target")) + { + pData->pszTarget=es_str2cstr(pvals[i].val.d.estr,NULL); + continue; + } + else{ + DBGPRINTF("mmgrok: program error, non-handled " + "param '%s'\n", actpblk.descr[i].name); + } + } + if(pData->pszTarget == NULL) { + CHKmalloc(pData->pszTarget = strdup("!")); + } +CODE_STD_FINALIZERnewActInst + cnfparamvalsDestruct(pvals, &actpblk); +ENDnewActInst + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + DBGPRINTF("mmgrok\n"); +ENDdbgPrintInstInfo + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +static inline grok_t *CreateGrok() +{ + grok_t *grok = grok_new(); + if(grok == NULL){ + DBGPRINTF("mmgrok: create a grok faile!"); + exit(1); + } + grok_init(grok); + return grok; +} + +/* the parseing is complate message into json */ +static rsRetVal +msg_to_json(GList *list,instanceData *pData) +{ + GList *it= list; + + struct json_object *json; + struct json_object *jval; + + DEFiRet; + + json = json_object_new_object(); + if(json == NULL) + { + ABORT_FINALIZE(RS_RET_ERR); + } + for(;it;it= it->next) + { + int key_len = ((result_t *)it->data)->key_len; + char *key = (char *)malloc(key_len+1); + snprintf(key,key_len+1,"%.*s",key_len,((result_t *)it->data)->key); + int value_len = ((result_t *)it->data)->value_len; + char *value = (char *)malloc(value_len+1); + snprintf(value,value_len+1,"%.*s",value_len,((result_t*)it->data)->value); + jval = json_object_new_string(value); + json_object_object_add(json,key,jval); + free(key); + free(value); + } + msgAddJSON(pData->pmsg,(uchar*)pData->pszTarget,json,0,0); +finalize_it: + RETiRet; +} + +/* store parse result ,use list in glib*/ +static rsRetVal +parse_result_store(const grok_match_t gm,instanceData *pData) +{ + GList *re_list = NULL; + char *pname; + const char *pdata; + int pname_len,pdata_len; + + char *key; + char *type; + DEFiRet; + + grok_match_walk_init(&gm); //grok API + + while(grok_match_walk_next(&gm,&pname,&pname_len,&pdata,&pdata_len) == 0) + { + /* parse key and value type from patterns */ + key = strchr(pname,':'); + + if(key!=NULL) + { + int key_len; + result_t *result = g_new0(result_t,1); + key_len = pname_len - ((key+1) - pname); + key = key+1; + pname_len = key_len; + type = strchr(key,':'); + int type_len; + if(type!=NULL) + { + key_len = (type - key); + type = type+1; + type_len = pname_len - key_len -1; + sprintf(type,"%.*s",type_len,type); + } + else{type = "null";} + /* store parse result into list */ + result->key = key; + result->key_len = key_len; + result->value = pdata; + result->value_len = pdata_len; + result->type = type; + /* the value of merger the same key*/ + re_list = g_list_append(re_list,result); + } + } + msg_to_json(re_list,pData); + g_list_free(re_list); + grok_match_walk_end(&gm); + RETiRet; +} + +/* motify message for per line */ +static rsRetVal +MotifyLine(char *line,grok_t *grok,instanceData *pData) +{ + grok_match_t gm; + DEFiRet; + grok_patterns_import_from_file(grok,pData->pszPatternDir); + int compile = grok_compile(grok,pData->pszMatch); + if(compile!=GROK_OK) + { + DBGPRINTF("mmgrok: grok_compile faile!exit code: %d\n",compile); + ABORT_FINALIZE(RS_RET_ERR); + } + int exe = grok_exec(grok,line,&gm); + if(exe!=GROK_OK) + { + DBGPRINTF("mmgrok: grok_exec faile!exit code: %d\n",exe); + ABORT_FINALIZE(RS_RET_ERR); + } + parse_result_store(gm,pData); +finalize_it: + RETiRet; +} + +/* motify rsyslog messages */ +static rsRetVal +MotifyMessage(instanceData *pData) +{ + DEFiRet; + grok_t *grok = CreateGrok(); + char *msg = strdup(pData->pszSource); + char *line = NULL; + line = strtok(msg,"\n"); + while(line!=NULL) + { + MotifyLine(line,grok,pData); + line = strtok(NULL,"\n"); + } + free(msg);msg=NULL; + RETiRet; +} + + +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; + uchar *buf; + instanceData *pData; + +CODESTARTdoAction + pData = pWrkrData->pData; + buf = getMSG(pMsg); + pData->pmsg = pMsg; + while(*buf && isspace(*buf)) { + ++buf; + } + + if(*buf == '\0' ) { + DBGPRINTF("mmgrok: not msg for mmgrok!"); + ABORT_FINALIZE(RS_RET_NO_CEE_MSG); + } + pData->pszSource = (char *)buf; + CHKiRet(MotifyMessage(pData)); + +finalize_it: +ENDdoAction + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + if(strncmp((char*) p, ":mmgrok:", sizeof(":mmgrok:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + p += sizeof(":mmgrok:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_TPL_AS_MSG, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINmodExit +CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES +CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_QUERIES +ENDqueryEtryPt + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + +BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bMsgPassingSupported; +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("mmgrok: module compiled with rsyslog version %s.\n", VERSION); + bMsgPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", + &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_MSG) + bMsgPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, not acceptable */ + } + + if(!bMsgPassingSupported) { + DBGPRINTF("mmgrok: msg-passing is not supported by rsyslog core, " + "can not continue.\n"); + ABORT_FINALIZE(RS_RET_NO_MSG_PASSING); + } + + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit diff --git a/contrib/mmsequence/mmsequence.c b/contrib/mmsequence/mmsequence.c index 766d17438..8de624787 100644 --- a/contrib/mmsequence/mmsequence.c +++ b/contrib/mmsequence/mmsequence.c @@ -312,15 +312,15 @@ getCounter(struct hashtable *ht, char *str, int initial) { } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; struct json_object *json; int val = 0; int *pCounter; instanceData *pData; CODESTARTdoAction pData = pWrkrData->pData; - pMsg = (msg_t*) ppString[0]; switch(pData->mode) { case mmSequenceRandom: diff --git a/contrib/omczmq/omczmq.c b/contrib/omczmq/omczmq.c index 27064becc..42c1a8b4c 100644 --- a/contrib/omczmq/omczmq.c +++ b/contrib/omczmq/omczmq.c @@ -47,36 +47,81 @@ DEFobjCurrIf(errmsg) static pthread_mutex_t mutDoAct = PTHREAD_MUTEX_INITIALIZER; typedef struct _instanceData { - zactor_t *authActor; - zactor_t *beaconActor; - zsock_t *sock; - zcert_t *clientCert; - zcert_t *serverCert; - char *sockEndpoints; - int sockType; - char *authType; - char *clientCertPath; - char *serverCertPath; - uchar *tplName; - char *beacon; - int beaconport; - char *topicList; + zactor_t *authActor; /* reference to a CZMQ auth actor */ + zactor_t *beaconActor; /* reference to a CZMQ beacon actor */ + zsock_t *sock; /* zeromq socket */ + bool serverish; /* whether socket will be a server (bind) or client */ + int sendTimeout; /* send timeout for socket (defaults to -1) */ + zcert_t *clientCert; /* client curve certificate */ + zcert_t *serverCert; /* server curve certificate */ + zlist_t *topics; /* publish topics if set */ + bool sendError; /* set to true of a socket send timed out */ + char *sockEndpoints; /* (required) comma delimited list of endpoints */ + int sockType; /* (required) the type of our zeromq socket */ + char *authType; /* (optional) type of authentication */ + char *clientCertPath; /* (optional) path to curve client cert(s) */ + char *serverCertPath; /* (optional) path to curve server cert */ + uchar *tplName; /* (optional) name of template to use */ + char *beacon; /* (optional) content of beacon if beaconing */ + int beaconport; /* (optional) port of beacon if beaconing */ + char *topicList; /* (optional) topic list */ + bool topicFrame; /* (optional) whether to set topic as separate frame */ } instanceData; typedef struct wrkrInstanceData { instanceData *pData; } wrkrInstanceData_t; +/* ------------------------------------------------------------- + * Explanation of Options + * ---------------------- + * endpoints + * A comma delimited list of endpoints to connect to. Whether the + * endpoint connects or binds is determined by defaults for + * each socket type. + * socktype + * The type of zeromq socket to use; supports PUB, PUSH and DEALER. + * sndTimeout + * Send timeout. Defaults to -1, which is "block forever" and the + * standard ZeroMQ default. 0 = immediate, otherwise is value + * in milliseconds. + * beacon + * The beacon is lit! If beacon is set to a value, that value + * will be used to advertise the existence of this endpoint + * over udp multicast beacons. + * beaconport + * The port to use for the beacon + * authtype + * The authentication type to use - currently only supports + * CURVESERVER and CURVECLIENT. + * clientcertpath + * The path to client certificate(s). Must be set if authtype is + * set to CURVE. + * servercertpath + * The path to server certificate. Must be set if authtype is + * set to CURVE + * template + * The template to use to format outgoing logs. If not set, + * defaults to RSYSLOG_ForwardFormat. + * topics + * A comma delimited list of topics to publish messages on. + * If this is set, each message will be sent once per topic + * in the list. Note that ZeroMQ "topics" are simply matches + * against the first bytes in a message, so dynamic topics + * per message can be constructed using templates as well. + * ------------------------------------------------------------- */ static struct cnfparamdescr actpdescr[] = { { "endpoints", eCmdHdlrGetWord, 1 }, { "socktype", eCmdHdlrGetWord, 1 }, + { "sendtimeout", eCmdHdlrGetWord, 0 }, { "beacon", eCmdHdlrGetWord, 0 }, { "beaconport", eCmdHdlrGetWord, 0 }, { "authtype", eCmdHdlrGetWord, 0 }, { "clientcertpath", eCmdHdlrGetWord, 0 }, { "servercertpath", eCmdHdlrGetWord, 0 }, { "template", eCmdHdlrGetWord, 0 }, - { "topics", eCmdHdlrGetWord, 0 } + { "topics", eCmdHdlrGetWord, 0 }, + { "topicframe", eCmdHdlrGetWord, 0} }; static struct cnfparamblk actpblk = { @@ -88,125 +133,151 @@ static struct cnfparamblk actpblk = { static rsRetVal initCZMQ(instanceData* pData) { DEFiRet; - /* tell czmq to not use it's own signal handler */ - putenv ("ZSYS_SIGHANDLER=false"); + /* Turn off CZMQ signal handling */ + /* ----------------------------- */ + + putenv ("ZSYS_SIGHANDLER=false"); + + /* Create the authentication actor */ + /* ------------------------------ */ - /* create new auth actor */ - DBGPRINTF ("omczmq: starting auth actor...\n"); pData->authActor = zactor_new(zauth, NULL); + if (!pData->authActor) { errmsg.LogError(0, RS_RET_NO_ERRCODE, - "omczmq: could not create auth service"); - ABORT_FINALIZE (RS_RET_NO_ERRCODE); + "omczmq: could not create auth actor"); + ABORT_FINALIZE (RS_RET_SUSPENDED); } - /* create our zeromq socket */ - DBGPRINTF ("omczmq: creating zeromq socket...\n"); + DBGPRINTF ("omczmq: auth actor started\n"); + + /* Create the zeromq socket */ + /* ------------------------ */ + pData->sock = zsock_new(pData->sockType); + if (!pData->sock) { errmsg.LogError(0, RS_RET_NO_ERRCODE, "omczmq: new socket failed for endpoints: %s", pData->sockEndpoints); - ABORT_FINALIZE(RS_RET_NO_ERRCODE); + ABORT_FINALIZE(RS_RET_SUSPENDED); } - bool is_server = false; + zsock_set_sndtimeo (pData->sock, pData->sendTimeout); + DBGPRINTF ("omczmq: created socket with timeout %d...\n", pData->sendTimeout); + + /* Create the beacon actor if configured */ + /* ------------------------------------- */ - /* if a beacon is set start it */ if((pData->beacon != NULL) && (pData->beaconport > 0)) { - DBGPRINTF ("omczmq: starting beacon actor...\n"); - /* create the beacon actor, if it fails abort */ pData->beaconActor = zactor_new(zbeacon, NULL); + if (!pData->beaconActor) { errmsg.LogError(0, RS_RET_NO_ERRCODE, "omczmq: could not create beacon service"); - ABORT_FINALIZE (RS_RET_NO_ERRCODE); + ABORT_FINALIZE (RS_RET_SUSPENDED); } - /* configure the beacon and abort if UDP broadcasting ont available */ zsock_send(pData->beaconActor, "si", "CONFIGURE", pData->beaconport); char *hostname = zstr_recv(pData->beaconActor); + if (!*hostname) { errmsg.LogError(0, RS_RET_NO_ERRCODE, "omczmq: no UDP broadcasting available"); - ABORT_FINALIZE (RS_RET_NO_ERRCODE); + ABORT_FINALIZE (RS_RET_SUSPENDED); } - /* start publishing the beacon */ zsock_send(pData->beaconActor, "sbi", "PUBLISH", pData->beacon, strlen(pData->beacon)); + + DBGPRINTF ("omczmq: beacon is lit: hostname: '%s', port: '%d'...\n", + hostname, pData->beaconport); + + zstr_free (&hostname); } - /* if we are a CURVE server */ - if ((pData->authType != NULL) && (!strcmp(pData->authType, "CURVESERVER"))) { - DBGPRINTF("omczmq: we are a curve server...\n"); + /* Load certs for auth if auth is used */ + /* ----------------------------------- */ + + if (pData->authType != NULL) { + + /* CURVESERVER */ + /* ---------- */ + + if (!strcmp(pData->authType, "CURVESERVER")) { - is_server = true; + zsock_set_zap_domain(pData->sock, "global"); + zsock_set_curve_server(pData->sock, 1); - /* set global auth domain */ - zsock_set_zap_domain(pData->sock, "global"); + pData->serverCert = zcert_load(pData->serverCertPath); - /* set that we are a curve server */ - zsock_set_curve_server(pData->sock, 1); + if (!pData->serverCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); + ABORT_FINALIZE(RS_RET_ERR); + } - /* get and set our server cert */ - DBGPRINTF("omczmq: server cert is %s...\n", pData->serverCertPath); - pData->serverCert = zcert_load(pData->serverCertPath); - if (!pData->serverCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); - ABORT_FINALIZE(RS_RET_ERR); + zcert_apply(pData->serverCert, pData->sock); + + zstr_sendx(pData->authActor, "CURVE", pData->clientCertPath, NULL); + zsock_wait(pData->authActor); + + DBGPRINTF("omczmq: CURVESERVER: serverCertPath: '%s'\n", pData->serverCertPath); + DBGPRINTF("omczmq: CURVESERVER: clientCertPath: '%s'\n", pData->clientCertPath); } - zcert_apply(pData->serverCert, pData->sock); + /* CURVECLIENT */ + /* ----------- */ - /* set allowed clients */ - DBGPRINTF("omczmq: allowed clients are %s...\n", pData->clientCertPath); - zstr_sendx(pData->authActor, "CURVE", pData->clientCertPath, NULL); - zsock_wait(pData->authActor); - } + else if (!strcmp(pData->authType, "CURVECLIENT")) { + if (!strcmp(pData->clientCertPath, "*")) { + pData->clientCert = zcert_new(); + } + else { + pData->clientCert = zcert_load(pData->clientCertPath); + } + + if (!pData->clientCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load client cert"); + ABORT_FINALIZE(RS_RET_ERR); + } - /* if we are a CURVE client */ - if ((pData->authType != NULL) && (!strcmp(pData->authType, "CURVECLIENT"))) { - DBGPRINTF("omczmq: we are a curve client...\n"); + zcert_apply(pData->clientCert, pData->sock); - is_server = false; + pData->serverCert = zcert_load(pData->serverCertPath); - /* get our client cert */ - pData->clientCert = zcert_load(pData->clientCertPath); - if (!pData->clientCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load client cert"); - ABORT_FINALIZE(RS_RET_ERR); + if (!pData->serverCert) { + errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); + ABORT_FINALIZE(RS_RET_ERR); + } + + char *server_key = zcert_public_txt(pData->serverCert); + zsock_set_curve_serverkey (pData->sock, server_key); + + DBGPRINTF("omczmq: CURVECLIENT: serverCertPath: '%s'\n", pData->serverCertPath); + DBGPRINTF("omczmq: CURVECLIENT: clientCertPath: '%s'\n", pData->clientCertPath); + DBGPRINTF("omczmq: CURVECLIENT: server_key: '%s'\n", server_key); } - - /* apply the client cert to the socket */ - zcert_apply(pData->clientCert, pData->sock); - - /* get the server cert */ - DBGPRINTF("omczmq: server cert is %s...\n", pData->serverCertPath); - pData->serverCert = zcert_load(pData->serverCertPath); - if (!pData->serverCert) { - errmsg.LogError(0, NO_ERRCODE, "could not load server cert"); - ABORT_FINALIZE(RS_RET_ERR); + else { + errmsg.LogError(0, NO_ERRCODE, "unrecognized auth type: '%s'", pData->authType); + ABORT_FINALIZE(RS_RET_ERR); } - - /* get the server public key and set it for the socket */ - char *server_key = zcert_public_txt(pData->serverCert); - DBGPRINTF("omczmq: server public key is %s...\n", server_key); - zsock_set_curve_serverkey (pData->sock, server_key); + } + + switch (pData->sockType) { + case ZMQ_PUB: + pData->serverish = true; + break; + case ZMQ_PUSH: + case ZMQ_DEALER: + pData->serverish = false; + break; } - /* if we are a PUB server */ - if (pData->sockType == ZMQ_PUB) { - DBGPRINTF("omczmq: we are a pub server...\n"); - is_server = true; - } - - /* we default to CONNECT unless told otherwise */ - int rc = zsock_attach(pData->sock, (const char*)pData->sockEndpoints, is_server); + int rc = zsock_attach(pData->sock, (const char*)pData->sockEndpoints, pData->serverish); if (rc == -1) { errmsg.LogError(0, NO_ERRCODE, "zsock_attach to %s failed", pData->sockEndpoints); - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_SUSPENDED); } finalize_it: @@ -221,61 +292,35 @@ rsRetVal outputCZMQ(uchar* msg, instanceData* pData) { CHKiRet(initCZMQ(pData)); } - /* if we have a ZMQ_PUB sock and topics, send with topics */ - if (pData->sockType == ZMQ_PUB && pData->topicList) { - char topic[256], *list = pData->topicList; - int rc; + /* if we have a ZMQ_PUB sock and topics, send once per topic */ + if (pData->sockType == ZMQ_PUB && pData->topics) { + char *topic = zlist_first(pData->topics); + + while (topic) { + if (pData->topicFrame) { + int rc = zstr_sendx(pData->sock, topic, (char*)msg, NULL); + if (rc != 0) { + pData->sendError = true; + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } + else { + int rc = zstr_sendf(pData->sock, "%s%s", topic, (char*)msg); + if (rc != 0) { + pData->sendError = true; + ABORT_FINALIZE(RS_RET_SUSPENDED); + } - while (list) { - char *delimiter = strchr(list, ','); - if (!delimiter) { - delimiter = list + strlen (list); } - - if (delimiter - list > 255) { - errmsg.LogError(0, NO_ERRCODE, - "pData->topicList must be under 256 characters"); - ABORT_FINALIZE(RS_RET_ERR); - } - - memcpy(topic, list, delimiter - list); - topic[delimiter - list] = 0; - - /* send the topic */ - rc = zstr_sendm(pData->sock, topic); - - /* something is very wrong */ - if (rc == -1) { - errmsg.LogError(0, NO_ERRCODE, "omczmq: send of topic %s failed" - ": %s", topic, zmq_strerror(errno)); - ABORT_FINALIZE(RS_RET_ERR); - } - - /* send the log line */ - rc = zstr_send(pData->sock, (char*)msg); - - /* something is very wrong */ - if (rc == -1) { - errmsg.LogError(0, NO_ERRCODE, "omczmq: send of %s failed: %s", - msg, zmq_strerror(errno)); - ABORT_FINALIZE(RS_RET_ERR); - } - - if (*delimiter == 0) { - break; - } - - list = delimiter + 1; + topic = zlist_next(pData->topics); } - } else { - /* send the log line */ + } + /* otherwise do a normal send */ + else { int rc = zstr_send(pData->sock, (char*)msg); - - /* something is very wrong */ - if (rc == -1) { - errmsg.LogError(0, NO_ERRCODE, "omczmq: send of %s failed: %s", - msg, zmq_strerror(errno)); - ABORT_FINALIZE(RS_RET_ERR); + if (rc != 0) { + pData->sendError = true; + ABORT_FINALIZE(RS_RET_SUSPENDED); } } finalize_it: @@ -286,16 +331,22 @@ static inline void setInstParamDefaults(instanceData* pData) { pData->sockEndpoints = NULL; pData->sock = NULL; + pData->sendError = false; + pData->serverish = false; pData->clientCert = NULL; pData->serverCert = NULL; pData->tplName = NULL; pData->sockType = -1; + pData->sendTimeout = -1; + pData->topics = NULL; pData->authActor = NULL; pData->beaconActor = NULL; + pData->beaconport = 0; pData->authType = NULL; pData->clientCertPath = NULL; pData->serverCertPath = NULL; pData->topicList = NULL; + pData->topicFrame = true; } @@ -324,6 +375,8 @@ ENDdbgPrintInstInfo BEGINfreeInstance CODESTARTfreeInstance + if (pData->topics != NULL) + zlist_destroy(&pData->topics); zsock_destroy(&pData->sock); zactor_destroy(&pData->authActor); zactor_destroy(&pData->beaconActor); @@ -345,19 +398,24 @@ ENDfreeWrkrInstance BEGINtryResume + instanceData *pData; CODESTARTtryResume pthread_mutex_lock(&mutDoAct); - if(NULL == pWrkrData->pData->sock) { - iRet = initCZMQ(pWrkrData->pData); - } + pData = pWrkrData->pData; + DBGPRINTF("omczmq: trying to resume...\n"); + zsock_destroy(&pData->sock); + zactor_destroy(&pData->beaconActor); + zactor_destroy(&pData->authActor); + iRet = initCZMQ(pData); pthread_mutex_unlock(&mutDoAct); ENDtryResume BEGINdoAction - instanceData *pData = pWrkrData->pData; + instanceData *pData; CODESTARTdoAction pthread_mutex_lock(&mutDoAct); + pData = pWrkrData->pData; iRet = outputCZMQ(ppString[0], pData); pthread_mutex_unlock(&mutDoAct); ENDdoAction @@ -375,45 +433,66 @@ CODESTARTnewActInst setInstParamDefaults(pData); CODE_STD_STRING_REQUESTnewActInst(1) + + /* handle our configuration arguments + * - endpoints - required + * - socktype - required + * - beacon - optional + * - beaconport - required if beacon is set + * - authtype - optional and NULL == no auth + * - clientcertpath - required if authtype is CURVESERVER or CURVECLIENT + * - servercertpath - required if authtype is CURVESERVER or CURVECLIENT + * - template - optional defaults to RSYSLOG_ForwardFormat + * - topics - optional + */ + + /* Handle options from the configuration */ + /* ------------------------------------- */ + for (i = 0; i < actpblk.nParams; ++i) { if (!pvals[i].bUsed) { continue; } - /* get the socket endpoints to use */ if (!strcmp(actpblk.descr[i].name, "endpoints")) { pData->sockEndpoints = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get the output template to use */ else if (!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); - } - - /* get the socket type */ + } + else if (!strcmp(actpblk.descr[i].name, "sendtimeout")) { + pData->sendTimeout = atoi(es_str2cstr(pvals[i].val.d.estr, NULL)); + } else if (!strcmp(actpblk.descr[i].name, "socktype")){ char *stringType = es_str2cstr(pvals[i].val.d.estr, NULL); - - if (!strcmp("PUB", stringType)) { - pData->sockType = ZMQ_PUB; + if(stringType != NULL){ + if (!strcmp("PUB", stringType)) { + pData->sockType = ZMQ_PUB; + } + else if (!strcmp("PUSH", stringType)) { + pData->sockType = ZMQ_PUSH; + } + else if (!strcmp("DEALER", stringType)) { + pData->sockType = ZMQ_DEALER; + } + else { + free(stringType); + errmsg.LogError(0, RS_RET_CONFIG_ERROR, + "omczmq: invalid socktype"); + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + } + free(stringType); } - - else if (!strcmp("PUSH", stringType)) { - pData->sockType = ZMQ_PUSH; - } - - else { - errmsg.LogError(0, RS_RET_CONFIG_ERROR, - "omczmq: invalid socktype"); - ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + else{ + errmsg.LogError(0, RS_RET_OUT_OF_MEMORY, + "omczmq: out of memory"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } } - - /* get the authentication type to use */ else if (!strcmp(actpblk.descr[i].name, "authtype")) { pData->authType = es_str2cstr(pvals[i].val.d.estr, NULL); - /* make sure defined type is supported */ + /* auth must be NULL, CURVESERVER or CURVECLIENT */ if ((pData->authType != NULL) && strcmp("CURVESERVER", pData->authType) && strcmp("CURVECLIENT", pData->authType)) @@ -425,28 +504,27 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } } - - /* get client cert argument */ else if (!strcmp(actpblk.descr[i].name, "clientcertpath")) { pData->clientCertPath = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get server cert argument */ else if (!strcmp(actpblk.descr[i].name, "servercertpath")) { pData->serverCertPath = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get beacon argument */ else if (!strcmp(actpblk.descr[i].name, "beacon")) { pData->beacon = es_str2cstr(pvals[i].val.d.estr, NULL); } - - /* get beacon port */ else if (!strcmp(actpblk.descr[i].name, "beaconport")) { pData->beaconport = atoi(es_str2cstr(pvals[i].val.d.estr, NULL)); } - - /* get the subscription topics */ + else if (!strcmp(actpblk.descr[i].name, "topicframe")) { + int tframe = atoi(es_str2cstr(pvals[i].val.d.estr, NULL)); + if (tframe == 1) { + pData->topicFrame = true; + } + else { + pData->topicFrame = false; + } + } else if(!strcmp(actpblk.descr[i].name, "topics")) { if (pData->sockType != ZMQ_PUB) { errmsg.LogError(0, RS_RET_CONFIG_ERROR, @@ -454,38 +532,62 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } - pData->topicList = es_str2cstr(pvals[i].val.d.estr, NULL); - } + // create a list of topics + pData->topics = zlist_new(); + char *topics = es_str2cstr(pvals[i].val.d.estr, NULL); + char *topics_org = topics; + char topic[256]; + if(topics == NULL){ + errmsg.LogError(0, RS_RET_OUT_OF_MEMORY, + "out of memory"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + while (*topics) { + char *delimiter = strchr(topics, ','); + if (!delimiter) { + delimiter = topics + strlen(topics); + } + if (delimiter - topics > 255) { + errmsg.LogError(0, RS_RET_CONFIG_ERROR, + "topics must be under 256 characters"); + free(topics_org); + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + } + memcpy (topic, topics, delimiter - topics); + topic[delimiter-topics] = 0; + char *current_topic = strdup(topic); + zlist_append (pData->topics, current_topic); + if (*delimiter == 0) { + break; + } + topics = delimiter + 1; + } + free(topics_org); - /* the config has a bad option */ + } else { - errmsg.LogError(0, NO_ERRCODE, "omczmq: %s is not a valid option", + errmsg.LogError(0, NO_ERRCODE, + "omczmq: config error - '%s' is not a valid option", actpblk.descr[i].name); ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } } - /* if we don't have a template name, - * use the default RSYSLOG_ForwardFormat template */ + /* Defaults and sanity checking */ + /* ----------------------------- */ + if (pData->tplName == NULL) { CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*)strdup("RSYSLOG_ForwardFormat"), OMSR_NO_RQD_TPL_OPTS)); } - - /* use the requested template */ else { CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*)pData->tplName, OMSR_NO_RQD_TPL_OPTS)); } - /* error if no socket endpoints are defined */ - if (NULL == pData->sockEndpoints) { - errmsg.LogError(0, RS_RET_CONFIG_ERROR, "omczmq: sockEndpoint required"); - ABORT_FINALIZE(RS_RET_CONFIG_ERROR); - } - - /* error if no socket type is defined */ - if (pData->sockType == -1) { - errmsg.LogError(0, RS_RET_CONFIG_ERROR, "omczmq: socktype required"); + if ((pData->beacon != NULL) && (pData->beaconport == 0)) { + errmsg.LogError(0, NO_ERRCODE, + "omczmq: config error - beaconport is required if beacon is set"); ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } diff --git a/contrib/omhttpfs/omhttpfs.c b/contrib/omhttpfs/omhttpfs.c index 76653434a..abb1a19b0 100644 --- a/contrib/omhttpfs/omhttpfs.c +++ b/contrib/omhttpfs/omhttpfs.c @@ -43,11 +43,8 @@ #include "statsobj.h" #include "unicode-helper.h" -// output module MODULE_TYPE_OUTPUT -// do not need to be kept dynamically linked MODULE_TYPE_NOKEEP -// module name MODULE_CNFNAME("omhttpfs") /* internal structures @@ -133,9 +130,7 @@ static struct cnfparamdescr actpdescr[] = { { "host", eCmdHdlrGetWord, 0 }, { "port", eCmdHdlrInt, 0 }, { "user", eCmdHdlrGetWord, 0 }, - //{ "ip", eCmdHdlrGetWord, 0 }, { "https", eCmdHdlrBinary, 0 }, - //{ "timeout", eCmdHdlrInt, 0 }, { "file", eCmdHdlrGetWord, CNFPARAM_REQUIRED }, { "isdynfile", eCmdHdlrBinary, 0 }, { "template", eCmdHdlrGetWord, 0 }, @@ -167,12 +162,12 @@ httpfs_init_curl(wrkrInstanceData_t *pWrkrData, instanceData *pData) if (pData->https) { DBGPRINTF("%s(): Enable HTTPS\n", __FUNCTION__); - // for ssl + /* for ssl */ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); } } else { - // LOG + /* LOG */ errmsg.LogError(0, RS_RET_OBJ_CREATION_FAILED, "omhttpfs: failed to init cURL\n"); return RS_RET_OBJ_CREATION_FAILED; @@ -195,10 +190,8 @@ httpfs_init_curl(wrkrInstanceData_t *pWrkrData, instanceData *pData) static rsRetVal httpfs_build_url(wrkrInstanceData_t *pWrkrData, char* op, es_str_t** url_buf) { - // *url_buf = es_newStr(HTTPFS_URL_BUFFER_LENGTH); - // scheme if (pWrkrData->pData->https) { es_addBuf(url_buf, "https://", sizeof("https://")-1); } else { @@ -349,15 +342,24 @@ static size_t httpfs_curl_result_callback(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; + char *newreply = NULL; wrkrInstanceData_t *mem = (wrkrInstanceData_t *)userp; - - mem->reply = realloc(mem->reply, mem->replyLen + realsize + 1); - if(mem->reply == NULL) { + + newreply = realloc(mem->reply, mem->replyLen + realsize + 1); + if (newreply == NULL) { /* out of memory! */ - printf("not enough memory (realloc returned NULL)\n"); + dbgprintf("not enough memory (realloc returned NULL)\n"); + + if (mem->reply != NULL) + free(mem->reply); + + mem->reply = NULL; + mem->replyLen = 0; + return 0; } + mem->reply = newreply; memcpy(&(mem->reply[mem->replyLen]), contents, realsize); mem->replyLen += realsize; mem->reply[mem->replyLen] = 0; @@ -427,7 +429,9 @@ int httpfs_permission_to_string(int permission, char* perm_string) */ static rsRetVal httpfs_parse_exception(char* buf, int length, httpfs_json_remote_exception* jre) -{ +{ + DEFiRet; + if (!length) { return RS_RET_JSON_PARSE_ERR; } @@ -438,29 +442,43 @@ httpfs_parse_exception(char* buf, int length, httpfs_json_remote_exception* jre) struct json_object *json; json = json_tokener_parse_ex(jt, buf, length); if (!json_object_is_type(json, json_type_object)) { - // not an object ? - return RS_RET_JSON_PARSE_ERR; + ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR); } - if (!json_object_object_get_ex(json, "RemoteException", &json)) { - return RS_RET_JSON_PARSE_ERR; + if (!RS_json_object_object_get_ex(json, "RemoteException", &json)) { + ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR); } struct json_object *jobj; memset(jre, 0, sizeof(*jre)); - json_object_object_get_ex(json, "javaClassName", &jobj); - strncpy(jre->class, (char*) json_object_get_string(jobj), json_object_get_string_len(jobj)); + const char *str; + size_t len; - json_object_object_get_ex(json, "exception", &jobj); - strncpy(jre->exception, (char*) json_object_get_string(jobj), json_object_get_string_len(jobj)); + RS_json_object_object_get_ex(json, "javaClassName", &jobj); + str = json_object_get_string(jobj); + len = strlen(str); + strncpy(jre->class, str, len); - json_object_object_get_ex(json, "message", &jobj); - strncpy(jre->message, (char*) json_object_get_string(jobj), json_object_get_string_len(jobj)); + RS_json_object_object_get_ex(json, "exception", &jobj); + str = json_object_get_string(jobj); + len = strlen(str); + strncpy(jre->exception, str, len); + + RS_json_object_object_get_ex(json, "message", &jobj); + str = json_object_get_string(jobj); + len = strlen(str); + strncpy(jre->message, str, len); + +finalize_it: + if(jt != NULL) + json_tokener_free(jt); + if(json != NULL) + json_object_put(json); + RETiRet; +} - return RS_RET_OK; -} #if 0 /** @@ -514,7 +532,7 @@ HTTPFS_CURL_VARS_RELEASE static rsRetVal httpfs_create_file(wrkrInstanceData_t *pWrkrData, uchar* buf) { - // httpfs.create automatically create folders, no mkdirs needed. + /* httpfs.create automatically create folders, no mkdirs needed. */ /* curl -b /tmp/c.tmp -c /tmp/c.tmp -d 'aaaaabbbbb' -i -H 'Content-Type: application/octet-stream' -X PUT \ @@ -544,7 +562,6 @@ HTTPFS_CURL_EXEC int success = 0; if (response_code == 201) { - //&& !strncmp(result->buf, HTTPFS_JSON_BOOLEAN_TRUE, strlen(HTTPFS_JSON_BOOLEAN_TRUE))) { success = 1; } @@ -590,7 +607,7 @@ HTTPFS_CURL_EXEC if (response_code == 200) { success = 1; } else if (response_code == 404) { - // TODO: 404 ? + /* TODO: 404 ? */ } HTTPFS_CURL_VARS_RELEASE @@ -627,11 +644,11 @@ HTTPFS_CURL_EXEC if (response_code == 200) { success = 1; } else if (response_code == 404) { - // TODO: 404 ? + /* TODO: 404 ? */ } - // TODO: not success? + /* TODO: not success? */ HTTPFS_CURL_VARS_RELEASE if (success) { @@ -672,7 +689,7 @@ httpfs_log(wrkrInstanceData_t *pWrkrData, uchar* buf) curl_easy_getinfo(pWrkrData->curl, CURLINFO_RESPONSE_CODE, &response_code); if (response_code != 404) { - // todo: log error + /* TODO: log error */ DBGPRINTF("omhttpfs: Append fail HTTP %ld: %s\n", response_code, pWrkrData->file); return RS_RET_FALSE; } @@ -691,21 +708,19 @@ httpfs_log(wrkrInstanceData_t *pWrkrData, uchar* buf) if (response_code == 500) { DBGPRINTF("omhttpfs: Create file failed HTTP %ld: %s\n", response_code, pWrkrData->file); - // retry httpfs_parse_exception(pWrkrData->reply, pWrkrData->replyLen, &jre); if (!strncmp(jre.exception, HTTPFS_FILEALREADYEXISTSEXCEPTION, strlen(HTTPFS_FILEALREADYEXISTSEXCEPTION))) { - // file exists, go to append + /* file exists, go to append */ DBGPRINTF("omhttpfs: File already exists, append again: %s\n", pWrkrData->file); iRet = httpfs_append_file(pWrkrData, buf); if (iRet == RS_RET_OK) { DBGPRINTF("omhttpfs: Re-Append success: %s\n", pWrkrData->file); - //free(&jre); return RS_RET_OK; } else { DBGPRINTF("omhttpfs: Re-Append failed: %s\n", pWrkrData->file); - // error - // exit + /* error + exit */ } } else { @@ -715,7 +730,6 @@ httpfs_log(wrkrInstanceData_t *pWrkrData, uchar* buf) DBGPRINTF("omhttpfs: Create file failed: %s %s\n", pWrkrData->file, pWrkrData->reply); } - // TODO: ... return RS_RET_FALSE; } @@ -734,7 +748,6 @@ ENDcreateInstance BEGINcreateWrkrInstance CODESTARTcreateWrkrInstance DBGPRINTF("omhttpfs: createWrkrInstance\n"); - // init worker resource pWrkrData->curl = NULL; CHKiRet(httpfs_init_curl(pWrkrData, pWrkrData->pData)); finalize_it: @@ -783,7 +796,7 @@ ENDdbgPrintInstInfo BEGINtryResume CODESTARTtryResume DBGPRINTF("omhttpfs: tryResume called\n"); - // TODO: test networking + /* TODO: test networking */ iRet = RS_RET_OK; ENDtryResume @@ -793,14 +806,14 @@ ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF("omhttpfs: doAction\n"); - // dynamic file name + /* dynamic file name */ if (pWrkrData->pData->isDynFile) { pWrkrData->file = ustrdup(ppString[1]); } else { pWrkrData->file = ustrdup(pWrkrData->pData->file); } - // ppString[0] -> log content + /* ppString[0] -> log content */ iRet = httpfs_log(pWrkrData, ppString[0]); if(iRet != RS_RET_OK) { @@ -849,7 +862,7 @@ CODESTARTnewActInst if(!strcmp(actpblk.descr[i].name, "host")) { pData->host = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "port")) { - pData->port = (int) pvals[i].val.d.n, NULL; + pData->port = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "user")) { pData->user = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); @@ -868,16 +881,13 @@ CODESTARTnewActInst DBGPRINTF("omhttpfs: program error, non-handled param '%s'\n", actpblk.descr[i].name); } } - // empty user if(pData->user == NULL || pData->user[0] == '\0') { pData->user = ustrdup((uchar*) OMHTTPFS_DEFAULT_USER); } - // empty host if(pData->host == NULL || pData->host[0] == '\0') { pData->host = ustrdup((uchar*) OMHTTPFS_DEFAULT_HOST); } - // register template for file name if (pData->isDynFile) { CODE_STD_STRING_REQUESTparseSelectorAct(2) @@ -886,7 +896,6 @@ CODESTARTnewActInst CODE_STD_STRING_REQUESTparseSelectorAct(1) } - // register template for log content tplToUse = ustrdup((pData->tplName == NULL) ? (uchar* ) "RSYSLOG_FileFormat" : pData->tplName); CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS)); @@ -897,8 +906,6 @@ ENDnewActInst BEGINparseSelectorAct CODESTARTparseSelectorAct - // this is for the legacy configuration format - // forget it ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct diff --git a/contrib/omrabbitmq/omrabbitmq.c b/contrib/omrabbitmq/omrabbitmq.c index 8ea7e6233..73176992b 100644 --- a/contrib/omrabbitmq/omrabbitmq.c +++ b/contrib/omrabbitmq/omrabbitmq.c @@ -41,6 +41,9 @@ #include "cfsysline.h" #include +#include + +#define RABBITMQ_CHANNEL 1 MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP @@ -69,9 +72,13 @@ typedef struct _instanceData { uchar *vhost; uchar *user; uchar *password; - uchar *exchange; + char *exchange; uchar *routing_key; uchar *tplName; + char *exchange_type; + int durable; + int auto_delete; + int delivery_mode; } instanceData; typedef struct wrkrInstanceData { @@ -89,7 +96,11 @@ static struct cnfparamdescr actpdescr[] = { { "password", eCmdHdlrGetWord, 0 }, { "exchange", eCmdHdlrGetWord, 0 }, { "routing_key", eCmdHdlrGetWord, 0 }, - { "template", eCmdHdlrGetWord, 0 } + { "template", eCmdHdlrGetWord, 0 }, + { "exchange_type", eCmdHdlrGetWord, 0}, + { "durable", eCmdHdlrNonNegInt, 0}, + { "auto_delete", eCmdHdlrNonNegInt, 0}, + { "delivery_mode", eCmdHdlrNonNegInt, 0} }; static struct cnfparamblk actpblk = { @@ -108,9 +119,9 @@ die_on_error(int x, char const *context) int retVal = 0; // false if (x < 0) { - char *errstr = amqp_error_string(-x); + const char *errstr = amqp_error_string2(-x); errmsg.LogError(0, RS_RET_ERR, "omrabbitmq: %s: %s", context, errstr); - free(errstr); + retVal = 1; // true } @@ -136,7 +147,7 @@ die_on_amqp_error(amqp_rpc_reply_t x, char const *context) break; case AMQP_RESPONSE_LIBRARY_EXCEPTION: - errmsg.LogError(0, RS_RET_ERR, "omrabbitmq: %s: %s", context, amqp_error_string(x.library_error)); + errmsg.LogError(0, RS_RET_ERR, "omrabbitmq: %s: %s", context, amqp_error_string2(x.library_error)); break; case AMQP_RESPONSE_SERVER_EXCEPTION: @@ -195,19 +206,26 @@ closeAMQPConnection(instanceData *pData) static rsRetVal initRabbitMQ(instanceData *pData) { - int sockfd; + amqp_socket_t *asocket; + amqp_exchange_declare_t edReq; DEFiRet; DBGPRINTF("omrabbitmq: trying connect to '%s' at port %d\n", pData->host, pData->port); pData->conn = amqp_new_connection(); - if (die_on_error(sockfd = amqp_open_socket((char*) pData->host, pData->port), "Opening socket")) { + asocket = amqp_tcp_socket_new(pData->conn); + if (!asocket) { + errmsg.LogError(0, RS_RET_ERR, "omrabbitmq: Error allocating tcp socket"); + pData->conn = NULL; ABORT_FINALIZE(RS_RET_SUSPENDED); } - amqp_set_sockfd(pData->conn, sockfd); + if (die_on_error(amqp_socket_open(asocket, (char*) pData->host, pData->port), "Opening socket")) { + pData->conn = NULL; + ABORT_FINALIZE(RS_RET_SUSPENDED); + } if (die_on_amqp_error(amqp_login(pData->conn, (char*) pData->vhost, 0, 131072, 0, AMQP_SASL_METHOD_PLAIN, pData->user, pData->password), "Logging in")) { @@ -215,13 +233,30 @@ initRabbitMQ(instanceData *pData) ABORT_FINALIZE(RS_RET_SUSPENDED); } - amqp_channel_open(pData->conn, 1); - + amqp_channel_open(pData->conn, RABBITMQ_CHANNEL); if (die_on_amqp_error(amqp_get_rpc_reply(pData->conn), "Opening channel")) { pData->conn = NULL; ABORT_FINALIZE(RS_RET_SUSPENDED); } + if(pData->exchange_type != NULL) { + edReq.ticket = 0; + edReq.exchange = amqp_cstring_bytes(pData->exchange); + edReq.type = amqp_cstring_bytes(pData->exchange_type); + edReq.passive = 0; + edReq.durable = pData->durable; + edReq.auto_delete = pData->auto_delete; + edReq.internal = 0; + edReq.nowait = 0; + edReq.arguments = amqp_empty_table; + + amqp_simple_rpc_decoded(pData->conn, RABBITMQ_CHANNEL, AMQP_EXCHANGE_DECLARE_METHOD, AMQP_EXCHANGE_DECLARE_OK_METHOD, &edReq); + if(die_on_amqp_error(amqp_get_rpc_reply(pData->conn), "Declaring exchange")) { + pData->conn = NULL; + ABORT_FINALIZE(RS_RET_SUSPENDED); + } + } + finalize_it: RETiRet; } @@ -262,6 +297,7 @@ CODESTARTfreeInstance free(pData->exchange); free(pData->routing_key); free(pData->tplName); + free(pData->exchange_type); ENDfreeInstance @@ -281,6 +317,10 @@ CODESTARTdbgPrintInstInfo dbgprintf("\texchange='%s'\n", pData->exchange); dbgprintf("\trouting_key='%s'\n", pData->routing_key); dbgprintf("\ttemplate='%s'\n", pData->tplName); + dbgprintf("\texchange_type='%s'\n", pData->exchange_type); + dbgprintf("\tauto_delete=%d\n", pData->auto_delete); + dbgprintf("\tdurable=%d\n", pData->durable); + dbgprintf("\tdelivery_mode=%d\n", pData->delivery_mode); ENDdbgPrintInstInfo @@ -341,7 +381,7 @@ CODESTARTdoAction body_bytes = amqp_cstring_bytes((char *)ppString[0]); - if (die_on_error(amqp_basic_publish(pData->conn, 1, + if (die_on_error(amqp_basic_publish(pData->conn, RABBITMQ_CHANNEL, cstring_bytes((char *) pData->exchange), cstring_bytes((char *) pData->routing_key), 0, 0, &pData->props, body_bytes), "amqp_basic_publish")) { @@ -365,6 +405,10 @@ setInstParamDefaults(instanceData *pData) pData->exchange = NULL; pData->routing_key = NULL; pData->tplName = NULL; + pData->exchange_type = NULL; + pData->auto_delete = 0; + pData->durable = 0; + pData->delivery_mode = 2; } @@ -396,11 +440,19 @@ CODESTARTnewActInst } else if (!strcmp(actpblk.descr[i].name, "password")) { pData->password = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if (!strcmp(actpblk.descr[i].name, "exchange")) { - pData->exchange = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + pData->exchange = es_str2cstr(pvals[i].val.d.estr, NULL); } else if (!strcmp(actpblk.descr[i].name, "routing_key")) { pData->routing_key = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if (!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if (!strcmp(actpblk.descr[i].name, "exchange_type")) { + pData->exchange_type = es_str2cstr(pvals[i].val.d.estr, NULL); + } else if (!strcmp(actpblk.descr[i].name, "auto_delete")) { + pData->auto_delete = (int) pvals[i].val.d.n; + } else if (!strcmp(actpblk.descr[i].name, "durable")) { + pData->durable = (int) pvals[i].val.d.n; + } else if (!strcmp(actpblk.descr[i].name, "delivery_mode")) { + pData->delivery_mode = (int) pvals[i].val.d.n; } else { dbgprintf("omrabbitmq: program error, non-handled param '%s'\n", actpblk.descr[i].name); } @@ -439,7 +491,7 @@ CODESTARTnewActInst // RabbitMQ properties initialization memset(&pData->props, 0, sizeof pData->props); pData->props._flags = AMQP_BASIC_DELIVERY_MODE_FLAG; - pData->props.delivery_mode = 2; /* persistent delivery mode */ + pData->props.delivery_mode = pData->delivery_mode; pData->props._flags |= AMQP_BASIC_CONTENT_TYPE_FLAG; pData->props.content_type = amqp_cstring_bytes("application/json"); diff --git a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c index e03f44bb0..38a3449fe 100644 --- a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c +++ b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -41,7 +41,6 @@ MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP -MODULE_CNFNAME("pmaixforwardedfrom") PARSER_NAME("rsyslog.aixforwardedfrom") /* internal structures @@ -83,11 +82,9 @@ CODESTARTparse --lenMsg; ++p2parse; } -dbgprintf("pmaixforwardedfrom: msg to look at: [%d]'%s'\n", lenMsg, p2parse); if((unsigned) lenMsg < 42) { /* too short, can not be "our" message */ /* minimum message, 16 character timestamp, 'Message forwarded from ", 1 character name, ': '*/ -dbgprintf("msg too short!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } @@ -97,7 +94,7 @@ dbgprintf("msg too short!\n"); /* if there is the string "Message forwarded from " were the hostname should be */ if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { /* wrong opening text */ -dbgprintf("not a AIX message forwarded from mangled log!\n"); + DBGPRINTF("not a AIX message forwarded from mangled log!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* bump the message portion up by 23 characters to overwrite the "Message forwarded from " with the hostname */ @@ -113,7 +110,7 @@ dbgprintf("not a AIX message forwarded from mangled log!\n"); ++p2parse; } if (lenMsg && *p2parse != ':') { -dbgprintf("not a AIX message forwarded from mangled log but similar enough that the preamble has been removed\n"); + DBGPRINTF("not a AIX message forwarded from mangled log but similar enough that the preamble has been removed\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* bump the message portion up by one character to overwrite the extra : */ diff --git a/contrib/pmcisconames/pmcisconames.c b/contrib/pmcisconames/pmcisconames.c index 9dc7fc838..4951ba949 100644 --- a/contrib/pmcisconames/pmcisconames.c +++ b/contrib/pmcisconames/pmcisconames.c @@ -82,11 +82,9 @@ CODESTARTparse --lenMsg; ++p2parse; } -dbgprintf("pmcisconames: msg to look at: [%d]'%s'\n", lenMsg, p2parse); if((unsigned) lenMsg < 34) { /* too short, can not be "our" message */ /* minimum message, 16 character timestamp, 1 character name, ' : %ASA-1-000000: '*/ -dbgprintf("msg too short!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* check if the timestamp is a 16 character or 21 character timestamp @@ -123,7 +121,7 @@ dbgprintf("msg too short!\n"); /* if the syslog tag is : and the next thing starts with a % assume that this is a mangled cisco log and fix it */ if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { /* wrong opening text */ -dbgprintf("not a cisco name mangled log!\n"); + DBGPRINTF("not a cisco name mangled log!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } /* bump the message portion up by two characters to overwrite the extra : */ diff --git a/contrib/pmpanngfw/Makefile.am b/contrib/pmpanngfw/Makefile.am new file mode 100644 index 000000000..50d75560c --- /dev/null +++ b/contrib/pmpanngfw/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmpanngfw.la + +pmpanngfw_la_SOURCES = pmpanngfw.c +pmpanngfw_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmpanngfw_la_LDFLAGS = -module -avoid-version +pmpanngfw_la_LIBADD = + +EXTRA_DIST = diff --git a/contrib/pmpanngfw/README.md b/contrib/pmpanngfw/README.md new file mode 100644 index 000000000..cc5b6a5cf --- /dev/null +++ b/contrib/pmpanngfw/README.md @@ -0,0 +1,87 @@ +# pmpanngfw + +Module to detect and transform PAN-OS NGFW logs into a format compatible with mmnormalize + +## How it works + +Original log: + + Apr 10 02:48:29 1,2012/04/10 02:48:29,001606001116,THREAT,url,1,2012/04/10 02:48:28,##IP##,##IP##,##IP##,##IP##,rule1,counselor,,facebook- base,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 02:48:28,27555,1,8450,80,0,0,0x208000,tcp,alert,"www.facebook. com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?__a=1&ajaxpipe=1&ajaxpipe_token=AXgw4BUd5yCuD2rQ&data={""profile_id"":603261604,""start"":1333263600,""end"":1335855599,""query_type"":5,""page_index"":1,""section_container_id"":""ucp7d6_26"",""section_pagelet_id"":""pagelet_timeline_recent"",""unit_container_id"":""ucp7d6_25"",""current_scrubber_key"":""recent"",""time_cutoff"":null,""buffer"":1300,""require_click"":false,""num_visible_units"":5,""remove_dupes"":true}&__user=857280013&__adt=3&__att=iframe",(9999),social-networking,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,text/html + +Transformed: + + Apr 10 02:48:29 12012/04/10 02:48:29001606001116THREATurl12012/04/10 02:48:28##IP####IP####IP####IP##rule1counselorfacebook-basevsys1trustuntrustethernet1/2ethernet1/1forwardAll 2012/04/10 02:48:28 27555184508000x208000tcp alertwww.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?__a=1&ajaxpipe=1&ajaxpipe_token=AXgw4BUd5yCuD2rQ&data={"profile_id":603261604,"start":1333263600,"end":1335855599,"query_type":5,"page_index":1,"section_container_id":"ucp7d6_26","section_pagelet_id":"pagelet_timeline_recent","unit_container_id":"ucp7d6_25","current_scrubber_key":"recent","time_cutoff":null,"buffer":1300,"require_click":false,"num_visible_units":5,"remove_dupes":true}&__user=857280013&__adt=3&__att=iframe(9999)social-networkinginformationalclient-to-server00x0192.168.0.0-192.168.255.255United States0text/html + +## How to compile + + $ autoreconf -fvi + $ ./configure --enable-pmpanngfw + $ cd contrib/pmpanngfw/ + $ make + +The resulting plugin should be found in contrib/pmpanngfw/.libs/ + +## Example config + + module(load="imtcp") + module(load="pmpanngfw") + module(load="mmnormalize") + module(load="omrabbitmq") + + template(name="csv" type="list") { + property(name="$!src_ip" format="csv") + constant(value=",") + property(name="$!dest_ip" format="csv") + constant(value=",") + property(name="$!url") + constant(value="\n") + } + + $template alljson,"%$!all-json%\n" + + template(name="mmeld_json" type="list") { + constant(value="{") + property(outname="@timestamp" name="timestamp" format="jsonf" dateFormat="rfc3339") + constant(value=",") + property(outname="@source_host" name="source" format="jsonf") + constant(value=",") + property(outname="@message" name="msg" format="jsonf") + constant(value=",") + property(outname="@timegenerated" name="timegenerated" format="jsonf" dateFormat="rfc3339") + constant(value=",") + property(outname="@timereported" name="timereported" format="jsonf" dateFormat="rfc3339") + constant(value=",") + property(outname="src_ip" name="$!src_ip" format="jsonf") + constant(value=",") + property(outname="dest_ip" name="$!dest_ip" format="jsonf") + constant(value=",") + property(outname="url" name="$!url" format="jsonf") + constant(value=",") + property(outname="tags" name="$!event.tags" format="jsonf") + constant(value="}") + constant(value="\n") + } + + ruleset(name="pan-ngfw" parser=["rsyslog.panngfw", "rsyslog.rfc5424", "rsyslog.rfc3164"]) { + action(type="mmnormalize" rulebase="palo_alto_networks.rb" userawmsg="on") + if strlen($!unparsed-data) == 0 then { + if $!log_subtype == "url" then set $!url = $!misc; + *.* action(type="omrabbitmq" + host="localhost" + virtual_host="/" + user="guest" + password="guest" + exchange="mmeld-syslog" + routing_key="" + exchange_type="fanout" + template="alljson") + } + *.* stop + } + + input(type="imtcp" port="13514" ruleset="pan-ngfw") + +## mmnormalize rulebase + +See https://gist.github.com/jtschichold/87f59b99d98c8eac1da5 + diff --git a/contrib/pmpanngfw/pmpanngfw.c b/contrib/pmpanngfw/pmpanngfw.c new file mode 100644 index 000000000..0972023c0 --- /dev/null +++ b/contrib/pmpanngfw/pmpanngfw.c @@ -0,0 +1,297 @@ +/* pmpanngfw.c + * + * this detects logs sent by Palo Alto Networks NGFW and transforms CSV into tab-separated fields + * for handling inside the mmnormalize + * + * Example: foo,"bar,""baz""",qux becomes foobar,"baz"qux + * + * created 2010-12-13 by Luigi Mori (lmori@paloaltonetworks.com) based on pmsnare + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" +#include "typedefs.h" + +MODULE_TYPE_PARSER +MODULE_TYPE_NOKEEP +PARSER_NAME("rsyslog.panngfw") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + +typedef struct { + uint64 value; + uint64 mask; +} log_type_t; + +const log_type_t log_types[] = { + { 0x002c544145524854ULL, 0x00FFFFFFFFFFFFFFULL }, /* THREAT, */ + { 0x2c43494646415254ULL, 0xFFFFFFFFFFFFFFFFULL }, /* TRAFFIC, */ + { 0x002c4d4554535953ULL, 0x00FFFFFFFFFFFFFFULL }, /* CONFIG */ + { 0x002c4749464e4f43ULL, 0x00FFFFFFFFFFFFFFULL } /* SYSTEM */ +}; + +#define NUM_LOG_TYPES (sizeof(log_types)/sizeof(log_type_t)) + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + + +BEGINparse + uchar *p2parse; + uchar *p2target; + uchar *msgend; + int lenMsg, lenDelta; + int state; + int num_fields = 4; + uchar *f3_commas[3]; + int cur_comma = 0; + uint64 log_type; + int j; +CODESTARTparse + #define CSV_DELIMITER '\t' + #define STATE_FIELD_START 0 + #define STATE_IN_FIELD 1 + #define STATE_IN_QUOTE 2 + #define STATE_IN_QUOTE_QUOTE 3 + + state = STATE_FIELD_START; + + dbgprintf("Message will now be parsed by fix Palo Alto Networks NGFW parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + msgend = p2parse+lenMsg; + + dbgprintf("pmpanngfw: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + + /* pass the first 3 fields */ + while(p2parse < msgend) { + if (*p2parse == ',') { + f3_commas[cur_comma] = p2parse; + if (cur_comma == 2) { + break; + } + cur_comma++; + } + p2parse++; + } + + /* check number of fields detected so far */ + if (cur_comma != 2) { + dbgprintf("not a PAN-OS syslog message: first 3 fields not found\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* check msg length */ + p2parse++; + if ((p2parse > msgend) || ((msgend - p2parse) < sizeof(uint64))) { + dbgprintf("not a PAN-OS syslog message: too short\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* check log type */ + log_type = *((uint64 *)p2parse); + for(j = 0; j < NUM_LOG_TYPES; j++) { + if ((log_type & log_types[j].mask) == log_types[j].value) + break; + } + if (j == NUM_LOG_TYPES) { + dbgprintf("not a PAN-OS syslog message, log type: %llx\n", log_type); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* set the delimiter */ + *f3_commas[0] = CSV_DELIMITER; + *f3_commas[1] = CSV_DELIMITER; + *f3_commas[2] = CSV_DELIMITER; + + p2target = p2parse; + + while(p2parse < msgend) { + /* dbgprintf("state: %d char: %c p2parse: %16x p2target: %16x\n", state, *p2parse, p2parse, p2target); */ + switch(state) { + case STATE_FIELD_START: + switch(*p2parse) { + case '"': + state = STATE_IN_QUOTE; + p2parse++; + break; + + case ',': + state = STATE_FIELD_START; + *p2target = CSV_DELIMITER; + num_fields++; + p2parse++; + p2target++; + break; + + default: + state = STATE_IN_FIELD; + *p2target = *p2parse; + p2parse++; + p2target++; + } + break; + + case STATE_IN_FIELD: + switch(*p2parse) { + case ',': + state = STATE_FIELD_START; + *p2target = CSV_DELIMITER; + num_fields++; + p2parse++; + p2target++; + break; + + default: + *p2target = *p2parse; + p2parse++; + p2target++; + } + break; + + case STATE_IN_QUOTE: + switch(*p2parse) { + case '"': + state = STATE_IN_QUOTE_QUOTE; + p2parse++; + break; + + default: + *p2target = *p2parse; + p2parse++; + p2target++; + } + break; + + case STATE_IN_QUOTE_QUOTE: + switch(*p2parse) { + case '"': + state = STATE_IN_QUOTE; + *p2target = *p2parse; + p2parse++; + p2target++; + break; + + case ',': + state = STATE_FIELD_START; + *p2target = CSV_DELIMITER; + num_fields++; + p2parse++; + p2target++; + break; + + default: + dbgprintf("pmpanngfw: martian char (%d) after quote in quote\n", *p2parse); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + break; + + default: + dbgprintf("how could I have reached this state ?!?\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + } + + if(p2parse != p2target) { + lenDelta = p2parse - p2target; + + assert(lenDelta >= 2); + + *p2target = 0; + + pMsg->iLenRawMsg -= lenDelta; + pMsg->iLenMSG -= lenDelta; + } + + lenMsg = p2target - (pMsg->pszRawMsg + pMsg->offAfterPRI); + + DBGPRINTF("pmpanngfw: new message: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + DBGPRINTF("pmpanngfw: # fields: %d\n", num_fields); + + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + DBGPRINTF("panngfw parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/grammar/lexer.l b/grammar/lexer.l index ac212131e..675766778 100644 --- a/grammar/lexer.l +++ b/grammar/lexer.l @@ -160,7 +160,7 @@ int fileno(FILE *stream); "startswith" { return CMP_STARTSWITH; } "startswith_i" { return CMP_STARTSWITHI; } 0[0-7]+ | /* octal number */ -0x[0-7a-f] | /* hex number, following rule is dec; strtoll handles all! */ +0x[0-9a-f]+ | /* hex number, following rule is dec; strtoll handles all! */ ([1-9][0-9]*|0) { yylval.n = strtoll(yytext, NULL, 0); return NUMBER; } \$[$!./]{0,1}[@a-z][!@a-z0-9\-_\.\[\]]* { yylval.s = strdup(yytext+1); return VAR; } \'([^'\\]|\\['"\\$bntr]|\\x[0-9a-f][0-9a-f]|\\[0-7][0-7][0-7])*\' { @@ -232,6 +232,8 @@ int fileno(FILE *stream); BEGIN INOBJ; return BEGINOBJ; } "lookup_table"[ \n\t]*"(" { yylval.objType = CNFOBJ_LOOKUP_TABLE; BEGIN INOBJ; return BEGINOBJ; } +"dyn_stats"[ \n\t]*"(" { yylval.objType = CNFOBJ_DYN_STATS; + BEGIN INOBJ; return BEGINOBJ; } "action"[ \n\t]*"(" { BEGIN INOBJ; return BEGIN_ACTION; } ^[ \t]*:\$?[a-z\-]+[ ]*,[ ]*!?[a-z]+[ ]*,[ ]*\"(\\\"|[^\"])*\" { yylval.s = strdup(rmLeadingSpace(yytext)); diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 70424d1ad..953a46647 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -112,7 +112,19 @@ tokenToString(const int token) case CMP_STARTSWITH: tokstr ="CMP_STARTSWITH"; break; case CMP_STARTSWITHI: tokstr ="CMP_STARTSWITHI"; break; case UMINUS: tokstr ="UMINUS"; break; - default: snprintf(tokbuf, sizeof(tokbuf), "%c[%d]", token, token); + case '&': tokstr ="&"; break; + case '+': tokstr ="+"; break; + case '-': tokstr ="-"; break; + case '*': tokstr ="*"; break; + case '/': tokstr ="/"; break; + case '%': tokstr ="%"; break; + case 'M': tokstr ="M"; break; + case 'N': tokstr ="N"; break; + case 'S': tokstr ="S"; break; + case 'V': tokstr ="V"; break; + case 'F': tokstr ="F"; break; + case 'A': tokstr ="A"; break; + default: snprintf(tokbuf, sizeof(tokbuf), "%c[%d]", token, token); tokstr = tokbuf; break; } return tokstr; @@ -768,7 +780,7 @@ doGetGID(struct nvlst *valnode, struct cnfparamdescr *param, } else { val->val.datatype = 'N'; val->val.d.n = resultBuf->gr_gid; - dbgprintf("param '%s': uid %d obtained for group '%s'\n", + DBGPRINTF("param '%s': uid %d obtained for group '%s'\n", param->name, (int) resultBuf->gr_gid, cstr); r = 1; } @@ -795,7 +807,7 @@ doGetUID(struct nvlst *valnode, struct cnfparamdescr *param, } else { val->val.datatype = 'N'; val->val.d.n = resultBuf->pw_uid; - dbgprintf("param '%s': uid %d obtained for user '%s'\n", + DBGPRINTF("param '%s': uid %d obtained for user '%s'\n", param->name, (int) resultBuf->pw_uid, cstr); r = 1; } @@ -1001,7 +1013,7 @@ nvlstGetParam(struct nvlst *valnode, struct cnfparamdescr *param, r = 1; /* this *is* valid! */ break; default: - dbgprintf("error: invalid param type\n"); + DBGPRINTF("error: invalid param type\n"); r = 0; break; } @@ -1026,7 +1038,7 @@ nvlstGetParams(struct nvlst *lst, struct cnfparamblk *params, struct cnfparamdescr *param; if(params->version != CNFPARAMBLK_VERSION) { - dbgprintf("nvlstGetParams: invalid param block version " + DBGPRINTF("nvlstGetParams: invalid param block version " "%d, expected %d\n", params->version, CNFPARAMBLK_VERSION); return NULL; @@ -1086,7 +1098,7 @@ cnfparamvalsIsSet(struct cnfparamblk *params, struct cnfparamvals *vals) if(vals == NULL) return 0; if(params->version != CNFPARAMBLK_VERSION) { - dbgprintf("nvlstGetParams: invalid param block version " + DBGPRINTF("nvlstGetParams: invalid param block version " "%d, expected %d\n", params->version, CNFPARAMBLK_VERSION); return 0; @@ -1105,6 +1117,9 @@ cnfparamsPrint(const struct cnfparamblk *params, const struct cnfparamvals *vals int i; char *cstr; + if(!Debug) + return; + for(i = 0 ; i < params->nParams ; ++i) { dbgprintf("%s: ", params->descr[i].name); if(vals[i].bUsed) { @@ -1188,16 +1203,44 @@ done: } -/* ensure that retval is a number; if string is no number, - * try to convert it to one. The semantics from es_str2num() - * are used (bSuccess tells if the conversion went well or not). +static inline int64_t +str2num(es_str_t *s, int *bSuccess) +{ + size_t i; + int neg; + int64_t num = 0; + const uchar *const c = es_getBufAddr(s); + + if(c[0] == '-') { + neg = -1; + i = -1; + } else { + neg = 1; + i = 0; + } + while(i < s->lenStr && isdigit(c[i])) { + num = num * 10 + c[i] - '0'; + ++i; + } + num *= neg; + if(bSuccess != NULL) + *bSuccess = (i == s->lenStr) ? 1 : 0; + return num; +} + +/* We support decimal integers. Unfortunately, previous versions + * said they support oct and hex, but that wasn't really the case. + * Everything based on JSON was just dec-converted. As this was/is + * the norm, we fix that inconsistency. Luckly, oct and hex support + * was never documented. + * rgerhards, 2015-11-12 */ static long long var2Number(struct var *r, int *bSuccess) { long long n = 0; if(r->datatype == 'S') { - n = es_str2num(r->d.estr, bSuccess); + n = str2num(r->d.estr, bSuccess); } else { if(r->datatype == 'J') { #ifdef HAVE_JSON_OBJECT_NEW_INT64 @@ -1262,15 +1305,15 @@ var2CString(struct var *__restrict__ const r, int *__restrict__ const bMustFree) int SKIP_NOTHING = 0x0; int SKIP_STRING = 0x1; -int SKIP_JSON = 0x2; static void varFreeMembersSelectively(const struct var *r, const int skipMask) { - int kill_string = ! (skipMask & SKIP_STRING); - if(kill_string && (r->datatype == 'S')) es_deleteStr(r->d.estr); - int kill_json = ! (skipMask & SKIP_JSON); - if(kill_json && (r->datatype == 'J')) json_object_put(r->d.json); + if(r->datatype == 'J') { + json_object_put(r->d.json); + } else if( !(skipMask & SKIP_STRING) && (r->datatype == 'S')) { + es_deleteStr(r->d.estr); + } } static void @@ -1303,7 +1346,7 @@ doExtractFieldByChar(uchar *str, uchar delim, const int matchnbr, uchar **resstr ++iCurrFld; } } - dbgprintf("field() field requested %d, field found %d\n", matchnbr, iCurrFld); + DBGPRINTF("field() field requested %d, field found %d\n", matchnbr, iCurrFld); if(iCurrFld == matchnbr) { /* field found, now extract it */ @@ -1315,12 +1358,12 @@ doExtractFieldByChar(uchar *str, uchar delim, const int matchnbr, uchar **resstr * step back a little not to copy it as part of the field. */ /* we got our end pointer, now do the copy */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - allocLen = iLen + 1; -# ifdef VALGRIND + allocLen = iLen + 1; +# ifdef VALGRIND allocLen += (3 - (iLen % 4)); - /*older versions of valgrind have a problem with strlen inspecting 4-bytes at a time*/ -# endif - CHKmalloc(pBuf = MALLOC(allocLen * sizeof(uchar))); + /*older versions of valgrind have a problem with strlen inspecting 4-bytes at a time*/ +# endif + CHKmalloc(pBuf = MALLOC(allocLen)); /* now copy */ memcpy(pBuf, pFld, iLen); pBuf[iLen] = '\0'; /* terminate it */ @@ -1355,7 +1398,7 @@ doExtractFieldByStr(uchar *str, char *delim, const rs_size_t lenDelim, const int ++iCurrFld; } } - dbgprintf("field() field requested %d, field found %d\n", matchnbr, iCurrFld); + DBGPRINTF("field() field requested %d, field found %d\n", matchnbr, iCurrFld); if(iCurrFld == matchnbr) { /* field found, now extract it */ @@ -1368,7 +1411,7 @@ doExtractFieldByStr(uchar *str, char *delim, const rs_size_t lenDelim, const int iLen = pFldEnd - pFld; } /* we got our end pointer, now do the copy */ - CHKmalloc(pBuf = MALLOC((iLen + 1) * sizeof(uchar))); + CHKmalloc(pBuf = MALLOC(iLen + 1)); /* now copy */ memcpy(pBuf, pFld, iLen); pBuf[iLen] = '\0'; /* terminate it */ @@ -1419,16 +1462,16 @@ doFunc_re_extract(struct cnffunc *func, struct var *ret, void* usrptr) int iREstat; iREstat = regexp.regexec(func->funcdata, (char*)(str + iOffs), submatchnbr+1, pmatch, 0); - dbgprintf("re_extract: regexec return is %d\n", iREstat); + DBGPRINTF("re_extract: regexec return is %d\n", iREstat); if(iREstat == 0) { if(pmatch[0].rm_so == -1) { - dbgprintf("oops ... start offset of successful regexec is -1\n"); + DBGPRINTF("oops ... start offset of successful regexec is -1\n"); break; } if(iTry == matchnbr) { bFound = 1; } else { - dbgprintf("re_extract: regex found at offset %d, new offset %d, tries %d\n", + DBGPRINTF("re_extract: regex found at offset %d, new offset %d, tries %d\n", iOffs, (int) (iOffs + pmatch[0].rm_eo), iTry); iOffs += pmatch[0].rm_eo; ++iTry; @@ -1437,7 +1480,7 @@ doFunc_re_extract(struct cnffunc *func, struct var *ret, void* usrptr) break; } } - dbgprintf("re_extract: regex: end search, found %d\n", bFound); + DBGPRINTF("re_extract: regex: end search, found %d\n", bFound); if(!bFound) { bHadNoMatch = 1; goto finalize_it; @@ -1590,12 +1633,18 @@ doRandomGen(struct var *__restrict__ const sourceVal) { int success = 0; long long max = var2Number(sourceVal, &success); if (! success) { - dbgprintf("rainerscript: random(max) didn't get a valid 'max' limit, defaulting random-number value to 0"); + DBGPRINTF("rainerscript: random(max) didn't get a valid 'max' limit, defaulting random-number value to 0"); + return 0; + } + if(max == 0) { + DBGPRINTF("rainerscript: random(max) invalid, 'max' is zero, , defaulting random-number value to 0"); return 0; } long int x = randomNumber(); if (max > MAX_RANDOM_NUMBER) { - dbgprintf("rainerscript: desired random-number range [0 - %lld) is wider than supported limit of [0 - %d)", max, MAX_RANDOM_NUMBER); + DBGPRINTF("rainerscript: desired random-number range [0 - %lld] " + "is wider than supported limit of [0 - %d)", + max, MAX_RANDOM_NUMBER); } return x % max; } @@ -1607,7 +1656,6 @@ static inline void doFuncCall(struct cnffunc *__restrict__ const func, struct var *__restrict__ const ret, void *__restrict__ const usrptr) { - char *fname; char *envvar; int bMustFree; es_str_t *estr; @@ -1623,7 +1671,7 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct var *__restrict__ con uint8_t lookup_key_type; lookup_t *lookup_table; - dbgprintf("rainerscript: executing function id %d\n", func->fID); + DBGPRINTF("rainerscript: executing function id %d\n", func->fID); switch(func->fID) { case CNFFUNC_STRLEN: if(func->expr[0]->nodetype == 'S') { @@ -1717,6 +1765,7 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct var *__restrict__ con varFreeMembers(&r[0]); } ret->datatype = 'N'; + DBGPRINTF("JSONorString: cnum node type %c result %d\n", func->expr[0]->nodetype, (int) ret->d.n); break; case CNFFUNC_RE_MATCH: cnfexprEval(func->expr[0], &r[0], usrptr); @@ -1783,7 +1832,6 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct var *__restrict__ con ret->datatype = 'N'; break; case CNFFUNC_LOOKUP: - dbgprintf("DDDD: executing lookup\n"); ret->datatype = 'S'; if(func->funcdata == NULL) { ret->d.estr = es_newStrFromCStr("TABLE-NOT-FOUND", sizeof("TABLE-NOT-FOUND")-1); @@ -1806,9 +1854,21 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct var *__restrict__ con } varFreeMembers(&r[1]); break; + case CNFFUNC_DYN_INC: + ret->datatype = 'N'; + if(func->funcdata == NULL) { + ret->d.n = -1; + break; + } + cnfexprEval(func->expr[1], &r[1], usrptr); + str = (char*) var2CString(&r[1], &bMustFree); + ret->d.n = dynstats_inc(func->funcdata, (uchar*)str); + if(bMustFree) free(str); + varFreeMembers(&r[1]); + break; default: if(Debug) { - fname = es_str2cstr(func->fname, NULL); + char *fname = es_str2cstr(func->fname, NULL); dbgprintf("rainerscript: invalid function id %u (name '%s')\n", (unsigned) func->fID, fname); free(fname); @@ -1827,21 +1887,31 @@ evalVar(struct cnfvar *__restrict__ const var, void *__restrict__ const usrptr, unsigned short bMustBeFreed = 0; rsRetVal localRet; struct json_object *json; + uchar *cstr; if(var->prop.id == PROP_CEE || var->prop.id == PROP_LOCAL_VAR || var->prop.id == PROP_GLOBAL_VAR ) { - localRet = msgGetJSONPropJSON((msg_t*)usrptr, &var->prop, &json); - ret->datatype = 'J'; - ret->d.json = (localRet == RS_RET_OK) ? json : NULL; - - DBGPRINTF("rainerscript: var %d:%s: '%s'\n", var->prop.id, var->prop.name, + localRet = msgGetJSONPropJSONorString((msg_t*)usrptr, &var->prop, &json, &cstr); + if(json != NULL) { + ret->datatype = 'J'; + ret->d.json = (localRet == RS_RET_OK) ? json : NULL; + DBGPRINTF("rainerscript: (json) var %d:%s: '%s'\n", + var->prop.id, var->prop.name, (ret->d.json == NULL) ? "" : json_object_get_string(ret->d.json)); + } else { /* we have a string */ + ret->datatype = 'S'; + ret->d.estr = (localRet == RS_RET_OK) ? + es_newStrFromCStr((char*) cstr, strlen((char*) cstr)) + : es_newStr(1); + DBGPRINTF("rainerscript: (json/string) var %d: '%s'\n", var->prop.id, cstr); + free(cstr); + } } else { ret->datatype = 'S'; pszProp = (uchar*) MsgGetProp((msg_t*)usrptr, NULL, &var->prop, &propLen, &bMustBeFreed, NULL); ret->d.estr = es_newStrFromCStr((char*)pszProp, propLen); - DBGPRINTF("rainerscript: var %d: '%s'\n", var->prop.id, pszProp); + DBGPRINTF("rainerscript: (string) var %d: '%s'\n", var->prop.id, pszProp); if(bMustBeFreed) free(pszProp); } @@ -2410,11 +2480,12 @@ cnfexprEval(const struct cnfexpr *__restrict__ const expr, struct var *__restric default: ret->datatype = 'N'; ret->d.n = 0ll; - dbgprintf("eval error: unknown nodetype %u['%c']\n", + DBGPRINTF("eval error: unknown nodetype %u['%c']\n", (unsigned) expr->nodetype, (char) expr->nodetype); break; } - DBGPRINTF("eval expr %p, return datatype '%c'\n", expr, ret->datatype); + DBGPRINTF("eval expr %p, return datatype '%c':%d\n", expr, ret->datatype, + (ret->datatype == 'N') ? (int)ret->d.n: 0); } //--------------------------------------------------------- @@ -2533,8 +2604,15 @@ struct json_object* cnfexprEvalCollection(struct cnfexpr *__restrict__ const expr, void *__restrict__ const usrptr) { struct var ret; + void *retptr; cnfexprEval(expr, &ret, usrptr); - return ret.d.json;/*caller is supposed to free the returned json-object*/ + if(ret.datatype == 'J') { + retptr = ret.d.json; /*caller is supposed to free the returned json-object*/ + } else { + retptr = NULL; + varFreeMembers(&ret); /* we must free the element */ + } + return retptr; } inline static void @@ -2969,7 +3047,7 @@ cnfstmtDestruct(struct cnfstmt *stmt) free(stmt->d.s_reload_lookup_table.stub_value); } default: - dbgprintf("error: unknown stmt type during destruct %u\n", + DBGPRINTF("error: unknown stmt type during destruct %u\n", (unsigned) stmt->nodetype); break; } @@ -3003,10 +3081,10 @@ cnfNewIterator(char *var, struct cnfexpr *collection) static void cnfIteratorDestruct(struct cnfitr *itr) { - if (itr->var != NULL) free(itr->var); - itr->var = NULL; - if (itr->collection != NULL) cnfexprDestruct(itr->collection); - itr->collection = NULL; + free(itr->var); + if(itr->collection != NULL) + cnfexprDestruct(itr->collection); + free(itr); } struct cnfstmt * @@ -3434,7 +3512,7 @@ cnfexprOptimize(struct cnfexpr *expr) long long ln, rn; struct cnfexpr *exprswap; - dbgprintf("optimize expr %p, type '%s'\n", expr, tokenToString(expr->nodetype)); + DBGPRINTF("optimize expr %p, type '%s'\n", expr, tokenToString(expr->nodetype)); switch(expr->nodetype) { case '&': constFoldConcat(expr); @@ -3748,7 +3826,7 @@ cnfstmtOptimize(struct cnfstmt *root) DBGPRINTF("optimizer error: we see a NOP, how come?\n"); break; default: - dbgprintf("error: unknown stmt type %u during optimizer run\n", + DBGPRINTF("error: unknown stmt type %u during optimizer run\n", (unsigned) stmt->nodetype); break; } @@ -3772,10 +3850,10 @@ cnffparamlstNew(struct cnfexpr *expr, struct cnffparamlst *next) static const char* const numInWords[] = {"zero", "one", "two", "three", "four", "five", "six"}; #define GENERATE_FUNC_WITH_NARG_RANGE(name, minArg, maxArg, funcId, errMsg) \ - if(nParams < minArg || nParams > maxArg) { \ - parser_errmsg(errMsg, name, nParams); \ - return CNFFUNC_INVALID; \ - } \ + if(nParams < minArg || nParams > maxArg) { \ + parser_errmsg(errMsg, name, nParams); \ + return CNFFUNC_INVALID; \ + } \ return funcId @@ -3824,6 +3902,8 @@ funcName2ID(es_str_t *fname, unsigned short nParams) GENERATE_FUNC("prifilt", 1, CNFFUNC_PRIFILT); } else if(FUNC_NAME("lookup")) { GENERATE_FUNC("lookup", 2, CNFFUNC_LOOKUP); + } else if(FUNC_NAME("dyn_inc")) { + GENERATE_FUNC("dyn_inc", 2, CNFFUNC_DYN_INC); } else if(FUNC_NAME("replace")) { GENERATE_FUNC_WITH_ERR_MSG( "replace", 3, CNFFUNC_REPLACE, @@ -3832,10 +3912,10 @@ funcName2ID(es_str_t *fname, unsigned short nParams) "but is %d."); } else if(FUNC_NAME("wrap")) { GENERATE_FUNC_WITH_NARG_RANGE("wrap", 2, 3, CNFFUNC_WRAP, - "number of parameters for %s() must either be " - "two (operand_string, wrapper) or" - "three (operand_string, wrapper, wrapper_escape_str)" - "but is %d."); + "number of parameters for %s() must either be " + "two (operand_string, wrapper) or" + "three (operand_string, wrapper, wrapper_escape_str)" + "but is %d."); } else if(FUNC_NAME("random")) { GENERATE_FUNC("random", 1, CNFFUNC_RANDOM); } else { @@ -3982,6 +4062,36 @@ finalize_it: RETiRet; } +static inline rsRetVal +initFunc_dyn_stats(struct cnffunc *func) +{ + uchar *cstr = NULL; + DEFiRet; + + func->destructable_funcdata = 0; + + if(func->nParams != 2) { + parser_errmsg("rsyslog logic error in line %d of file %s\n", + __LINE__, __FILE__); + FINALIZE; + } + + func->funcdata = NULL; + if(func->expr[0]->nodetype != 'S') { + parser_errmsg("dyn-stats bucket-name (param 1) of dyn-stats manipulating functions like dyn_inc must be a constant string"); + FINALIZE; + } + + cstr = (uchar*)es_str2cstr(((struct cnfstringval*) func->expr[0])->estr, NULL); + if((func->funcdata = dynstats_findBucket(cstr)) == NULL) { + parser_errmsg("dyn-stats bucket '%s' not found", cstr); + FINALIZE; + } + +finalize_it: + free(cstr); + RETiRet; +} struct cnffunc * cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) @@ -4027,6 +4137,9 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) case CNFFUNC_EXEC_TEMPLATE: initFunc_exec_template(func); break; + case CNFFUNC_DYN_INC: + initFunc_dyn_stats(func); + break; default:break; } } @@ -4127,13 +4240,13 @@ cnfDoInclude(char *name) } if(S_ISREG(fileInfo.st_mode)) { /* config file */ - dbgprintf("requested to include config file '%s'\n", cfgFile); + DBGPRINTF("requested to include config file '%s'\n", cfgFile); cnfSetLexFile(cfgFile); } else if(S_ISDIR(fileInfo.st_mode)) { /* config directory */ - dbgprintf("requested to include directory '%s'\n", cfgFile); + DBGPRINTF("requested to include directory '%s'\n", cfgFile); cnfDoInclude(cfgFile); } else { - dbgprintf("warning: unable to process IncludeConfig directive '%s'\n", cfgFile); + DBGPRINTF("warning: unable to process IncludeConfig directive '%s'\n", cfgFile); } } diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h index d6db353a0..1b21a1daa 100644 --- a/grammar/rainerscript.h +++ b/grammar/rainerscript.h @@ -46,6 +46,7 @@ enum cnfobjType { CNFOBJ_LOOKUP_TABLE, CNFOBJ_PARSER, CNFOBJ_TIMEZONE, + CNFOBJ_DYN_STATS, CNFOBJ_INVALID = 0 }; @@ -81,6 +82,8 @@ cnfobjType2str(enum cnfobjType ot) return "main_queue"; case CNFOBJ_LOOKUP_TABLE: return "lookup_table"; + case CNFOBJ_DYN_STATS: + return "dyn_stats"; break; default:return "error: invalid cnfobjType"; } @@ -278,7 +281,8 @@ enum cnffuncid { CNFFUNC_EXEC_TEMPLATE, CNFFUNC_REPLACE, CNFFUNC_WRAP, - CNFFUNC_RANDOM + CNFFUNC_RANDOM, + CNFFUNC_DYN_INC }; struct cnffunc { diff --git a/outchannel.c b/outchannel.c index ac2e08794..aa8d66a5b 100644 --- a/outchannel.c +++ b/outchannel.c @@ -208,7 +208,7 @@ struct outchannel *ochAddLine(char* pName, uchar** ppRestOfConfLine) return NULL; pOch->iLenName = strlen(pName); - pOch->pszName = (char*) MALLOC(sizeof(char) * (pOch->iLenName + 1)); + pOch->pszName = (char*) MALLOC(pOch->iLenName + 1); if(pOch->pszName == NULL) { dbgprintf("ochAddLine could not alloc memory for outchannel name!"); pOch->iLenName = 0; diff --git a/parse.c b/parse.c index 4b01c1f0d..c07e8bb9c 100644 --- a/parse.c +++ b/parse.c @@ -453,6 +453,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) /* mask bits follow, let's parse them! */ ++pThis->iCurrPos; /* eat slash */ if((iRet = parsInt(pThis, pBits)) != RS_RET_OK) { + free((*pIP)->addr.NetAddr); + free((*pIP)->addr.HostWildcard); free (pszIP); free (*pIP); FINALIZE; @@ -489,6 +491,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) /* mask bits follow, let's parse them! */ ++pThis->iCurrPos; /* eat slash */ if((iRet = parsInt(pThis, pBits)) != RS_RET_OK) { + free((*pIP)->addr.NetAddr); + free((*pIP)->addr.HostWildcard); free (pszIP); free (*pIP); FINALIZE; diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 9db590159..607d51e08 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -196,22 +196,17 @@ finalize_it: RETiRet; } - -/* actually submit a message to the rsyslog core +/* submit a generated numeric-suffix message to the rsyslog core */ static rsRetVal -doInjectMsg(int iNum, ratelimit_t *ratelimiter) +doInjectMsg(uchar *szMsg, ratelimit_t *ratelimiter) { - uchar szMsg[1024]; msg_t *pMsg; struct syslogTime stTime; time_t ttGenTime; DEFiRet; - snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), - "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); - - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); @@ -226,6 +221,19 @@ finalize_it: RETiRet; } +/* submit a generated numeric-suffix message to the rsyslog core + */ +static rsRetVal +doInjectNumericSuffixMsg(int iNum, ratelimit_t *ratelimiter) +{ + uchar szMsg[1024]; + DEFiRet; + snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum); + CHKiRet(doInjectMsg(szMsg, ratelimiter)); +finalize_it: + RETiRet; +} /* This function injects messages. Command format: * injectmsg @@ -235,29 +243,38 @@ static rsRetVal injectMsg(uchar *pszCmd, tcps_sess_t *pSess) { uchar wordBuf[1024]; - int iFrom; - int nMsgs; + int iFrom, nMsgs; + uchar *litteralMsg; int i; ratelimit_t *ratelimit = NULL; DEFiRet; - /* we do not check errors here! */ - getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); - iFrom = atoi((char*)wordBuf); - getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); - nMsgs = atoi((char*)wordBuf); + litteralMsg = NULL; + CHKiRet(ratelimitNew(&ratelimit, "imdiag", "injectmsg")); - - for(i = 0 ; i < nMsgs ; ++i) { - doInjectMsg(i + iFrom, ratelimit); + /* we do not check errors here! */ + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf), TO_LOWERCASE); + if (ustrcmp(UCHAR_CONSTANT("litteral"), wordBuf) == 0) { + /* user has provided content for a message */ + ++pszCmd; /* ignore following space */ + CHKiRet(doInjectMsg(pszCmd, ratelimit)); + nMsgs = 1; + } else { /* assume 2 args, (from_idx, to_idx) */ + iFrom = atoi((char*)wordBuf); + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf), TO_LOWERCASE); + nMsgs = atoi((char*)wordBuf); + for(i = 0 ; i < nMsgs ; ++i) { + CHKiRet(doInjectNumericSuffixMsg(i + iFrom, ratelimit)); + } } - CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); + DBGPRINTF("imdiag: %d messages injected\n", nMsgs); finalize_it: if(ratelimit != NULL) ratelimitDestruct(ratelimit); + free(litteralMsg); RETiRet; } @@ -335,12 +352,12 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) * WITHOUT a termination \0 char. So we need to convert it to one * before proceeding. */ - CHKmalloc(pszMsg = MALLOC(sizeof(uchar) * (iLenMsg + 1))); + CHKmalloc(pszMsg = MALLOC(iLenMsg + 1)); pToFree = pszMsg; memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; - getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf), TO_LOWERCASE); dbgprintf("imdiag received command '%s'\n", cmdBuf); if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 6ff986d6e..de71c0d99 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -12,11 +12,11 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -112,6 +112,7 @@ typedef struct lstn_s { regex_t end_preg; /* compiled version of startRegex */ uchar *prevLineSegment; /* previous line segment (in regex mode) */ sbool escapeLF; /* escape LF inside the MSG content? */ + sbool reopenOnTruncate; sbool addMetadata; ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ ratelimit_t *ratelimiter; @@ -146,6 +147,7 @@ struct instanceConf_s { uint8_t readMode; uchar *startRegex; sbool escapeLF; + sbool reopenOnTruncate; sbool addMetadata; int maxLinesAtOnce; ruleset_t *pBindRuleset; /* ruleset to bind listener to (use system default if unspecified) */ @@ -174,7 +176,7 @@ struct modConfData_s { static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current load process */ -#if HAVE_INOTIFY_INIT +#if HAVE_INOTIFY_INIT /* support for inotify mode */ /* we need to track directories */ @@ -259,6 +261,7 @@ static struct cnfparamdescr inppdescr[] = { { "readmode", eCmdHdlrInt, 0 }, { "startmsg.regex", eCmdHdlrString, 0 }, { "escapelf", eCmdHdlrBinary, 0 }, + { "reopenontruncate", eCmdHdlrBinary, 0 }, { "maxlinesatonce", eCmdHdlrInt, 0 }, { "maxsubmitatonce", eCmdHdlrInt, 0 }, { "removestateondelete", eCmdHdlrBinary, 0 }, @@ -284,9 +287,9 @@ static void dbg_wdmapPrint(char *msg) { int i; - dbgprintf("%s\n", msg); + DBGPRINTF("%s\n", msg); for(i = 0 ; i < nWdmap ; ++i) - dbgprintf("wdmap[%d]: wd: %d, file %d, dir %d\n", i, + DBGPRINTF("wdmap[%d]: wd: %d, file %d, dir %d\n", i, wdmap[i].wd, wdmap[i].fIdx, wdmap[i].dirIdx); } #endif @@ -306,18 +309,18 @@ finalize_it: /* looks up a wdmap entry by dirIdx and returns it's index if found * or -1 if not found. */ -static int +static int wdmapLookupListner(lstn_t* pLstn) { - int i = 0; - int wd = -1; + int i = 0; + int wd = -1; /* Loop through */ for(i = 0 ; i < nWdmap; ++i) { if (wdmap[i].pLstn == pLstn) - wd = wdmap[i].wd; + wd = wdmap[i].wd; } - return wd; + return wd; } /* compare function for bsearch() */ @@ -368,14 +371,13 @@ wdmapAdd(int wd, const int dirIdx, lstn_t *const pLstn) } if(i < nWdmap) { /* we need to shift to make room for new entry */ - dbgprintf("DDDD: imfile doing wdmap mmemmov(%d, %d, %d) for ADD\n", i,i+1,nWdmap-i); - memmove(wdmap + i, wdmap + i + 1, nWdmap - i); + memmove(wdmap + i + 1, wdmap + i, sizeof(wd_map_t) * (nWdmap - i)); } wdmap[i].wd = wd; wdmap[i].dirIdx = dirIdx; wdmap[i].pLstn = pLstn; ++nWdmap; - dbgprintf("DDDD: imfile: enter into wdmap[%d]: wd %d, dir %d, lstn %s:%s\n",i,wd,dirIdx, + DBGPRINTF("imfile: enter into wdmap[%d]: wd %d, dir %d, lstn %s:%s\n",i,wd,dirIdx, (pLstn == NULL) ? "DIRECTORY" : "FILE", (pLstn == NULL) ? dirs[dirIdx].dirName : pLstn->pszFileName); @@ -398,11 +400,10 @@ wdmapDel(const int wd) if(i < nWdmap-1) { /* we need to shift to delete it (see comment at wdmap definition) */ - dbgprintf("DDDD: imfile doing wdmap mmemmov(%d, %d, %d) for DEL\n", i,i+1,nWdmap-i-1); - memmove(wdmap + i, wdmap + i+1, nWdmap - i-1); + memmove(wdmap + i, wdmap + i + 1, sizeof(wd_map_t) * (nWdmap - i - 1)); } --nWdmap; - dbgprintf("DDDD: imfile: wd %d deleted, was idx %d\n", wd, i); + DBGPRINTF("imfile: wd %d deleted, was idx %d\n", wd, i); finalize_it: RETiRet; @@ -455,7 +456,7 @@ static rsRetVal enqLine(lstn_t *const __restrict__ pLstn, CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, pInputName); - MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); + MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStrNoNULL(cstrLine), cstrLen(cstrLine)); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pLstn->pszTag, pLstn->lenTag); @@ -486,18 +487,18 @@ openFile(lstn_t *pLstn) DBGPRINTF("imfile: trying to open state for '%s', state file '%s'\n", pLstn->pszFileName, statefn); /* Construct file name */ - lenSFNam = snprintf((char*)pszSFNam, sizeof(pszSFNam) / sizeof(uchar), "%s/%s", + lenSFNam = snprintf((char*)pszSFNam, sizeof(pszSFNam), "%s/%s", (char*) glbl.GetWorkDir(), (char*)statefn); /* check if the file exists */ if(stat((char*) pszSFNam, &stat_buf) == -1) { if(errno == ENOENT) { - dbgprintf("imfile: clean startup, state file for '%s'\n", pLstn->pszFileName); + DBGPRINTF("imfile: clean startup, state file for '%s'\n", pLstn->pszFileName); ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); } else { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - dbgprintf("imfile: error trying to access state file for '%s':%s\n", + DBGPRINTF("imfile: error trying to access state file for '%s':%s\n", pLstn->pszFileName, errStr); ABORT_FINALIZE(RS_RET_IO_ERROR); } @@ -513,6 +514,7 @@ openFile(lstn_t *pLstn) /* read back in the object */ CHKiRet(obj.Deserialize(&pLstn->pStrm, (uchar*) "strm", psSF, NULL, pLstn)); + CHKiRet(strm.SetbReopenOnTruncate(pLstn->pStrm, pLstn->reopenOnTruncate)); DBGPRINTF("imfile: deserialized state file, state file base name '%s', " "configured base name '%s'\n", pLstn->pStrm->pszFName, pLstn->pszFileName); @@ -639,6 +641,7 @@ createInstance(instanceConf_t **pinst) inst->startRegex = NULL; inst->bRMStateOnDel = 1; inst->escapeLF = 1; + inst->reopenOnTruncate = 0; inst->addMetadata = ADD_METADATA_UNSPECIFIED; /* node created, let's add to config */ @@ -693,6 +696,12 @@ checkInstance(instanceConf_t *inst) char errStr[512]; DEFiRet; + /* this is primarily for the clang static analyzer, but also + * guards against logic errors in the config handler. + */ + if(inst->pszFileName == NULL) + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + i = getBasename(basen, inst->pszFileName); memcpy(dirn, inst->pszFileName, i); /* do not copy slash */ dirn[i] = '\0'; @@ -761,6 +770,7 @@ addInstance(void __attribute__((unused)) *pVal, uchar *pNewVal) inst->iPersistStateInterval = cs.iPersistStateInterval; inst->readMode = cs.readMode; inst->escapeLF = 0; + inst->reopenOnTruncate = 0; inst->addMetadata = 0; inst->bRMStateOnDel = 0; @@ -785,7 +795,7 @@ lstnAdd(lstn_t **newLstn) { lstn_t *pLstn; DEFiRet; - + CHKmalloc(pLstn = (lstn_t*) MALLOC(sizeof(lstn_t))); if(runModConf->pRootLstn == NULL) { runModConf->pRootLstn = pLstn; @@ -797,7 +807,7 @@ lstnAdd(lstn_t **newLstn) runModConf->pTailLstn = pLstn; pLstn->next = NULL; *newLstn = pLstn; - + finalize_it: RETiRet; } @@ -806,7 +816,7 @@ finalize_it: static void lstnDel(lstn_t *pLstn) { - dbgprintf("imfile: lstnDel called for %s\n", pLstn->pszFileName); + DBGPRINTF("imfile: lstnDel called for %s\n", pLstn->pszFileName); if(pLstn->pStrm != NULL) { /* stream open? */ persistStrmState(pLstn); strm.Destruct(&(pLstn->pStrm)); @@ -836,7 +846,7 @@ lstnDel(lstn_t *pLstn) * the ppExisting parameter. */ static rsRetVal -lstnDup(lstn_t ** ppExisting, uchar *const __restrict__ newname) +lstnDup(lstn_t **ppExisting, uchar *const __restrict__ newname) { DEFiRet; lstn_t *const existing = *ppExisting; @@ -845,7 +855,10 @@ lstnDup(lstn_t ** ppExisting, uchar *const __restrict__ newname) CHKiRet(lstnAdd(&pThis)); pThis->pszDirName = existing->pszDirName; /* read-only */ pThis->pszBaseName = (uchar*)strdup((char*)newname); - asprintf((char**)&pThis->pszFileName, "%s/%s", (char*)pThis->pszDirName, (char*)newname); + if(asprintf((char**)&pThis->pszFileName, "%s/%s", (char*)pThis->pszDirName, (char*)newname) == -1) { + DBGPRINTF("imfile/lstnDup: asprintf failed, malfunction can happen\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } pThis->pszTag = (uchar*) strdup((char*) existing->pszTag); pThis->lenTag = ustrlen(pThis->pszTag); pThis->pszStateFile = existing->pszStateFile == NULL ? NULL : (uchar*) strdup((char*) existing->pszStateFile); @@ -862,12 +875,13 @@ lstnDup(lstn_t ** ppExisting, uchar *const __restrict__ newname) pThis->startRegex = existing->startRegex; /* no strdup, as it is read-only */ if(pThis->startRegex != NULL) // TODO: make this a single function with better error handling if(regcomp(&pThis->end_preg, (char*)pThis->startRegex, REG_EXTENDED)) { - dbgprintf("imfile: error regex compile\n"); + DBGPRINTF("imfile: error regex compile\n"); ABORT_FINALIZE(RS_RET_ERR); } pThis->bRMStateOnDel = existing->bRMStateOnDel; pThis->hasWildcard = existing->hasWildcard; pThis->escapeLF = existing->escapeLF; + pThis->reopenOnTruncate = existing->reopenOnTruncate; pThis->addMetadata = existing->addMetadata; pThis->pRuleset = existing->pRuleset; pThis->nRecords = 0; @@ -928,11 +942,12 @@ addListner(instanceConf_t *inst) pThis->startRegex = inst->startRegex; /* no strdup, as it is read-only */ if(pThis->startRegex != NULL) if(regcomp(&pThis->end_preg, (char*)pThis->startRegex, REG_EXTENDED)) { - dbgprintf("imfile: error regex compile\n"); + DBGPRINTF("imfile: error regex compile\n"); ABORT_FINALIZE(RS_RET_ERR); } pThis->bRMStateOnDel = inst->bRMStateOnDel; pThis->escapeLF = inst->escapeLF; + pThis->reopenOnTruncate = inst->reopenOnTruncate; pThis->addMetadata = (inst->addMetadata == ADD_METADATA_UNSPECIFIED) ? hasWildcard : inst->addMetadata; pThis->pRuleset = inst->pBindRuleset; @@ -958,7 +973,7 @@ CODESTARTnewInpInst } if(Debug) { - dbgprintf("input param blk in imfile:\n"); + DBGPRINTF("input param blk in imfile:\n"); cnfparamsPrint(&inppblk, pvals); } @@ -991,6 +1006,8 @@ CODESTARTnewInpInst inst->addMetadata = (sbool) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "escapelf")) { inst->escapeLF = (sbool) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "reopenontruncate")) { + inst->reopenOnTruncate = (sbool) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "maxlinesatonce")) { if( loadModConf->opMode == OPMODE_INOTIFY && pvals[i].val.d.n > 0) { @@ -1005,7 +1022,7 @@ CODESTARTnewInpInst } else if(!strcmp(inppblk.descr[i].name, "maxsubmitatonce")) { inst->nMultiSub = pvals[i].val.d.n; } else { - dbgprintf("imfile: program error, non-handled " + DBGPRINTF("imfile: program error, non-handled " "param '%s'\n", inppblk.descr[i].name); } } @@ -1056,7 +1073,7 @@ CODESTARTsetModCnf } if(Debug) { - dbgprintf("module (global) param blk for imfile:\n"); + DBGPRINTF("module (global) param blk for imfile:\n"); cnfparamsPrint(&modpblk, pvals); } @@ -1077,7 +1094,7 @@ CODESTARTsetModCnf free(cstr); } } else { - dbgprintf("imfile: program error, non-handled " + DBGPRINTF("imfile: program error, non-handled " "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); } } @@ -1100,7 +1117,7 @@ CODESTARTendCnfLoad /* persist module-specific settings from legacy config system */ loadModConf->iPollInterval = cs.iPollInterval; } - dbgprintf("imfile: opmode is %d, polling interval is %d\n", + DBGPRINTF("imfile: opmode is %d, polling interval is %d\n", loadModConf->opMode, loadModConf->iPollInterval); @@ -1182,7 +1199,7 @@ ENDfreeCnf * So even if we found some lines, it is highly unlikely to find a new one * just now. Trying it would result in a performance-costly additional try * which in the very, very vast majority of cases will never find any new - * lines. + * lines. * On spamming the main queue: keep in mind that it will automatically rate-limit * ourselfes if we begin to overrun it. So we really do not need to care here. */ @@ -1211,7 +1228,7 @@ doPolling(void) if(glbl.GetGlobalInputTermState() == 0) srSleep(runModConf->iPollInterval, 10); } - + RETiRet; } @@ -1233,10 +1250,10 @@ fileTableDisplay(fileTable_t *tab) { int f; uchar *baseName; - dbgprintf("DDDD: imfile: dirs.currMaxfiles %d\n", tab->currMax); + DBGPRINTF("imfile: dirs.currMaxfiles %d\n", tab->currMax); for(f = 0 ; f < tab->currMax ; ++f) { baseName = tab->listeners[f].pLstn->pszBaseName; - dbgprintf("DDDD: imfile: TABLE %p CONTENTS, %d->%p:'%s'\n", tab, f, tab->listeners[f].pLstn, (char*)baseName); + DBGPRINTF("imfile: TABLE %p CONTENTS, %d->%p:'%s'\n", tab, f, tab->listeners[f].pLstn, (char*)baseName); } } */ @@ -1246,7 +1263,7 @@ fileTableSearch(fileTable_t *const __restrict__ tab, uchar *const __restrict__ f { int f; uchar *baseName = NULL; -/* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ + /* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ for(f = 0 ; f < tab->currMax ; ++f) { baseName = tab->listeners[f].pLstn->pszBaseName; if(!fnmatch((char*)baseName, (char*)fn, FNM_PATHNAME | FNM_PERIOD)) @@ -1254,7 +1271,7 @@ fileTableSearch(fileTable_t *const __restrict__ tab, uchar *const __restrict__ f } if(f == tab->currMax) f = -1; - dbgprintf("DDDD: imfile: fileTableSearch file '%s' - '%s', found:%d\n", fn, baseName, f); + DBGPRINTF("imfile: fileTableSearch file '%s' - '%s', found:%d\n", fn, baseName, f); return f; } @@ -1263,7 +1280,7 @@ fileTableSearchNoWildcard(fileTable_t *const __restrict__ tab, uchar *const __re { int f; uchar *baseName = NULL; -/* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ + /* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ for(f = 0 ; f < tab->currMax ; ++f) { baseName = tab->listeners[f].pLstn->pszBaseName; if (strcmp((const char*)baseName, (const char*)fn) == 0) @@ -1271,18 +1288,17 @@ fileTableSearchNoWildcard(fileTable_t *const __restrict__ tab, uchar *const __re } if(f == tab->currMax) f = -1; - dbgprintf("DDDD: imfile: fileTableSearchNoWildcard file '%s' - '%s', found:%d\n", fn, baseName, f); + DBGPRINTF("imfile: fileTableSearchNoWildcard file '%s' - '%s', found:%d\n", fn, baseName, f); return f; } -/* add file to file table */ +/* add file to file table */ static rsRetVal fileTableAddFile(fileTable_t *const __restrict__ tab, lstn_t *const __restrict__ pLstn) { int j; DEFiRet; -dbgprintf("DDDDD: imfile: fileTableAddFile\n"); -/* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ + /* UNCOMMENT FOR DEBUG fileTableDisplay(tab); */ for(j = 0 ; j < tab->currMax && tab->listeners[j].pLstn != pLstn ; ++j) ; /* just scan */ if(j < tab->currMax) { @@ -1354,6 +1370,7 @@ dirsAdd(uchar *dirName) errmsg.LogError(0, RS_RET_OUT_OF_MEMORY, "cannot alloc memory to monitor directory '%s' - ignoring", dirName); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } dirs = newDirTab; allocMaxDirs = newMax; @@ -1366,7 +1383,7 @@ dirsAdd(uchar *dirName) CHKiRet(fileTableInit(&dirs[currMaxDirs].configured, INIT_FILE_IN_DIR_TAB_SIZE)); ++currMaxDirs; - dbgprintf("DDDD: imfile: added to dirs table: '%s'\n", dirName); + DBGPRINTF("imfile: added to dirs table: '%s'\n", dirName); finalize_it: RETiRet; } @@ -1384,7 +1401,6 @@ dirsFindDir(uchar *dir) ; /* just scan, all done in for() */ if(i == currMaxDirs) i = -1; - //dbgprintf("DDDD: dir '%s', found:%d\n", dir, i); return i; } @@ -1429,9 +1445,9 @@ dirsAddFile(lstn_t *__restrict__ pLstn, const int bActive) dir = dirs + dirIdx; CHKiRet(fileTableAddFile((bActive ? &dir->active : &dir->configured), pLstn)); - dbgprintf("DDDD: imfile: associated file [%s] to directory %d[%s], Active = %d\n", + DBGPRINTF("imfile: associated file [%s] to directory %d[%s], Active = %d\n", pLstn->pszFileName, dirIdx, dir->dirName, bActive); -/* UNCOMMENT FOR DEBUG fileTableDisplay(bActive ? &dir->active : &dir->configured); */ + /* UNCOMMENT FOR DEBUG fileTableDisplay(bActive ? &dir->active : &dir->configured); */ finalize_it: RETiRet; } @@ -1448,7 +1464,7 @@ in_setupDirWatch(const int dirIdx) goto done; } wdmapAdd(wd, dirIdx, NULL); - dbgprintf("DDDD: imfile: watch %d added for dir %s\n", wd, dirs[dirIdx].dirName); + DBGPRINTF("imfile: watch %d added for dir %s\n", wd, dirs[dirIdx].dirName); done: return; } @@ -1473,7 +1489,7 @@ startLstnFile(lstn_t *const __restrict__ pLstn) goto done; } wdmapAdd(wd, -1, pLstn); - dbgprintf("DDDD: imfile: watch %d added for file %s\n", wd, pLstn->pszFileName); + DBGPRINTF("imfile: watch %d added for file %s\n", wd, pLstn->pszFileName); dirsAddFile(pLstn, ACTIVE_FILE); pollFile(pLstn, NULL); done: return; @@ -1493,7 +1509,7 @@ in_setupFileWatchDynamic(lstn_t *pLstn, uchar *const __restrict__ newBaseName) if(stat(fullfn, &fileInfo) != 0) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - dbgprintf("imfile: ignoring file '%s' cannot stat(): %s\n", + DBGPRINTF("imfile: ignoring file '%s' cannot stat(): %s\n", fullfn, errStr); goto done; } @@ -1516,7 +1532,7 @@ done: return; * happen only for things after the watch has been activated. */ static void -in_setupFileWatchStatic(lstn_t *const __restrict__ pLstn) +in_setupFileWatchStatic(lstn_t *pLstn) { DBGPRINTF("imfile: adding file '%s' to configured table\n", pLstn->pszFileName); @@ -1570,72 +1586,64 @@ static void in_dbg_showEv(struct inotify_event *ev) { if(ev->mask & IN_IGNORED) { - dbgprintf("watch was REMOVED\n"); + DBGPRINTF("watch was REMOVED\n"); } else if(ev->mask & IN_MODIFY) { - dbgprintf("watch was MODIFID\n"); + DBGPRINTF("watch was MODIFID\n"); } else if(ev->mask & IN_ACCESS) { - dbgprintf("watch IN_ACCESS\n"); + DBGPRINTF("watch IN_ACCESS\n"); } else if(ev->mask & IN_ATTRIB) { - dbgprintf("watch IN_ATTRIB\n"); + DBGPRINTF("watch IN_ATTRIB\n"); } else if(ev->mask & IN_CLOSE_WRITE) { - dbgprintf("watch IN_CLOSE_WRITE\n"); + DBGPRINTF("watch IN_CLOSE_WRITE\n"); } else if(ev->mask & IN_CLOSE_NOWRITE) { - dbgprintf("watch IN_CLOSE_NOWRITE\n"); + DBGPRINTF("watch IN_CLOSE_NOWRITE\n"); } else if(ev->mask & IN_CREATE) { - dbgprintf("file was CREATED: %s\n", ev->name); + DBGPRINTF("file was CREATED: %s\n", ev->name); } else if(ev->mask & IN_DELETE) { - dbgprintf("watch IN_DELETE\n"); + DBGPRINTF("watch IN_DELETE\n"); } else if(ev->mask & IN_DELETE_SELF) { - dbgprintf("watch IN_DELETE_SELF\n"); + DBGPRINTF("watch IN_DELETE_SELF\n"); } else if(ev->mask & IN_MOVE_SELF) { - dbgprintf("watch IN_MOVE_SELF\n"); + DBGPRINTF("watch IN_MOVE_SELF\n"); } else if(ev->mask & IN_MOVED_FROM) { - dbgprintf("watch IN_MOVED_FROM\n"); + DBGPRINTF("watch IN_MOVED_FROM\n"); } else if(ev->mask & IN_MOVED_TO) { - dbgprintf("watch IN_MOVED_TO\n"); + DBGPRINTF("watch IN_MOVED_TO\n"); } else if(ev->mask & IN_OPEN) { - dbgprintf("watch IN_OPEN\n"); + DBGPRINTF("watch IN_OPEN\n"); } else if(ev->mask & IN_ISDIR) { - dbgprintf("watch IN_ISDIR\n"); + DBGPRINTF("watch IN_ISDIR\n"); } else { - dbgprintf("unknown mask code %8.8x\n", ev->mask); + DBGPRINTF("unknown mask code %8.8x\n", ev->mask); } } -static void -filesDisplay(void) -{ - lstn_t *pLstn; - for(pLstn = runModConf->pRootLstn ; pLstn != NULL ; pLstn = pLstn->next) - dbgprintf("DDDD: imfile: files: [%p]: '%s'\n", pLstn, pLstn->pszFileName); -} /* inotify told us that a file's wd was closed. We now need to remove * the file from our internal structures. Remember that a different inode * with the same name may already be in processing. */ static void -in_removeFile(const struct inotify_event *const ev, - const int dirIdx, +in_removeFile(const int dirIdx, lstn_t *const __restrict__ pLstn) { -filesDisplay(); // TODO: remove after initial unstable release(s) uchar statefile[MAXFNAME]; uchar toDel[MAXFNAME]; int bDoRMState; + int wd; uchar *statefn; - DBGPRINTF("imfile: remove listener '%s', wd %d\n", - pLstn->pszFileName, ev->wd); + DBGPRINTF("imfile: remove listener '%s', dirIdx %d\n", + pLstn->pszFileName, dirIdx); if(pLstn->bRMStateOnDel) { statefn = getStateFileName(pLstn, statefile, sizeof(statefile)); - snprintf((char*)toDel, sizeof(toDel) / sizeof(uchar), "%s/%s", + snprintf((char*)toDel, sizeof(toDel), "%s/%s", glbl.GetWorkDir(), (char*)statefn); bDoRMState = 1; } else { bDoRMState = 0; } pollFile(pLstn, NULL); /* one final try to gather data */ - /* delete listener data */ + /* delete listener data */ DBGPRINTF("imfile: DELETING listener data for '%s' - '%s'\n", pLstn->pszBaseName, pLstn->pszFileName); lstnDel(pLstn); fileTableDelFile(&dirs[dirIdx].active, pLstn); @@ -1648,6 +1656,8 @@ filesDisplay(); // TODO: remove after initial unstable release(s) "file \"%s\": %s", toDel, errStr); } } + wd = wdmapLookupListner(pLstn); + wdmapDel(wd); } static void @@ -1659,17 +1669,17 @@ in_handleDirEventCREATE(struct inotify_event *ev, const int dirIdx) if(ftIdx >= 0) { pLstn = dirs[dirIdx].active.listeners[ftIdx].pLstn; } else { - dbgprintf("imfile: file '%s' not active in dir '%s'\n", + DBGPRINTF("imfile: file '%s' not active in dir '%s'\n", ev->name, dirs[dirIdx].dirName); ftIdx = fileTableSearch(&dirs[dirIdx].configured, (uchar*)ev->name); if(ftIdx == -1) { - dbgprintf("imfile: file '%s' not associated with dir '%s'\n", + DBGPRINTF("imfile: file '%s' not associated with dir '%s'\n", ev->name, dirs[dirIdx].dirName); goto done; } pLstn = dirs[dirIdx].configured.listeners[ftIdx].pLstn; } - dbgprintf("DDDD: imfile: file '%s' associated with dir '%s'\n", ev->name, dirs[dirIdx].dirName); + DBGPRINTF("imfile: file '%s' associated with dir '%s'\n", ev->name, dirs[dirIdx].dirName); in_setupFileWatchDynamic(pLstn, (uchar*)ev->name); done: return; } @@ -1685,20 +1695,20 @@ in_handleDirEventDELETE(struct inotify_event *const ev, const int dirIdx) { const int ftIdx = fileTableSearch(&dirs[dirIdx].active, (uchar*)ev->name); if(ftIdx == -1) { - dbgprintf("imfile: deleted file '%s' not active in dir '%s'\n", + DBGPRINTF("imfile: deleted file '%s' not active in dir '%s'\n", ev->name, dirs[dirIdx].dirName); goto done; } - dbgprintf("DDDD: imfile: imfile delete processing for '%s'\n", + DBGPRINTF("imfile: imfile delete processing for '%s'\n", dirs[dirIdx].active.listeners[ftIdx].pLstn->pszFileName); - in_removeFile(ev, dirIdx, dirs[dirIdx].active.listeners[ftIdx].pLstn); + in_removeFile(dirIdx, dirs[dirIdx].active.listeners[ftIdx].pLstn); done: return; } static void in_handleDirEvent(struct inotify_event *const ev, const int dirIdx) { - dbgprintf("DDDD: imfile: handle dir event for %s\n", dirs[dirIdx].dirName); + DBGPRINTF("imfile: handle dir event for %s\n", dirs[dirIdx].dirName); if((ev->mask & IN_CREATE)) { in_handleDirEventCREATE(ev, dirIdx); } else if((ev->mask & IN_DELETE)) { @@ -1727,40 +1737,30 @@ in_processEvent(struct inotify_event *ev) wd_map_t *etry; lstn_t *pLstn; int iRet; - struct inotify_event evFileHelper; int ftIdx; - int wd; + int wd; - DBGPRINTF("DDDD: imfile: in_processEvent (wd=%d) event Mask='0x%.8X'\n", ev->wd, ev->mask); - if (ev->mask & IN_IGNORED) { - wdmapDel(ev->wd); + if(ev->mask & IN_IGNORED) { goto done; - } else if (ev->mask & IN_MOVED_FROM) { + } else if(ev->mask & IN_MOVED_FROM) { /* Find wd entry and remove it */ etry = wdmapLookup(ev->wd); if(etry != NULL) { ftIdx = fileTableSearchNoWildcard(&dirs[etry->dirIdx].active, (uchar*)ev->name); - DBGPRINTF("DDDD: imfile: IN_MOVED_FROM Event (ftIdx=%d, name=%s)\n", ftIdx, ev->name); + DBGPRINTF("imfile: IN_MOVED_FROM Event (ftIdx=%d, name=%s)\n", ftIdx, ev->name); if(ftIdx >= 0) { /* Find listener and wd table index*/ - pLstn = dirs[etry->dirIdx].active.listeners[ftIdx].pLstn; - wd = wdmapLookupListner(pLstn); + pLstn = dirs[etry->dirIdx].active.listeners[ftIdx].pLstn; + wd = wdmapLookupListner(pLstn); /* Remove file from inotify watch */ iRet = inotify_rm_watch(ino_fd, wd); /* Note this will TRIGGER IN_IGNORED Event! */ if (iRet != 0) { DBGPRINTF("imfile: inotify_rm_watch error %d (ftIdx=%d, wd=%d, name=%s)\n", errno, ftIdx, wd, ev->name); } else { - DBGPRINTF("DDDD: imfile: inotify_rm_watch successfully removed file from watch (ftIdx=%d, wd=%d, name=%s)\n", ftIdx, wd, ev->name); + DBGPRINTF("imfile: inotify_rm_watch successfully removed file from watch (ftIdx=%d, wd=%d, name=%s)\n", ftIdx, wd, ev->name); } - - /* Create Event to remove file*/ - evFileHelper.wd = wd; - evFileHelper.mask = IN_DELETE; - evFileHelper.cookie = 0; - evFileHelper.len = ev->len; - evFileHelper.name[0] = ev->name[0]; - in_removeFile(&evFileHelper, etry->dirIdx, pLstn); + in_removeFile(etry->dirIdx, pLstn); DBGPRINTF("imfile: IN_MOVED_FROM Event file removed file (wd=%d, name=%s)\n", wd, ev->name); } } @@ -1771,7 +1771,6 @@ in_processEvent(struct inotify_event *ev) DBGPRINTF("imfile: could not lookup wd %d\n", ev->wd); goto done; } - dbgprintf("DDDD: imfile: wd %d got file %p, dir %d\n", ev->wd, etry->pLstn, etry->dirIdx); if(etry->pLstn == NULL) { /* directory? */ in_handleDirEvent(ev, etry->dirIdx); } else { @@ -1781,6 +1780,9 @@ done: return; } /* Monitor files in inotify mode */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-align" /* TODO: how can we fix these warnings? */ +/* Problem with the warnings: they seem to stem back from the way the API is structured */ static rsRetVal do_inotify() { @@ -1793,6 +1795,10 @@ do_inotify() CHKiRet(wdmapInit()); CHKiRet(dirsInit()); ino_fd = inotify_init(); + if(ino_fd < 0) { + errmsg.LogError(1, RS_RET_INOTIFY_INIT_FAILED, "imfile: Init inotify instance failed "); + return RS_RET_INOTIFY_INIT_FAILED; + } DBGPRINTF("imfile: inotify fd %d\n", ino_fd); in_setupInitialWatches(); @@ -1801,14 +1807,11 @@ do_inotify() if(rd < 0 && Debug) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - dbgprintf("imfile: error during inotify: %s\n", errStr); + DBGPRINTF("imfile: error during inotify: %s\n", errStr); } currev = 0; while(currev < rd) { ev = (struct inotify_event*) (iobuf+currev); - dbgprintf("DDDD: imfile event notification: rd %d[%d], wd (%d, mask " - "%8.8x, cookie %4.4x, len %d)\n", - (int) rd, currev, ev->wd, ev->mask, ev->cookie, ev->len); in_dbg_showEv(ev); in_processEvent(ev); currev += sizeof(struct inotify_event) + ev->len; @@ -1819,6 +1822,7 @@ finalize_it: close(ino_fd); RETiRet; } +#pragma GCC diagnostic pop #else /* #if HAVE_INOTIFY_INIT */ static rsRetVal @@ -1837,7 +1841,7 @@ do_inotify() */ BEGINrunInput CODESTARTrunInput - DBGPRINTF("imfile: working in %s mode\n", + DBGPRINTF("imfile: working in %s mode\n", (runModConf->opMode == OPMODE_POLLING) ? "polling" : "inotify"); if(runModConf->opMode == OPMODE_POLLING) iRet = doPolling(); @@ -1898,7 +1902,7 @@ persistStrmState(lstn_t *pLstn) finalize_it: if(psSF != NULL) strm.Destruct(&psSF); - + if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "imfile: could not persist state " "file %s - data may be repeated on next " diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index 348beaff5..35341cca4 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -435,7 +435,7 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) */ char *buf; int ret = 0; - CHKmalloc(buf = (char*) MALLOC(sizeof(char) * (glbl.GetMaxLine() + 1))); + CHKmalloc(buf = (char*) MALLOC(glbl.GetMaxLine() + 1)); prop.GetString(pSess->fromHostIP, &pszPeer, &lenPeer); diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c index 619400323..8f8b4d86f 100644 --- a/plugins/imjournal/imjournal.c +++ b/plugins/imjournal/imjournal.c @@ -64,6 +64,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(errmsg) struct modConfData_s { + int bIgnPrevMsg; }; static struct configSettings_s { @@ -185,7 +186,7 @@ enqMsg(uchar *msg, uchar *pszTag, int iFacility, int iSeverity, struct timeval * if(tp == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { - datetime.timeval2syslogTime(tp, &st); + datetime.timeval2syslogTime(tp, &st, TIME_IN_LOCALTIME); CHKiRet(msgConstructWithTime(&pMsg, &st, tp->tv_sec)); } MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -233,7 +234,6 @@ readjournal() { const void *get; const void *pidget; - char *parse; size_t length; size_t pidlength; @@ -334,64 +334,7 @@ readjournal() { /* get length of journal data prefix */ prefixlen = ((char *)equal_sign - (char *)get); - /* translate name fields to lumberjack names */ - parse = (char *)get; - - switch (*parse) - { - case '_': - ++parse; - if (*parse == 'P') { - if (!strncmp(parse+1, "ID=", 4)) { - name = strdup("pid"); - } else { - name = strndup(get, prefixlen); - } - } else if (*parse == 'G') { - if (!strncmp(parse+1, "ID=", 4)) { - name = strdup("gid"); - } else { - name = strndup(get, prefixlen); - } - } else if (*parse == 'U') { - if (!strncmp(parse+1, "ID=", 4)) { - name = strdup("uid"); - } else { - name = strndup(get, prefixlen); - } - } else if (*parse == 'E') { - if (!strncmp(parse+1, "XE=", 4)) { - name = strdup("exe"); - } else { - name = strndup(get, prefixlen); - } - } else if (*parse == 'C') { - parse++; - if (*parse == 'O') { - if (!strncmp(parse+1, "MM=", 4)) { - name = strdup("appname"); - } else { - name = strndup(get, prefixlen); - } - } else if (*parse == 'M') { - if (!strncmp(parse+1, "DLINE=", 7)) { - name = strdup("cmd"); - } else { - name = strndup(get, prefixlen); - } - } else { - name = strndup(get, prefixlen); - } - } else { - name = strndup(get, prefixlen); - } - break; - - default: - name = strndup(get, prefixlen); - break; - } - + name = strndup(get, prefixlen); CHKmalloc(name); prefixlen++; /* remove '=' */ @@ -588,6 +531,7 @@ finalize_it: } BEGINrunInput + int count = 0; CODESTARTrunInput CHKiRet(ratelimitNew(&ratelimiter, "imjournal", NULL)); dbgprintf("imjournal: ratelimiting burst %d, interval %d\n", cs.ratelimitBurst, @@ -616,7 +560,7 @@ CODESTARTrunInput * signalled to do so. This, however, is handled by the framework. */ while (glbl.GetGlobalInputTermState() == 0) { - int count = 0, r; + int r; r = sd_journal_next(j); if (r < 0) { diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index dc3870ba5..b799c519d 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -241,7 +241,7 @@ readklog(modConfData_t *pModConf) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - if((pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))) == NULL) { + if((pRcv = (uchar*) MALLOC(iMaxLine + 1)) == NULL) { iMaxLine = sizeof(bufRcv) - 1; /* better this than noting */ pRcv = bufRcv; } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 9fc26c3c6..a5962cb52 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -137,7 +137,7 @@ enqMsg(uchar *const __restrict__ msg, uchar* pszTag, const syslog_pri_t pri, str if(tp == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { - datetime.timeval2syslogTime(tp, &st); + datetime.timeval2syslogTime(tp, &st, TIME_IN_LOCALTIME); CHKiRet(msgConstructWithTime(&pMsg, &st, tp->tv_sec)); } MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -206,7 +206,7 @@ rsRetVal imklogLogIntMsg(syslog_pri_t priority, char *fmt, ...) uchar msgBuf[2048]; /* we use the same size as sysklogd to remain compatible */ va_start(ap, fmt); - vsnprintf((char*)msgBuf, sizeof(msgBuf) / sizeof(char), fmt, ap); + vsnprintf((char*)msgBuf, sizeof(msgBuf), fmt, ap); va_end(ap); logmsgInternal(NO_ERRCODE, priority, msgBuf, 0); diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 961353ae9..7380017e5 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -341,6 +341,8 @@ CODESTARTsetModCnf mode = es_str2cstr(pvals[i].val.d.estr, NULL); if(!strcasecmp(mode, "json")) { loadModConf->statsFmt = statsFmt_JSON; + } else if(!strcasecmp(mode, "json-elasticsearch")) { + loadModConf->statsFmt = statsFmt_JSON_ES; } else if(!strcasecmp(mode, "cee")) { loadModConf->statsFmt = statsFmt_CEE; } else if(!strcasecmp(mode, "legacy")) { @@ -468,6 +470,7 @@ CODESTARTfreeCnf if(runModConf->logfd != -1) close(runModConf->logfd); free(runModConf->logfile); + free(runModConf->pszBindRuleset); ENDfreeCnf diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 6d8b0e16e..4dcddf50e 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -10,18 +10,18 @@ * * File begun on 2010-08-10 by RGerhards * - * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,7 @@ DEFobjCurrIf(statsobj) static void * wrkr(void *myself); #define DFLT_wrkrMax 2 +#define DFLT_inlineDispatchThreshold 1 #define COMPRESS_NEVER 0 #define COMPRESS_SINGLE_MSG 1 /* old, single-message compression */ @@ -142,6 +144,7 @@ struct modConfData_s { rsconf_t *pConf; /* our overall config object */ instanceConf_t *root, *tail; int wrkrMax; + int bProcessOnPoller; sbool configSetViaV2Method; }; @@ -150,7 +153,8 @@ static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current lo /* module-global parameters */ static struct cnfparamdescr modpdescr[] = { - { "threads", eCmdHdlrPositiveInt, 0 } + { "threads", eCmdHdlrPositiveInt, 0 }, + { "processOnPoller", eCmdHdlrBinary, 0 } }; static struct cnfparamblk modpblk = { CNFPARAMBLK_VERSION, @@ -270,12 +274,8 @@ struct ptcplstn_s { */ static struct wrkrInfo_s { pthread_t tid; /* the worker's thread ID */ - pthread_cond_t run; - struct epoll_event *event; /* event == NULL -> idle */ long long unsigned numCalled; /* how often was this called */ -} wrkrInfo[16]; -static pthread_mutex_t wrkrMut; -static pthread_cond_t wrkrIdle; +} *wrkrInfo; static int wrkrRunning; @@ -291,15 +291,31 @@ typedef enum { struct epolld_s { epolld_type_t typ; void *ptr; + int sock; struct epoll_event ev; }; +typedef struct io_req_s { + STAILQ_ENTRY(io_req_s) link; + epolld_t *epd; +} io_req_t; + +typedef struct io_q_s { + STAILQ_HEAD(ioq_s, io_req_s) q; + STATSCOUNTER_DEF(ctrEnq, mutCtrEnq); + int ctrMaxSz; //TODO: discuss potential problems around concurrent reads and writes + int sz; //current q size + statsobj_t *stats; + pthread_mutex_t mut; + pthread_cond_t wakeup_worker; +} io_q_t; /* global data */ pthread_attr_t wrkrThrdAttr; /* Attribute for session threads; read only after startup */ static ptcpsrv_t *pSrvRoot = NULL; static int epollfd = -1; /* (sole) descriptor for epoll */ static int iMaxLine; /* maximum size of a single message */ +static io_q_t io_q; /* forward definitions */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); @@ -747,10 +763,18 @@ finalize_it: * rgerhards, 2008-03-14 * EXTRACT from tcps_sess.c */ -static inline rsRetVal -processDataRcvd(ptcpsess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) +static rsRetVal +processDataRcvd(ptcpsess_t *const __restrict__ pThis, + char **buff, + const int buffLen, + struct syslogTime *stTime, + const time_t ttGenTime, + multi_submit_t *pMultiSub, + unsigned *const __restrict__ pnMsgs) { DEFiRet; + char c = **buff; + int octatesToCopy, octatesToDiscard; if(pThis->inputState == eAtStrtFram) { if(pThis->bSuppOctetFram && isdigit((int) c)) { @@ -797,42 +821,60 @@ processDataRcvd(ptcpsess_t *pThis, char c, struct syslogTime *stTime, time_t ttG } } else { assert(pThis->inputState == eInMsg); - if(pThis->iMsg >= iMaxLine) { - /* emergency, we now need to flush, no matter if we are at end of message or not... */ - DBGPRINTF("error: message received is larger than max msg size, we split it\n"); - doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); - /* we might think if it is better to ignore the rest of the - * message than to treat it as a new one. Maybe this is a good - * candidate for a configuration parameter... - * rgerhards, 2006-12-04 - */ - } - if(( (c == '\n') - || ((pThis->pLstn->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) - && (c == pThis->pLstn->pSrv->iAddtlFrameDelim)) - ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); - pThis->inputState = eAtStrtFram; - } else { - /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! - * If we have a message that is larger than the max msg size, we truncate it. This is the best - * we can do in light of what the engine supports. -- rgerhards, 2008-03-14 - */ - if(pThis->iMsg < iMaxLine) { - *(pThis->pMsg + pThis->iMsg++) = c; + if (pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { + if(pThis->iMsg >= iMaxLine) { + /* emergency, we now need to flush, no matter if we are at end of message or not... */ + DBGPRINTF("error: message received is larger than max msg size, we split it\n"); + doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); + /* we might think if it is better to ignore the rest of the + * message than to treat it as a new one. Maybe this is a good + * candidate for a configuration parameter... + * rgerhards, 2006-12-04 + */ } - } - if(pThis->eFraming == TCP_FRAMING_OCTET_COUNTING) { - /* do we need to find end-of-frame via octet counting? */ - pThis->iOctetsRemain--; - if(pThis->iOctetsRemain < 1) { + if ((c == '\n') + || ((pThis->pLstn->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) + && (c == pThis->pLstn->pSrv->iAddtlFrameDelim)) + ) { /* record delimiter? */ + doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); + pThis->inputState = eAtStrtFram; + } else { + /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! + * If we have a message that is larger than the max msg size, we truncate it. This is the best + * we can do in light of what the engine supports. -- rgerhards, 2008-03-14 + */ + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } + } + } else { + assert(pThis->eFraming == TCP_FRAMING_OCTET_COUNTING); + octatesToCopy = pThis->iOctetsRemain; + octatesToDiscard = 0; + if (buffLen < octatesToCopy) { + octatesToCopy = buffLen; + } + if (octatesToCopy + pThis->iMsg > iMaxLine) { + octatesToDiscard = octatesToCopy - (iMaxLine - pThis->iMsg); + octatesToCopy = iMaxLine - pThis->iMsg; + } + + memcpy(pThis->pMsg + pThis->iMsg, *buff, octatesToCopy); + pThis->iMsg += octatesToCopy; + pThis->iOctetsRemain -= (octatesToCopy + octatesToDiscard); + *buff += (octatesToCopy + octatesToDiscard - 1); + if (pThis->iOctetsRemain == 0) { /* we have end of frame! */ doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); pThis->inputState = eAtStrtFram; } } + } finalize_it: @@ -862,13 +904,14 @@ DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, struct syslogT multi_submit_t multiSub; msg_t *pMsgs[CONF_NUM_MULTISUB]; char *pEnd; + unsigned nMsgs = 0; DEFiRet; assert(pData != NULL); assert(iLen > 0); if(ttGenTime == 0) - datetime.getCurrTime(stTime, &ttGenTime); + datetime.getCurrTime(stTime, &ttGenTime, TIME_IN_LOCALTIME); multiSub.ppMsgs = pMsgs; multiSub.maxElem = CONF_NUM_MULTISUB; multiSub.nElem = 0; @@ -877,11 +920,15 @@ DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, struct syslogT pEnd = pData + iLen; /* this is one off, which is intensional */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++, stTime, ttGenTime, &multiSub)); + CHKiRet(processDataRcvd(pThis, &pData, pEnd - pData, stTime, ttGenTime, &multiSub, &nMsgs)); + pData++; } iRet = multiSubmitFlush(&multiSub); + if(glblSenderKeepTrack) + statsRecordSender(propGetSzStr(pThis->peerName), nMsgs, ttGenTime); + finalize_it: RETiRet; } @@ -899,7 +946,7 @@ DataRcvdCompressed(ptcpsess_t *pThis, char *buf, size_t len) // by simply updating the input and output sizes? uint64_t outtotal; - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); outtotal = 0; if(!pThis->bzInitDone) { @@ -977,11 +1024,12 @@ addEPollSock(epolld_type_t typ, void *ptr, int sock, epolld_t **pEpd) DEFiRet; epolld_t *epd = NULL; - CHKmalloc(epd = calloc(sizeof(epolld_t), 1)); + CHKmalloc(epd = calloc(1, sizeof(epolld_t))); epd->typ = typ; epd->ptr = ptr; + epd->sock = sock; *pEpd = epd; - epd->ev.events = EPOLLIN|EPOLLET; + epd->ev.events = EPOLLIN|EPOLLET|EPOLLONESHOT; epd->ev.data.ptr = (void*) epd; if(epoll_ctl(epollfd, EPOLL_CTL_ADD, sock, &(epd->ev)) != 0) { @@ -996,6 +1044,9 @@ addEPollSock(epolld_type_t typ, void *ptr, int sock, epolld_t **pEpd) finalize_it: if(iRet != RS_RET_OK) { + if (epd != NULL) { + errmsg.LogError(0, RS_RET_INTERNAL_ERROR, "error: could not initialize mutex for ptcp connection for socket: %d", sock); + } free(epd); } RETiRet; @@ -1101,7 +1152,7 @@ addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) ptcpsrv_t *pSrv = pLstn->pSrv; CHKmalloc(pSess = malloc(sizeof(ptcpsess_t))); - CHKmalloc(pSess->pMsg = malloc(iMaxLine * sizeof(uchar))); + CHKmalloc(pSess->pMsg = malloc(iMaxLine)); pSess->pLstn = pLstn; pSess->sock = sock; pSess->bSuppOctetFram = pLstn->bSuppOctetFram; @@ -1114,8 +1165,6 @@ addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) pSess->peerIP = peerIP; pSess->compressionMode = pLstn->pSrv->compressionMode; - CHKiRet(addEPollSock(epolld_sess, pSess, sock, &pSess->epd)); - /* add to start of server's listener list */ pSess->prev = NULL; pthread_mutex_lock(&pSrv->mutSessLst); @@ -1125,6 +1174,8 @@ addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) pSrv->pSess = pSess; pthread_mutex_unlock(&pSrv->mutSessLst); + CHKiRet(addEPollSock(epolld_sess, pSess, sock, &pSess->epd)); + finalize_it: if(iRet != RS_RET_OK) { if(pSess != NULL) { @@ -1368,15 +1419,14 @@ startWorkerPool(void) { int i; wrkrRunning = 0; - if(runModConf->wrkrMax > 16) - runModConf->wrkrMax = 16; /* TODO: make dynamic? */ DBGPRINTF("imptcp: starting worker pool, %d workers\n", runModConf->wrkrMax); - pthread_mutex_init(&wrkrMut, NULL); - pthread_cond_init(&wrkrIdle, NULL); + wrkrInfo = calloc(runModConf->wrkrMax, sizeof(struct wrkrInfo_s)); + if (wrkrInfo == NULL) { + DBGPRINTF("imptcp: worker-info array allocation failed.\n"); + return; + } for(i = 0 ; i < runModConf->wrkrMax ; ++i) { /* init worker info structure! */ - pthread_cond_init(&wrkrInfo[i].run, NULL); - wrkrInfo[i].event = NULL; wrkrInfo[i].numCalled = 0; pthread_create(&wrkrInfo[i].tid, &wrkrThrdAttr, wrkr, &(wrkrInfo[i])); } @@ -1390,14 +1440,14 @@ stopWorkerPool(void) { int i; DBGPRINTF("imptcp: stoping worker pool\n"); + pthread_mutex_lock(&io_q.mut); + pthread_cond_broadcast(&io_q.wakeup_worker); /* awake wrkr if not running */ + pthread_mutex_unlock(&io_q.mut); for(i = 0 ; i < runModConf->wrkrMax ; ++i) { - pthread_cond_signal(&wrkrInfo[i].run); /* awake wrkr if not running */ pthread_join(wrkrInfo[i].tid, NULL); DBGPRINTF("imptcp: info: worker %d was called %llu times\n", i, wrkrInfo[i].numCalled); - pthread_cond_destroy(&wrkrInfo[i].run); } - pthread_cond_destroy(&wrkrIdle); - pthread_mutex_destroy(&wrkrMut); + free(wrkrInfo); } @@ -1435,7 +1485,6 @@ startupServers() RETiRet; } - /* process new activity on listener. This means we need to accept a new * connection. */ @@ -1451,8 +1500,9 @@ lstnActivity(ptcplstn_t *pLstn) DBGPRINTF("imptcp: new connection on listen socket %d\n", pLstn->sock); while(glbl.GetGlobalInputTermState() == 0) { localRet = AcceptConnReq(pLstn, &newSock, &peerName, &peerIP); - if(localRet == RS_RET_NO_MORE_DATA || glbl.GetGlobalInputTermState() == 1) + if(localRet == RS_RET_NO_MORE_DATA || glbl.GetGlobalInputTermState() == 1) { break; + } CHKiRet(localRet); localRet = addSess(pLstn, newSock, peerName, peerIP); if(localRet != RS_RET_OK) { @@ -1467,12 +1517,11 @@ finalize_it: RETiRet; } - /* process new activity on session. This means we need to accept data * or close the session. */ static inline rsRetVal -sessActivity(ptcpsess_t *pSess) +sessActivity(ptcpsess_t *pSess, int *continue_polling) { int lenRcv; int lenBuf; @@ -1500,6 +1549,7 @@ sessActivity(ptcpsess_t *pSess) remsock = pSess->sock; bEmitOnClose = 1; } + *continue_polling = 0; CHKiRet(closeSess(pSess)); /* close may emit more messages in strmzip mode! */ if(bEmitOnClose) { errmsg.LogError(0, RS_RET_PEER_CLOSED_CONN, "imptcp session %d closed by " @@ -1510,6 +1560,7 @@ sessActivity(ptcpsess_t *pSess) if(errno == EAGAIN || errno == EWOULDBLOCK) break; DBGPRINTF("imptcp: error on session socket %d - closed.\n", pSess->sock); + *continue_polling = 0; closeSess(pSess); /* try clean-up by dropping session */ break; } @@ -1525,26 +1576,106 @@ finalize_it: * concurrently. */ static inline void -processWorkItem(struct epoll_event *event) +processWorkItem(epolld_t *epd) { - epolld_t *epd; + int continue_polling = 1; - epd = (epolld_t*) event->data.ptr; switch(epd->typ) { case epolld_lstn: + /* listener never stops polling (except server shutdown) */ lstnActivity((ptcplstn_t *) epd->ptr); break; case epolld_sess: - sessActivity((ptcpsess_t *) epd->ptr); + sessActivity((ptcpsess_t *) epd->ptr, &continue_polling); break; default: errmsg.LogError(0, RS_RET_INTERNAL_ERROR, - "error: invalid epolld_type_t %d after epoll", epd->typ); + "error: invalid epolld_type_t %d after epoll", epd->typ); break; } + if (continue_polling == 1) { + epoll_ctl(epollfd, EPOLL_CTL_MOD, epd->sock, &(epd->ev)); + } } +static rsRetVal +initIoQ() { + DEFiRet; + CHKiConcCtrl(pthread_mutex_init(&io_q.mut, NULL)); + CHKiConcCtrl(pthread_cond_init(&io_q.wakeup_worker, NULL)); + STAILQ_INIT(&io_q.q); + io_q.sz = 0; + io_q.ctrMaxSz = 0; /* TODO: discuss this and fix potential concurrent read/write issues */ + CHKiRet(statsobj.Construct(&io_q.stats)); + CHKiRet(statsobj.SetName(io_q.stats, (uchar*) "io-work-q")); + CHKiRet(statsobj.SetOrigin(io_q.stats, (uchar*) "imptcp")); + STATSCOUNTER_INIT(io_q.ctrEnq, io_q.mutCtrEnq); + CHKiRet(statsobj.AddCounter(io_q.stats, UCHAR_CONSTANT("enqueued"), + ctrType_IntCtr, CTR_FLAG_RESETTABLE, &io_q.ctrEnq)); + CHKiRet(statsobj.AddCounter(io_q.stats, UCHAR_CONSTANT("maxqsize"), + ctrType_Int, CTR_FLAG_NONE, &io_q.ctrMaxSz)); + CHKiRet(statsobj.ConstructFinalize(io_q.stats)); +finalize_it: + RETiRet; +} + +static void +destroyIoQ() { + io_req_t *n; + if (io_q.stats != NULL) { + statsobj.Destruct(&io_q.stats); + } + pthread_mutex_lock(&io_q.mut); + while (!STAILQ_EMPTY(&io_q.q)) { + n = STAILQ_FIRST(&io_q.q); + STAILQ_REMOVE_HEAD(&io_q.q, link); + errmsg.LogError(0, RS_RET_INTERNAL_ERROR, "imptcp: discarded enqueued io-work to allow shutdown - ignored"); + free(n); + } + io_q.sz = 0; + pthread_mutex_unlock(&io_q.mut); + pthread_cond_destroy(&io_q.wakeup_worker); + pthread_mutex_destroy(&io_q.mut); +} + +static inline rsRetVal +enqueueIoWork(epolld_t *epd, int dispatchInlineIfQueueFull) { + io_req_t *n; + int dispatchInline; + DEFiRet; + + CHKmalloc(n = malloc(sizeof(io_req_t))); + n->epd = epd; + + int inlineDispatchThreshold = DFLT_inlineDispatchThreshold * runModConf->wrkrMax; + dispatchInline = 0; + + pthread_mutex_lock(&io_q.mut); + if (dispatchInlineIfQueueFull && io_q.sz > inlineDispatchThreshold) { + dispatchInline = 1; + } else { + STAILQ_INSERT_TAIL(&io_q.q, n, link); + io_q.sz++; + STATSCOUNTER_INC(io_q.ctrEnq, io_q.mutCtrEnq); + STATSCOUNTER_SETMAX_NOMUT(io_q.ctrMaxSz, io_q.sz); + pthread_cond_signal(&io_q.wakeup_worker); + } + pthread_mutex_unlock(&io_q.mut); + + if (dispatchInline == 1) { + free(n); + processWorkItem(epd); + } +finalize_it: + if (iRet != RS_RET_OK) { + if (n == NULL) { + errmsg.LogError(0, iRet, "imptcp: couldn't allocate memory to enqueue io-request - ignored"); + } + } + RETiRet; +} + /* This function is called to process a complete workset, that * is a set of events returned from epoll. */ @@ -1552,46 +1683,20 @@ static inline void processWorkSet(int nEvents, struct epoll_event events[]) { int iEvt; - int i; int remainEvents; - remainEvents = nEvents; + epolld_t *epd; + for(iEvt = 0 ; (iEvt < nEvents) && (glbl.GetGlobalInputTermState() == 0) ; ++iEvt) { - if(remainEvents == 1) { + epd = (epolld_t*)events[iEvt].data.ptr; + if(runModConf->bProcessOnPoller && remainEvents == 1) { /* process self, save context switch */ - processWorkItem(events+iEvt); + processWorkItem(epd); } else { - pthread_mutex_lock(&wrkrMut); - /* check if there is a free worker */ - for(i = 0 ; (i < runModConf->wrkrMax) && (wrkrInfo[i].event != NULL) ; ++i) - /*do search*/; - if(i < runModConf->wrkrMax) { - /* worker free -> use it! */ - wrkrInfo[i].event = events+iEvt; - ++wrkrRunning; - pthread_cond_signal(&wrkrInfo[i].run); - pthread_mutex_unlock(&wrkrMut); - } else { - pthread_mutex_unlock(&wrkrMut); - /* no free worker, so we process this one ourselfs */ - processWorkItem(events+iEvt); - } + enqueueIoWork(epd, runModConf->bProcessOnPoller); } --remainEvents; } - - if(nEvents > 1) { - /* we now need to wait until all workers finish. This is because the - * rest of this module can not handle the concurrency introduced - * by workers running during the epoll call. - */ - pthread_mutex_lock(&wrkrMut); - while(wrkrRunning > 0) { - pthread_cond_wait(&wrkrIdle, &wrkrMut); - } - pthread_mutex_unlock(&wrkrMut); - } - } @@ -1601,26 +1706,37 @@ static void * wrkr(void *myself) { struct wrkrInfo_s *me = (struct wrkrInfo_s*) myself; - - pthread_mutex_lock(&wrkrMut); + pthread_mutex_lock(&io_q.mut); + ++wrkrRunning; + pthread_mutex_unlock(&io_q.mut); + + io_req_t *n; while(1) { - while(me->event == NULL && glbl.GetGlobalInputTermState() == 0) { - pthread_cond_wait(&me->run, &wrkrMut); + n = NULL; + pthread_mutex_lock(&io_q.mut); + if (io_q.sz == 0) { + --wrkrRunning; + if (glbl.GetGlobalInputTermState() != 0) { + pthread_mutex_unlock(&io_q.mut); + break; + } else { + pthread_cond_wait(&io_q.wakeup_worker, &io_q.mut); + } + ++wrkrRunning; } - if(glbl.GetGlobalInputTermState() == 1) - break; - pthread_mutex_unlock(&wrkrMut); + if (io_q.sz > 0) { + n = STAILQ_FIRST(&io_q.q); + STAILQ_REMOVE_HEAD(&io_q.q, link); + io_q.sz--; + } + pthread_mutex_unlock(&io_q.mut); - ++me->numCalled; - processWorkItem(me->event); - - pthread_mutex_lock(&wrkrMut); - me->event = NULL; /* indicate we are free again */ - --wrkrRunning; - pthread_cond_signal(&wrkrIdle); + if (n != NULL) { + ++me->numCalled; + processWorkItem(n->epd); + free(n); + } } - pthread_mutex_unlock(&wrkrMut); - return NULL; } @@ -1707,6 +1823,7 @@ CODESTARTbeginCnfLoad pModConf->pConf = pConf; /* init our settings */ loadModConf->wrkrMax = DFLT_wrkrMax; + loadModConf->bProcessOnPoller = 1; loadModConf->configSetViaV2Method = 0; bLegacyCnfModGlobalsPermitted = 1; /* init legacy config vars */ @@ -1735,6 +1852,8 @@ CODESTARTsetModCnf continue; if(!strcmp(modpblk.descr[i].name, "threads")) { loadModConf->wrkrMax = (int) pvals[i].val.d.n; + } else if(!strcmp(modpblk.descr[i].name, "processOnPoller")) { + loadModConf->bProcessOnPoller = (int) pvals[i].val.d.n; } else { dbgprintf("imptcp: program error, non-handled " "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); @@ -1855,6 +1974,7 @@ BEGINrunInput int nEvents; struct epoll_event events[128]; CODESTARTrunInput + initIoQ(); startWorkerPool(); DBGPRINTF("imptcp: now beginning to process input data\n"); while(glbl.GetGlobalInputTermState() == 0) { @@ -1893,7 +2013,7 @@ shutdownSrv(ptcpsrv_t *pSrv) pLstn = pLstn->next; DBGPRINTF("imptcp shutdown listen socket %d (rcvd %lld bytes, " "decompressed %lld)\n", lstnDel->sock, lstnDel->rcvdBytes, - lstnDel->rcvdDecompressed); + lstnDel->rcvdDecompressed); free(lstnDel->epd); free(lstnDel); } @@ -1914,6 +2034,7 @@ BEGINafterRun ptcpsrv_t *pSrv, *srvDel; CODESTARTafterRun stopWorkerPool(); + destroyIoQ(); /* we need to close everything that is still open */ pSrv = pSrvRoot; diff --git a/plugins/imsolaris/imsolaris.c b/plugins/imsolaris/imsolaris.c index 83cab0aed..5edd89351 100644 --- a/plugins/imsolaris/imsolaris.c +++ b/plugins/imsolaris/imsolaris.c @@ -248,7 +248,7 @@ getMsgs(thrdInfo_t *pThrd, int timeout) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - CHKmalloc(pRcv = (uchar*) malloc(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pRcv = (uchar*) malloc(iMaxLine + 1)); } while(pThrd->bShallStop != RSTRUE) { diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index eacdb5828..82a4a348b 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -417,7 +417,7 @@ processPacket(struct lstn_s *lstn, struct sockaddr_storage *frominetPrev, int *p *pbIsPermitted = 1; /* no check -> everything permitted */ } - DBGPRINTF("recv(%d,%d),acl:%d,msg:%.128s\n", lstn->sock, (int) lenRcvBuf, *pbIsPermitted, rcvBuf); + DBGPRINTF("recv(%d,%d),acl:%d,msg:%.*s\n", lstn->sock, (int) lenRcvBuf, *pbIsPermitted, (int)lenRcvBuf, rcvBuf); if(*pbIsPermitted != 0) { /* we now create our own message object and submit it to the queue */ @@ -459,7 +459,9 @@ processSocket(struct wrkrInfo_s *pWrkr, struct lstn_s *lstn, struct sockaddr_sto { DEFiRet; int iNbrTimeUsed; - time_t ttGenTime; + time_t ttGenTime = 0; /* to avoid clang static analyzer false positive */ + /* note: we do never use this time, because we always get a + * requery below on first loop iteration */ struct syslogTime stTime; char errStr[1024]; msg_t *pMsgs[CONF_NUM_MULTISUB]; @@ -507,7 +509,7 @@ processSocket(struct wrkrInfo_s *pWrkr, struct lstn_s *lstn, struct sockaddr_sto } if((runModConf->iTimeRequery == 0) || (iNbrTimeUsed++ % runModConf->iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); } pWrkr->ctrMsgsRcvd += nelem; @@ -580,7 +582,7 @@ processSocket(struct wrkrInfo_s *pWrkr, struct lstn_s *lstn, struct sockaddr_sto ++pWrkr->ctrMsgsRcvd; if((runModConf->iTimeRequery == 0) || (iNbrTimeUsed++ % runModConf->iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); } CHKiRet(processPacket(lstn, frominetPrev, pbIsPermitted, pWrkr->pRcvBuf, lenRcvBuf, &stTime, @@ -1103,7 +1105,7 @@ BEGINactivateCnf CODESTARTactivateCnf /* caching various settings */ iMaxLine = glbl.GetMaxLine(); - lenRcvBuf = (iMaxLine + 1) * sizeof(char); + lenRcvBuf = iMaxLine + 1; # ifdef HAVE_RECVMMSG lenRcvBuf *= runModConf->batchSize; # endif diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 59161ff9c..0f20ee25c 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -50,6 +50,7 @@ #include "parser.h" #include "prop.h" #include "debug.h" +#include "ruleset.h" #include "unlimited_select.h" #include "sd-daemon.h" #include "statsobj.h" @@ -99,6 +100,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(parser) DEFobjCurrIf(datetime) DEFobjCurrIf(statsobj) +DEFobjCurrIf(ruleset) statsobj_t *modStats; @@ -148,6 +150,7 @@ typedef struct lstn_s { sbool bUseSysTimeStamp; /* use timestamp from system (instead of from message) */ sbool bUnlink; /* unlink&re-create socket at start and end of processing */ sbool bUseSpecialParser;/* use "canned" log socket parser instead of parser chain? */ + ruleset_t *pRuleset; } lstn_t; static lstn_t *listeners; @@ -208,6 +211,8 @@ struct instanceConf_s { sbool bUnlink; sbool bUseSpecialParser; sbool bParseHost; + uchar *pszBindRuleset; /* name of ruleset to bind to */ + ruleset_t *pBindRuleset; /* ruleset to bind listener to (use system default if unspecified) */ struct instanceConf_s *next; }; @@ -273,6 +278,7 @@ static struct cnfparamdescr inppdescr[] = { { "usespecialparser", eCmdHdlrBinary, 0 }, { "parsehostname", eCmdHdlrBinary, 0 }, { "usepidfromsystem", eCmdHdlrBinary, 0 }, + { "ruleset", eCmdHdlrString, 0 }, { "ratelimit.interval", eCmdHdlrInt, 0 }, { "ratelimit.burst", eCmdHdlrInt, 0 }, { "ratelimit.severity", eCmdHdlrInt, 0 } @@ -283,8 +289,7 @@ static struct cnfparamblk inppblk = inppdescr }; -/* we do not use this, because we do not bind to a ruleset so far - * enable when this is changed: #include "im-helper.h" */ /* must be included AFTER the type definitions! */ +#include "im-helper.h" /* must be included AFTER the type definitions! */ static int bLegacyCnfModGlobalsPermitted;/* are legacy module-global config parameters permitted? */ @@ -300,6 +305,8 @@ createInstance(instanceConf_t **pinst) CHKmalloc(inst = MALLOC(sizeof(instanceConf_t))); inst->sockName = NULL; inst->pLogHostName = NULL; + inst->pszBindRuleset = NULL; + inst->pBindRuleset = NULL; inst->ratelimitInterval = DFLT_ratelimitInterval; inst->ratelimitBurst = DFLT_ratelimitBurst; inst->ratelimitSeverity = DFLT_ratelimitSeverity; @@ -422,6 +429,7 @@ addListner(instanceConf_t *inst) listeners[nfd].bWritePid = inst->bWritePid; listeners[nfd].bUseSysTimeStamp = inst->bUseSysTimeStamp; listeners[nfd].bUseSpecialParser = inst->bUseSpecialParser; + listeners[nfd].pRuleset = inst->pBindRuleset; CHKiRet(ratelimitNew(&listeners[nfd].dflt_ratelimiter, "imuxsock", NULL)); ratelimitSetLinuxLike(listeners[nfd].dflt_ratelimiter, listeners[nfd].ratelimitInterval, @@ -791,9 +799,9 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim findRatelimiter(pLstn, cred, &ratelimiter); /* ignore error, better so than others... */ if(ts == NULL) { - datetime.getCurrTime(&st, &tt); + datetime.getCurrTime(&st, &tt, TIME_IN_LOCALTIME); } else { - datetime.timeval2syslogTime(ts, &st); + datetime.timeval2syslogTime(ts, &st, TIME_IN_LOCALTIME); tt = ts->tv_sec; } @@ -888,6 +896,9 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim pmsgbuf[toffs+1] = '\0'; MsgSetRawMsg(pMsg, (char*)pmsgbuf, toffs + 1); + if (pmsgbuf != msgbuf) { + free(pmsgbuf); + } } } else { /* just add the unmodified message */ @@ -959,6 +970,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim MsgSetRcvFrom(pMsg, pLstn->hostName == NULL ? glbl.GetLocalHostNameProp() : pLstn->hostName); CHKiRet(MsgSetRcvFromIP(pMsg, pLocalHostIP)); + MsgSetRuleset(pMsg, pLstn->pRuleset); ratelimitAddMsg(ratelimiter, NULL, pMsg); STATSCOUNTER_INC(ctrSubmit, mutCtrSubmit); finalize_it: @@ -977,6 +989,9 @@ finalize_it: * of the socket which is to be processed. This eases access to the * growing number of properties. -- rgerhards, 2008-08-01 */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-align" /* TODO: how can we fix these warnings? */ +/* Problem with the warnings: they seem to stem back from the way the API is structured */ static rsRetVal readSocket(lstn_t *pLstn) { DEFiRet; @@ -1005,7 +1020,7 @@ static rsRetVal readSocket(lstn_t *pLstn) if((size_t) iMaxLine < sizeof(bufRcv) - 1) { pRcv = bufRcv; } else { - CHKmalloc(pRcv = (uchar*) MALLOC(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pRcv = (uchar*) MALLOC(iMaxLine + 1)); } memset(&msgh, 0, sizeof(msgh)); @@ -1060,6 +1075,7 @@ finalize_it: RETiRet; } +#pragma GCC diagnostic pop /* activate current listeners */ @@ -1285,6 +1301,8 @@ CODESTARTnewInpInst inst->bParseHost = (int) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "usespecialparser")) { inst->bUseSpecialParser = (int) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "ruleset")) { + inst->pszBindRuleset = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(inppblk.descr[i].name, "ratelimit.interval")) { inst->ratelimitInterval = (int) pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "ratelimit.burst")) { @@ -1328,8 +1346,20 @@ CODESTARTendCnfLoad ENDendCnfLoad +/* function to generate error message if framework does not find requested ruleset */ +static void +std_checkRuleset_genErrMsg(__attribute__((unused)) modConfData_t *modConf, instanceConf_t *inst) +{ + errmsg.LogError(0, NO_ERRCODE, "imuxsock: ruleset '%s' for socket %s not found - " + "using default ruleset instead", inst->pszBindRuleset, + inst->sockName); +} BEGINcheckCnf + instanceConf_t *inst; CODESTARTcheckCnf + for(inst = pModConf->root ; inst != NULL ; inst = inst->next) { + std_checkRuleset(pModConf, inst); + } ENDcheckCnf @@ -1384,6 +1414,7 @@ CODESTARTfreeCnf free(pModConf->pLogSockName); for(inst = pModConf->root ; inst != NULL ; ) { free(inst->sockName); + free(inst->pszBindRuleset); free(inst->pLogHostName); del = inst; inst = inst->next; @@ -1512,6 +1543,7 @@ CODESTARTmodExit objRelease(prop, CORE_COMPONENT); objRelease(statsobj, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -1573,6 +1605,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(statsobj, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); DBGPRINTF("imuxsock version %s initializing\n", PACKAGE_VERSION); diff --git a/plugins/mmanon/mmanon.c b/plugins/mmanon/mmanon.c index a85faa13e..6fdcbaeb3 100644 --- a/plugins/mmanon/mmanon.c +++ b/plugins/mmanon/mmanon.c @@ -361,13 +361,13 @@ done: *idx = i; } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *msg; int lenMsg; int i; CODESTARTdoAction - pMsg = (msg_t*) ppString[0]; lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); for(i = 0 ; i < lenMsg ; ++i) { diff --git a/plugins/mmaudit/mmaudit.c b/plugins/mmaudit/mmaudit.c index 01b539c76..2c71b583f 100644 --- a/plugins/mmaudit/mmaudit.c +++ b/plugins/mmaudit/mmaudit.c @@ -14,7 +14,7 @@ * * File begun on 2012-02-23 by RGerhards * - * Copyright 2013 Adiscon GmbH. + * Copyright 2013-2016 Adiscon GmbH. * * This file is part of rsyslog. * @@ -54,7 +54,6 @@ MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP -MODULE_CNFNAME("mmaudit") static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); @@ -191,7 +190,6 @@ audit_parse(uchar *buf, struct json_object **jsonRoot) json_object_object_add(*jsonRoot, "data", json); while(*buf) { -//dbgprintf("audit_parse, buf: '%s'\n", buf); CHKiRet(parseName(&buf, name, sizeof(name))); if(*buf != '=') { ABORT_FINALIZE(RS_RET_ERR); @@ -200,7 +198,6 @@ audit_parse(uchar *buf, struct json_object **jsonRoot) CHKiRet(parseValue(&buf, val, sizeof(val))); jval = json_object_new_string(val); json_object_object_add(json, name, jval); -dbgprintf("mmaudit: parsed %s=%s\n", name, val); } @@ -209,8 +206,9 @@ finalize_it: } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *buf; int typeID; struct json_object *jsonRoot; @@ -220,14 +218,12 @@ BEGINdoAction char auditID[1024]; int bSuccess = 0; CODESTARTdoAction - pMsg = (msg_t*) ppString[0]; /* note that we can performance-optimize the interface, but this also * requires changes to the libraries. For now, we accept message * duplication. -- rgerhards, 2010-12-01 */ buf = getMSG(pMsg); -dbgprintf("mmaudit: msg is '%s'\n", buf); while(*buf && isspace(*buf)) { ++buf; } diff --git a/plugins/mmexternal/mmexternal.c b/plugins/mmexternal/mmexternal.c index 933e26e26..9f3a06b34 100644 --- a/plugins/mmexternal/mmexternal.c +++ b/plugins/mmexternal/mmexternal.c @@ -143,7 +143,7 @@ CODESTARTfreeInstance for (i = 0; i < pData->iParams; i++) { free(pData->aParams[i]); } - free(pData->aParams); + free(pData->aParams); } ENDfreeInstance @@ -217,7 +217,6 @@ processProgramReply(wrkrInstanceData_t *__restrict__ const pWrkrData, msg_t *con int numCharsRead; char *newptr; -dbgprintf("mmexternal: checking prog output, fd %d\n", pWrkrData->fdPipeIn); numCharsRead = 0; do { if(pWrkrData->maxLenRespBuf < numCharsRead + 256) { /* 256 to permit at least a decent read */ @@ -242,7 +241,6 @@ dbgprintf("mmexternal: checking prog output, fd %d\n", pWrkrData->fdPipeIn); strcpy(pWrkrData->respBuf, "{}\n"); numCharsRead = 3; } -dbgprintf("mmexternal: read state %lld, data '%s'\n", (long long) r, pWrkrData->respBuf); if(Debug && r == -1) { DBGPRINTF("mmexternal: error reading from external program: %s\n", rs_strerror_r(errno, errStr, sizeof(errStr))); @@ -322,7 +320,7 @@ execBinary(wrkrInstanceData_t *pWrkrData, int fdStdin, int fdStdOutErr) */ rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("mmexternal: failed to execute binary '%s': %s\n", - pWrkrData->pData->szBinary, errStr); + pWrkrData->pData->szBinary, errStr); } /* we should never reach this point, but if we do, we terminate */ @@ -510,10 +508,11 @@ finalize_it: } -BEGINdoAction +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; instanceData *pData; CODESTARTdoAction -dbgprintf("DDDD:mmexternal processing message\n"); pData = pWrkrData->pData; if(pData->bForceSingleInst) pthread_mutex_lock(&pData->mut); @@ -521,13 +520,12 @@ dbgprintf("DDDD:mmexternal processing message\n"); openPipe(pWrkrData); } - iRet = callExtProg(pWrkrData, (msg_t*)ppString[0]); + iRet = callExtProg(pWrkrData, pMsg); if(iRet != RS_RET_OK) iRet = RS_RET_SUSPENDED; if(pData->bForceSingleInst) pthread_mutex_unlock(&pData->mut); -dbgprintf("DDDD:mmexternal DONE processing message\n"); ENDdoAction @@ -566,8 +564,8 @@ CODESTARTnewActInst if(!pvals[i].bUsed) continue; if(!strcmp(actpblk.descr[i].name, "binary")) { - estrBinary = pvals[i].val.d.estr; - estrParams = NULL; + estrBinary = pvals[i].val.d.estr; + estrParams = NULL; /* Search for space */ c = es_getBufAddr(pvals[i].val.d.estr); @@ -575,18 +573,22 @@ CODESTARTnewActInst while(iCnt < es_strlen(pvals[i].val.d.estr) ) { if (c[iCnt] == ' ') { /* Split binary name from parameters */ - estrBinary = es_newStrFromSubStr ( pvals[i].val.d.estr, 0, iCnt ); - estrParams = es_newStrFromSubStr ( pvals[i].val.d.estr, iCnt+1, es_strlen(pvals[i].val.d.estr)); + estrBinary = es_newStrFromSubStr ( pvals[i].val.d.estr, 0, iCnt ); + estrParams = es_newStrFromSubStr ( pvals[i].val.d.estr, iCnt+1, es_strlen(pvals[i].val.d.estr)); break; } iCnt++; } /* Assign binary and params */ pData->szBinary = (uchar*)es_str2cstr(estrBinary, NULL); - dbgprintf("mmexternal: szBinary = '%s'\n", pData->szBinary); + DBGPRINTF("mmexternal: szBinary = '%s'\n", pData->szBinary); /* Check for Params! */ if (estrParams != NULL) { - dbgprintf("mmexternal: szParams = '%s'\n", es_str2cstr(estrParams, NULL) ); + if(Debug) { + char *params = es_str2cstr(estrParams, NULL); + DBGPRINTF("mmexternal: szParams = '%s'\n", params); + free(params); + } /* Count parameters if set */ c = es_getBufAddr(estrParams); /* Reset to beginning */ @@ -594,10 +596,10 @@ CODESTARTnewActInst iCnt = 0; while(iCnt < es_strlen(estrParams) ) { if (c[iCnt] == ' ' && c[iCnt-1] != '\\') - pData->iParams++; + pData->iParams++; iCnt++; } - dbgprintf("mmexternal: iParams = '%d'\n", pData->iParams); + DBGPRINTF("mmexternal: iParams = '%d'\n", pData->iParams); /* Create argv Array */ CHKmalloc(pData->aParams = malloc( (pData->iParams+1) * sizeof(char*))); /* One more for first param */ @@ -605,32 +607,32 @@ CODESTARTnewActInst /* Second Loop, create parameter array*/ c = es_getBufAddr(estrParams); /* Reset to beginning */ iCnt = iStr = iPrm = 0; - estrTmp = NULL; - bInQuotes = FALSE; + estrTmp = NULL; + bInQuotes = FALSE; /* Set first parameter to binary */ - pData->aParams[iPrm] = strdup((char*)pData->szBinary); - dbgprintf("mmexternal: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); - iPrm++; + pData->aParams[iPrm] = strdup((char*)pData->szBinary); + DBGPRINTF("mmexternal: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + iPrm++; while(iCnt < es_strlen(estrParams) ) { if ( c[iCnt] == ' ' && !bInQuotes ) { /* Copy into Param Array! */ - estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr); + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr); } else if ( iCnt+1 >= es_strlen(estrParams) ) { /* Copy rest of string into Param Array! */ - estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr+1); + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr+1); } else if (c[iCnt] == '"') { /* switch inQuotes Mode */ - bInQuotes = !bInQuotes; + bInQuotes = !bInQuotes; } if ( estrTmp != NULL ) { - pData->aParams[iPrm] = es_str2cstr(estrTmp, NULL); + pData->aParams[iPrm] = es_str2cstr(estrTmp, NULL); iStr = iCnt+1; /* Set new start */ - dbgprintf("mmexternal: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + DBGPRINTF("mmexternal: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); es_deleteStr( estrTmp ); - estrTmp = NULL; + estrTmp = NULL; iPrm++; } @@ -638,7 +640,7 @@ CODESTARTnewActInst iCnt++; } /* NULL last parameter! */ - pData->aParams[iPrm] = NULL; + pData->aParams[iPrm] = NULL; } } else if(!strcmp(actpblk.descr[i].name, "output")) { @@ -660,7 +662,7 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_INVLD_INTERFACE_INPUT); } } else { - dbgprintf("mmexternal: program error, non-handled param '%s'\n", actpblk.descr[i].name); + DBGPRINTF("mmexternal: program error, non-handled param '%s'\n", actpblk.descr[i].name); } } diff --git a/plugins/mmfields/mmfields.c b/plugins/mmfields/mmfields.c index 2c3cb4477..8286de70e 100644 --- a/plugins/mmfields/mmfields.c +++ b/plugins/mmfields/mmfields.c @@ -226,22 +226,24 @@ parse_fields(instanceData *pData, msg_t *pMsg, uchar *msgtext, int lenMsg) DBGPRINTF("mmfields: field %d: '%s'\n", field, buf); snprintf((char*)fieldname, sizeof(fieldname), "f%d", field); fieldname[sizeof(fieldname)-1] = '\0'; - jval = json_object_new_string((char*)fieldbuf); + jval = json_object_new_string((char*)buf); json_object_object_add(json, (char*)fieldname, jval); field++; } msgAddJSON(pMsg, pData->jsonRoot, json, 0, 0); finalize_it: + if(buf != fieldbuf) + free(buf); RETiRet; } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *msg; int lenMsg; CODESTARTdoAction - pMsg = (msg_t*) ppString[0]; lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); CHKiRet(parse_fields(pWrkrData->pData, pMsg, msg, lenMsg)); diff --git a/plugins/mmjsonparse/mmjsonparse.c b/plugins/mmjsonparse/mmjsonparse.c index 4926d62f1..ae94d78bf 100644 --- a/plugins/mmjsonparse/mmjsonparse.c +++ b/plugins/mmjsonparse/mmjsonparse.c @@ -214,8 +214,9 @@ finalize_it: RETiRet; } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *buf; rs_size_t len; int bSuccess = 0; @@ -224,7 +225,6 @@ BEGINdoAction instanceData *pData; CODESTARTdoAction pData = pWrkrData->pData; - pMsg = (msg_t*) ppString[0]; /* note that we can performance-optimize the interface, but this also * requires changes to the libraries. For now, we accept message * duplication. -- rgerhards, 2010-12-01 @@ -260,7 +260,6 @@ ENDdoAction static inline void setInstParamDefaults(instanceData *pData) { - pData->cookie = NULL; pData->bUseRawMsg = 0; } @@ -297,8 +296,6 @@ CODESTARTnewActInst if(pData->container == NULL) CHKmalloc(pData->container = (uchar*) strdup("!")); - if(pData->cookie == NULL) - CHKmalloc(pData->cookie = strdup("@cee:")); pData->lenCookie = strlen(pData->cookie); CODE_STD_FINALIZERnewActInst cnfparamvalsDestruct(pvals, &actpblk); diff --git a/plugins/mmnormalize/mmnormalize.c b/plugins/mmnormalize/mmnormalize.c index f84aa6fc6..74b5530d7 100644 --- a/plugins/mmnormalize/mmnormalize.c +++ b/plugins/mmnormalize/mmnormalize.c @@ -222,15 +222,15 @@ BEGINtryResume CODESTARTtryResume ENDtryResume -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *buf; rs_size_t len; int r; struct json_object *json = NULL; unsigned short freeBuf = 0; CODESTARTdoAction - pMsg = (msg_t*) ppString[0]; if(pWrkrData->pData->bUseRawMsg) { getRawMsg(pMsg, &buf, &len); } else if (pWrkrData->pData->varDescr) { diff --git a/plugins/mmpstrucdata/mmpstrucdata.c b/plugins/mmpstrucdata/mmpstrucdata.c index b4fcc1518..bd0d611da 100644 --- a/plugins/mmpstrucdata/mmpstrucdata.c +++ b/plugins/mmpstrucdata/mmpstrucdata.c @@ -333,11 +333,11 @@ finalize_it: } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; CODESTARTdoAction DBGPRINTF("mmpstrucdata: enter\n"); - pMsg = (msg_t*) ppString[0]; if(!MsgHasStructuredData(pMsg)) { DBGPRINTF("mmpstrucdata: message does not have structured data\n"); FINALIZE; diff --git a/plugins/mmutf8fix/mmutf8fix.c b/plugins/mmutf8fix/mmutf8fix.c index e52968636..597a172cf 100644 --- a/plugins/mmutf8fix/mmutf8fix.c +++ b/plugins/mmutf8fix/mmutf8fix.c @@ -211,6 +211,8 @@ fixInvldMBSeq(instanceData *pData, uchar *msg, int lenMsg, int strtIdx, int *end { int i; + /* startIdx and seqLen always set if bytesLeft is set, + which is required before this function is called */ *endIdx = strtIdx + seqLen; if(*endIdx > lenMsg) *endIdx = lenMsg; @@ -222,16 +224,17 @@ static inline void doUTF8(instanceData *pData, uchar *msg, int lenMsg) { uchar c; - int8_t seqLen, bytesLeft = 0; + int8_t seqLen = 0, bytesLeft = 0; uint32_t codepoint; - int strtIdx, endIdx; + int strtIdx = 0, endIdx = 0; int i; for(i = 0 ; i < lenMsg ; ++i) { c = msg[i]; if(bytesLeft) { if((c & 0xc0) != 0x80) { - /* sequence invalid, invalidate all bytes */ + /* sequence invalid, invalidate all bytes + startIdx is always set if bytesLeft is set */ fixInvldMBSeq(pData, msg, lenMsg, strtIdx, &endIdx, seqLen); i = endIdx - 1; @@ -242,6 +245,8 @@ doUTF8(instanceData *pData, uchar *msg, int lenMsg) if(bytesLeft == 0) { /* too-large codepoint? */ if(codepoint > 0x10FFFF) { + /* sequence invalid, invalidate all bytes + startIdx is always set if bytesLeft is set */ fixInvldMBSeq(pData, msg, lenMsg, strtIdx, &endIdx, seqLen); @@ -285,12 +290,12 @@ doUTF8(instanceData *pData, uchar *msg, int lenMsg) } } -BEGINdoAction - msg_t *pMsg; +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; + msg_t *pMsg = ppMsg[0]; uchar *msg; int lenMsg; CODESTARTdoAction - pMsg = (msg_t*) ppString[0]; lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); if(pWrkrData->pData->mode == MODE_CC) { diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index cb2ad40a6..c17caaf78 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -4,7 +4,7 @@ * NOTE: read comments in module-template.h for more specifics! * * Copyright 2011 Nathan Scott. - * Copyright 2009-2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -38,6 +38,9 @@ #include #include #include +#if defined(__FreeBSD__) +#include +#endif #include "cJSON/cjson.h" #include "conf.h" #include "syslogd-types.h" @@ -49,6 +52,10 @@ #include "cfsysline.h" #include "unicode-helper.h" +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif + MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP MODULE_CNFNAME("omelasticsearch") @@ -148,7 +155,6 @@ ENDcreateInstance BEGINcreateWrkrInstance CODESTARTcreateWrkrInstance -dbgprintf("omelasticsearch: createWrkrInstance\n"); pWrkrData->restURL = NULL; if(pData->bulkmode) { pWrkrData->batch.currTpl1 = NULL; @@ -161,7 +167,6 @@ dbgprintf("omelasticsearch: createWrkrInstance\n"); } CHKiRet(curlSetup(pWrkrData, pWrkrData->pData)); finalize_it: -dbgprintf("DDDD: createWrkrInstance,pData %p/%p, pWrkrData %p\n", pData, pWrkrData->pData, pWrkrData); ENDcreateWrkrInstance BEGINisCompatibleWithFeature @@ -314,6 +319,8 @@ getIndexTypeAndParent(instanceData *pData, uchar **tpls, uchar **srchIndex, uchar **srchType, uchar **parent, uchar **bulkId) { + if(tpls == NULL) + return; if(pData->dynSrchIdx) { *srchIndex = tpls[1]; if(pData->dynSrchType) { @@ -380,7 +387,7 @@ static rsRetVal setCurlURL(wrkrInstanceData_t *pWrkrData, instanceData *pData, uchar **tpls) { char authBuf[1024]; - uchar *searchIndex; + uchar *searchIndex = 0; uchar *searchType; uchar *parent; uchar *bulkId; @@ -388,10 +395,12 @@ setCurlURL(wrkrInstanceData_t *pWrkrData, instanceData *pData, uchar **tpls) int rLocal; int r; DEFiRet; + char separator; + const int bulkmode = pData->bulkmode; setBaseURL(pData, &url); - if(pData->bulkmode) { + if(bulkmode) { r = es_addBuf(&url, "_bulk", sizeof("_bulk")-1); parent = NULL; } else { @@ -400,17 +409,23 @@ setCurlURL(wrkrInstanceData_t *pWrkrData, instanceData *pData, uchar **tpls) if(r == 0) r = es_addChar(&url, '/'); if(r == 0) r = es_addBuf(&url, (char*)searchType, ustrlen(searchType)); } - if(r == 0) r = es_addChar(&url, '?'); + + separator = '?'; if(pData->asyncRepl) { - if(r == 0) r = es_addBuf(&url, "replication=async&", - sizeof("replication=async&")-1); + if(r == 0) r = es_addChar(&url, separator); + if(r == 0) r = es_addBuf(&url, "replication=async", sizeof("replication=async")-1); + separator = '&'; } + if(pData->timeout != NULL) { + if(r == 0) r = es_addChar(&url, separator); if(r == 0) r = es_addBuf(&url, "timeout=", sizeof("timeout=")-1); if(r == 0) r = es_addBuf(&url, (char*)pData->timeout, ustrlen(pData->timeout)); - if(r == 0) r = es_addChar(&url, '&'); + separator = '&'; } + if(parent != NULL) { + if(r == 0) r = es_addChar(&url, separator); if(r == 0) r = es_addBuf(&url, "parent=", sizeof("parent=")-1); if(r == 0) es_addBuf(&url, (char*)parent, ustrlen(parent)); } @@ -447,9 +462,9 @@ buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) { int length = strlen((char *)message); int r; - uchar *searchIndex; + uchar *searchIndex = 0; uchar *searchType; - uchar *parent; + uchar *parent = NULL; uchar *bulkId = NULL; DEFiRet; # define META_STRT "{\"index\":{\"_index\": \"" @@ -522,7 +537,7 @@ getDataErrorDefault(wrkrInstanceData_t *pWrkrData,cJSON **pReplyRoot,uchar *reqm * Sections are marked by { and } */ static inline rsRetVal -getSection(const char* bulkRequest,char **bulkRequestNextSectionStart ) +getSection(const char* bulkRequest, const char **bulkRequestNextSectionStart ) { DEFiRet; char* index =0; @@ -545,24 +560,24 @@ getSection(const char* bulkRequest,char **bulkRequestNextSectionStart ) * and sets lastLocation pointer to the location till which bulkrequest has been parsed.(used as input to make function thread safe.) */ static inline rsRetVal -getSingleRequest(const char* bulkRequest, char** singleRequest ,char **lastLocation) +getSingleRequest(const char* bulkRequest, char** singleRequest, const char **lastLocation) { DEFiRet; - char *req = bulkRequest; - char *start = bulkRequest; + const char *req = bulkRequest; + const char *start = bulkRequest; if (getSection(req,&req)!=RS_RET_OK) ABORT_FINALIZE(RS_RET_ERR); if (getSection(req,&req)!=RS_RET_OK) ABORT_FINALIZE(RS_RET_ERR); - *singleRequest = (char*) calloc (req - start+ 1 + 1,sizeof(char));/* (req - start+ 1 == length of data + 1 for terminal char)*/ - if (*singleRequest==NULL) ABORT_FINALIZE(RS_RET_ERR); - memcpy(*singleRequest,start,req - start); - *lastLocation=req; + CHKmalloc(*singleRequest = (char*) calloc (req - start+ 1 + 1,1)); + /* (req - start+ 1 == length of data + 1 for terminal char)*/ + memcpy(*singleRequest,start,req - start); + *lastLocation=req; - finalize_it: - RETiRet; +finalize_it: + RETiRet; } /* @@ -608,7 +623,7 @@ parseRequestAndResponseForContext(wrkrInstanceData_t *pWrkrData,cJSON **pReplyRo numitems = cJSON_GetArraySize(items); DBGPRINTF("omelasticsearch: Entire request %s\n",reqmsg); - char *lastReqRead= (char*)reqmsg; + const char *lastReqRead= (char*)reqmsg; DBGPRINTF("omelasticsearch: %d items in reply\n", numitems); for(i = 0 ; i < numitems ; ++i) { @@ -654,7 +669,7 @@ parseRequestAndResponseForContext(wrkrInstanceData_t *pWrkrData,cJSON **pReplyRo response = cJSON_PrintUnformatted(create); - if(*response==NULL) + if(response==NULL) { free(request);/*as its has been assigned.*/ DBGPRINTF("omelasticsearch: Error getting cJSON_PrintUnformatted. Cannot continue\n"); @@ -719,8 +734,11 @@ getDataErrorOnly(context *ctx,int itemStatus,char *request,char *response) * Dumps all requests of bulk insert interleaved with request and response */ -static inline rsRetVal -getDataInterleaved(context *ctx,int itemStatus,char *request,char *response) +static rsRetVal +getDataInterleaved(context *ctx, + int __attribute__((unused)) itemStatus, + char *request, + char *response) { DEFiRet; cJSON *interleaved =0; @@ -1060,7 +1078,6 @@ finalize_it: BEGINbeginTransaction CODESTARTbeginTransaction -dbgprintf("omelasticsearch: beginTransaction, pWrkrData %p, pData %p\n", pWrkrData, pWrkrData->pData); if(!pWrkrData->pData->bulkmode) { FINALIZE; } @@ -1081,14 +1098,12 @@ CODESTARTdoAction ppString, 1)); } finalize_it: -dbgprintf("omelasticsearch: result doAction: %d (bulkmode %d)\n", iRet, pWrkrData->pData->bulkmode); ENDdoAction BEGINendTransaction char *cstr = NULL; CODESTARTendTransaction -dbgprintf("omelasticsearch: endTransaction init\n"); /* End Transaction only if batch data is not empty */ if (pWrkrData->batch.data != NULL ) { cstr = es_str2cstr(pWrkrData->batch.data, NULL); @@ -1099,7 +1114,6 @@ dbgprintf("omelasticsearch: endTransaction init\n"); dbgprintf("omelasticsearch: endTransaction, pWrkrData->batch.data is NULL, nothing to send. \n"); finalize_it: free(cstr); -dbgprintf("omelasticsearch: endTransaction done with %d\n", iRet); ENDendTransaction /* elasticsearch POST result string ... useful for debugging */ @@ -1209,7 +1223,7 @@ CODESTARTnewActInst }else if(!strcmp(actpblk.descr[i].name, "interleaved")) { pData->interleaved = pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "serverport")) { - pData->port = (int) pvals[i].val.d.n, NULL; + pData->port = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "uid")) { pData->uid = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "pwd")) { diff --git a/plugins/omhdfs/omhdfs.c b/plugins/omhdfs/omhdfs.c index d59725106..e43c0bce2 100644 --- a/plugins/omhdfs/omhdfs.c +++ b/plugins/omhdfs/omhdfs.c @@ -213,7 +213,7 @@ filePrepare(file_t *pFile) /* file does not exist, create it (and eventually parent directories */ if(1) { // check if bCreateDirs len = ustrlen(pFile->name) + 1; - CHKmalloc(pszWork = MALLOC(sizeof(uchar) * len)); + CHKmalloc(pszWork = MALLOC(len)); memcpy(pszWork, pFile->name, len); for(p = pszWork+1 ; *p ; p++) if(*p == '/') { @@ -312,7 +312,6 @@ fileWrite(file_t *pFile, uchar *buf, size_t *lenWrite) } } -dbgprintf("XXXXX: omhdfs writing %u bytes\n", *lenWrite); tSize num_written_bytes = hdfsWrite(pFile->fs, pFile->fh, buf, *lenWrite); if((unsigned) num_written_bytes != *lenWrite) { errmsg.LogError(errno, RS_RET_ERR_HDFS_WRITE, @@ -371,7 +370,6 @@ addData(instanceData *pData, uchar *buf) memcpy((char*) pData->ioBuf + pData->offsBuf, buf, len); pData->offsBuf += len; } else { -dbgprintf("XXXXX: not enough room, need to flush\n"); CHKiRet(fileWrite(pData->pFile, pData->ioBuf, &pData->offsBuf)); if(len >= sizeof(pData->ioBuf)) { CHKiRet(fileWrite(pData->pFile, buf, &len)); @@ -426,7 +424,7 @@ ENDtryResume BEGINbeginTransaction CODESTARTbeginTransaction -dbgprintf("omhdfs: beginTransaction\n"); + DBGPRINTF("omhdfs: beginTransaction\n"); ENDbeginTransaction @@ -444,7 +442,7 @@ ENDdoAction BEGINendTransaction instanceData *pData = pWrkrData->pData; CODESTARTendTransaction -dbgprintf("omhdfs: endTransaction\n"); + DBGPRINTF("omhdfs: endTransaction\n"); pthread_mutex_lock(&mutDoAct); if(pData->offsBuf != 0) { DBGPRINTF("omhdfs: data unwritten at end of transaction, persisting...\n"); diff --git a/plugins/omkafka/Makefile.am b/plugins/omkafka/Makefile.am index 48a88ae56..db77261c5 100644 --- a/plugins/omkafka/Makefile.am +++ b/plugins/omkafka/Makefile.am @@ -2,7 +2,7 @@ pkglib_LTLIBRARIES = omkafka.la omkafka_la_SOURCES = omkafka.c omkafka_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -omkafka_la_LDFLAGS = -module -avoid-version -lrdkafka +omkafka_la_LDFLAGS = -module -avoid-version $(LIBRDKAFKA_LIBS) omkafka_la_LIBADD = EXTRA_DIST = diff --git a/plugins/omkafka/omkafka.c b/plugins/omkafka/omkafka.c index d24970804..8666068c2 100644 --- a/plugins/omkafka/omkafka.c +++ b/plugins/omkafka/omkafka.c @@ -52,6 +52,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(statsobj) statsobj_t *kafkaStats; +int ctrQueueSize; STATSCOUNTER_DEF(ctrTopicSubmit, mutCtrTopicSubmit); STATSCOUNTER_DEF(ctrKafkaFail, mutCtrKafkaFail); STATSCOUNTER_DEF(ctrCacheMiss, mutCtrCacheMiss); @@ -113,7 +114,7 @@ typedef struct _instanceData { sbool autoPartition; int fixedPartition; int nPartitions; - int32_t currPartition; + uint32_t currPartition; int nConfParams; struct kafka_params *confParams; int nTopicConfParams; @@ -160,16 +161,16 @@ BEGINinitConfVars /* (re)set config variables to default values */ CODESTARTinitConfVars ENDinitConfVars -static inline int +static inline uint32_t getPartition(instanceData *const __restrict__ pData) { if (pData->autoPartition) { return RD_KAFKA_PARTITION_UA; } else { return (pData->fixedPartition == NO_FIXED_PARTITION) ? - ATOMIC_INC_AND_FETCH_int(&pData->currPartition, + ATOMIC_INC_AND_FETCH_unsigned(&pData->currPartition, &pData->mutCurrPartition) % pData->nPartitions - : pData->fixedPartition; + : (unsigned) pData->fixedPartition; } } @@ -830,6 +831,7 @@ finalize_it: if(iRet != RS_RET_OK) { iRet = RS_RET_SUSPENDED; } + STATSCOUNTER_SETMAX_NOMUT(ctrQueueSize, rd_kafka_outq_len(pData->rk)); STATSCOUNTER_INC(ctrTopicSubmit, mutCtrTopicSubmit); RETiRet; } @@ -938,18 +940,22 @@ CODESTARTnewActInst CHKmalloc(pData->confParams = malloc(sizeof(struct kafka_params) * pvals[i].val.d.ar->nmemb )); for(int j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { - CHKiRet(processKafkaParam(es_str2cstr(pvals[i].val.d.ar->arr[j], NULL), + char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + CHKiRet(processKafkaParam(cstr, &pData->confParams[j].name, &pData->confParams[j].val)); + free(cstr); } } else if(!strcmp(actpblk.descr[i].name, "topicconfparam")) { pData->nTopicConfParams = pvals[i].val.d.ar->nmemb; CHKmalloc(pData->topicConfParams = malloc(sizeof(struct kafka_params) * pvals[i].val.d.ar->nmemb )); for(int j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { - CHKiRet(processKafkaParam(es_str2cstr(pvals[i].val.d.ar->arr[j], NULL), + char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); + CHKiRet(processKafkaParam(cstr, &pData->topicConfParams[j].name, &pData->topicConfParams[j].val)); + free(cstr); } } else if(!strcmp(actpblk.descr[i].name, "errorfile")) { pData->errorFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); @@ -1055,6 +1061,9 @@ CODEmodInit_QueryRegCFSLineHdlr STATSCOUNTER_INIT(ctrTopicSubmit, mutCtrTopicSubmit); CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"submitted", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrTopicSubmit)); + ctrQueueSize = 0; + CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"maxoutqsize", + ctrType_Int, CTR_FLAG_NONE, &ctrQueueSize)); STATSCOUNTER_INIT(ctrKafkaFail, mutCtrKafkaFail); CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrKafkaFail)); diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 9b288c032..bd588755d 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -51,7 +51,7 @@ #include "conf.h" #undef HAVE_DBI_TXSUPP -#warning transaction support disabled in v8 -- TODO: reenable +/* transaction support disabled in v8 -- TODO: reenable */ MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP @@ -229,7 +229,7 @@ reportDBError(instanceData *pData, int bSilent) errmsg.LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain connection handle"); } else { /* we can ask dbi for the error description... */ uDBErrno = dbi_conn_error(pData->conn, &pszDbiErr); - snprintf(errMsg, sizeof(errMsg)/sizeof(char), "db error (%d): %s\n", uDBErrno, pszDbiErr); + snprintf(errMsg, sizeof(errMsg), "db error (%d): %s\n", uDBErrno, pszDbiErr); if(bSilent || uDBErrno == pData->uLastDBErrno) dbgprintf("libdbi, DBError(silent): %s\n", errMsg); else { @@ -617,7 +617,6 @@ INITLegCnfVars CODEmodInit_QueryRegCFSLineHdlr # ifndef HAVE_DBI_TXSUPP DBGPRINTF("omlibdbi: no transaction support in libdbi\n"); -# warning libdbi too old - transactions are not enabled (use 0.9 or later) # endif CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(regCfSysLineHdlr2((uchar *)"actionlibdbidriverdirectory", 0, eCmdHdlrGetWord, NULL, &cs.dbiDrvrDir, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index b80040156..e5a9f3e09 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -436,8 +436,8 @@ Send(int sock, char *msg, size_t len) DBGPRINTF("message not (smtp/tcp)send, errno %d", errno); ABORT_FINALIZE(RS_RET_TCP_SEND_ERROR); } - } else if(lenSend != (ssize_t) len) { - offsBuf += len; /* on to next round... */ + } else if(lenSend != (ssize_t) (len - offsBuf)) { + offsBuf += lenSend; /* on to next round... */ } else { FINALIZE; } @@ -502,7 +502,7 @@ finalize_it: /* read response line from server */ static rsRetVal -readResponseLn(wrkrInstanceData_t *pWrkrData, char *pLn, size_t lenLn) +readResponseLn(wrkrInstanceData_t *pWrkrData, char *pLn, size_t lenLn, size_t *const __restrict__ respLen) { DEFiRet; size_t i = 0; @@ -518,10 +518,11 @@ readResponseLn(wrkrInstanceData_t *pWrkrData, char *pLn, size_t lenLn) if(i < (lenLn - 1)) /* if line is too long, we simply discard the rest */ pLn[i++] = c; } while(1); - pLn[i] = '\0'; DBGPRINTF("smtp server response: %s\n", pLn); /* do not remove, this is helpful in troubleshooting SMTP probs! */ finalize_it: + pLn[i] = '\0'; + *respLen = i; RETiRet; } @@ -536,17 +537,16 @@ readResponse(wrkrInstanceData_t *pWrkrData, int *piState, int iExpected) DEFiRet; int bCont; char buf[128]; + size_t respLen; assert(pWrkrData != NULL); assert(piState != NULL); bCont = 1; do { - CHKiRet(readResponseLn(pWrkrData, buf, sizeof(buf))); - /* note: the code below is not 100% clean as we may have received less than 4 characters. - * However, as we have a fixed size this will not create a vulnerability. An error will - * also most likely be generated, so it is quite acceptable IMHO -- rgerhards, 2008-04-08 - */ + CHKiRet(readResponseLn(pWrkrData, buf, sizeof(buf), &respLen)); + if(respLen < 4) /* we treat too-short responses as error */ + ABORT_FINALIZE(RS_RET_SMTP_ERROR); if(buf[3] != '-') { /* last or only response line? */ bCont = 0; *piState = buf[0] - '0'; diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index ee3a2afac..7d13d9b9f 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -287,8 +287,8 @@ getDefaultBSON(msg_t *pMsg) /* TODO: move to datetime? Refactor in any case! rgerhards, 2012-03-30 */ ts_gen = (gint64) datetime.syslogTime2time_t(&pMsg->tTIMESTAMP) * 1000; /* ms! */ -dbgprintf("ommongodb: ts_gen is %lld\n", (long long) ts_gen); -dbgprintf("ommongodb: secfrac is %d, precision %d\n", pMsg->tTIMESTAMP.secfrac, pMsg->tTIMESTAMP.secfracPrecision); + DBGPRINTF("ommongodb: ts_gen is %lld\n", (long long) ts_gen); + DBGPRINTF("ommongodb: secfrac is %d, precision %d\n", pMsg->tTIMESTAMP.secfrac, pMsg->tTIMESTAMP.secfracPrecision); if(pMsg->tTIMESTAMP.secfracPrecision > 3) { secfrac = pMsg->tTIMESTAMP.secfrac / i10pow(pMsg->tTIMESTAMP.secfracPrecision - 3); } else if(pMsg->tTIMESTAMP.secfracPrecision < 3) { diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index 5a6fbb1bd..e0db8aecc 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -168,7 +168,7 @@ static void reportDBError(wrkrInstanceData_t *pWrkrData, int bSilent) errmsg.LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain MySQL handle"); } else { /* we can ask mysql for the error description... */ uMySQLErrno = mysql_errno(pWrkrData->hmysql); - snprintf(errMsg, sizeof(errMsg)/sizeof(char), "db error (%d): %s\n", uMySQLErrno, + snprintf(errMsg, sizeof(errMsg), "db error (%d): %s\n", uMySQLErrno, mysql_error(pWrkrData->hmysql)); if(bSilent || uMySQLErrno == pWrkrData->uLastMySQLErrno) dbgprintf("mysql, DBError(silent): %s\n", errMsg); @@ -205,7 +205,7 @@ static rsRetVal initMySQL(wrkrInstanceData_t *pWrkrData, int bSilent) int err=errno; if(fp==NULL){ char msg[512]; - snprintf(msg,sizeof(msg)/sizeof(char),"Could not open '%s' for reading",pData->configfile); + snprintf(msg,sizeof(msg),"Could not open '%s' for reading",pData->configfile); if(bSilent) { char errStr[512]; rs_strerror_r(err, errStr, sizeof(errStr)); @@ -332,7 +332,7 @@ CODESTARTnewActInst strncpy(pData->dbsrv, cstr, sizeof(pData->dbsrv)); free(cstr); } else if(!strcmp(actpblk.descr[i].name, "serverport")) { - pData->dbsrvPort = (int) pvals[i].val.d.n, NULL; + pData->dbsrvPort = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "db")) { cstr = es_str2cstr(pvals[i].val.d.estr, NULL); strncpy(pData->dbname, cstr, sizeof(pData->dbname)); @@ -366,7 +366,6 @@ CODESTARTnewActInst OMSR_RQD_TPL_OPT_SQL)); } CODE_STD_FINALIZERnewActInst -dbgprintf("XXXX: added param, iRet %d\n", iRet); cnfparamvalsDestruct(pvals, &actpblk); ENDnewActInst diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index e968e1d58..427926f4d 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -49,7 +49,6 @@ MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP -MODULE_CNFNAME("ompgsql") /* internal structures */ @@ -145,7 +144,7 @@ static void reportDBError(instanceData *pData, int bSilent) errmsg.LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain PgSQL handle"); } else { /* we can ask pgsql for the error description... */ ePgSQLStatus = PQstatus(pData->f_hpgsql); - snprintf(errMsg, sizeof(errMsg)/sizeof(char), "db error (%d): %s\n", ePgSQLStatus, + snprintf(errMsg, sizeof(errMsg), "db error (%d): %s\n", ePgSQLStatus, PQerrorMessage(pData->f_hpgsql)); if(bSilent || ePgSQLStatus == pData->eLastPgSQLStatus) dbgprintf("pgsql, DBError(silent): %s\n", errMsg); @@ -311,7 +310,6 @@ ENDdoAction BEGINendTransaction CODESTARTendTransaction iRet = writePgSQL((uchar*) "commit;", pWrkrData->pData); /* TODO: make user-configurable */ -dbgprintf("ompgsql: endTransaction\n"); ENDendTransaction #endif @@ -404,7 +402,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); -# warning: transaction support missing for v8 + /* TODO: transaction support missing for v8 */ bCoreSupportsBatching= 0; DBGPRINTF("ompgsql: transactions are not yet supported on v8\n"); diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index b3e74387a..2ca133b8f 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -139,7 +139,7 @@ CODESTARTfreeInstance for (i = 0; i < pData->iParams; i++) { free(pData->aParams[i]); } - free(pData->aParams); + free(pData->aParams); } ENDfreeInstance @@ -170,7 +170,6 @@ writeProgramOutput(wrkrInstanceData_t *__restrict__ const pWrkrData, char errStr[1024]; ssize_t r; -dbgprintf("omprog: writeProgramOutput, fd %d\n", pWrkrData->fdOutput); if(pWrkrData->fdOutput == -1) { pWrkrData->fdOutput = open((char*)pWrkrData->pData->outputFileName, O_WRONLY | O_APPEND | O_CREAT, 0600); @@ -282,10 +281,10 @@ execBinary(wrkrInstanceData_t *pWrkrData, int fdStdin, int fdStdOutErr) */ rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("omprog: failed to execute binary '%s': %s\n", - pWrkrData->pData->szBinary, errStr); + pWrkrData->pData->szBinary, errStr); openlog("rsyslogd", 0, LOG_SYSLOG); syslog(LOG_ERR, "omprog: failed to execute binary '%s': %s\n", - pWrkrData->pData->szBinary, errStr); + pWrkrData->pData->szBinary, errStr); } /* we should never reach this point, but if we do, we terminate */ @@ -512,8 +511,8 @@ CODESTARTnewActInst if(!pvals[i].bUsed) continue; if(!strcmp(actpblk.descr[i].name, "binary")) { - estrBinary = pvals[i].val.d.estr; - estrParams = NULL; + estrBinary = pvals[i].val.d.estr; + estrParams = NULL; /* Search for space */ c = es_getBufAddr(pvals[i].val.d.estr); @@ -521,18 +520,22 @@ CODESTARTnewActInst while(iCnt < es_strlen(pvals[i].val.d.estr) ) { if (c[iCnt] == ' ') { /* Split binary name from parameters */ - estrBinary = es_newStrFromSubStr ( pvals[i].val.d.estr, 0, iCnt ); - estrParams = es_newStrFromSubStr ( pvals[i].val.d.estr, iCnt+1, es_strlen(pvals[i].val.d.estr)); + estrBinary = es_newStrFromSubStr ( pvals[i].val.d.estr, 0, iCnt ); + estrParams = es_newStrFromSubStr ( pvals[i].val.d.estr, iCnt+1, es_strlen(pvals[i].val.d.estr)); break; } iCnt++; } /* Assign binary and params */ pData->szBinary = (uchar*)es_str2cstr(estrBinary, NULL); - dbgprintf("omprog: szBinary = '%s'\n", pData->szBinary); + DBGPRINTF("omprog: szBinary = '%s'\n", pData->szBinary); /* Check for Params! */ if (estrParams != NULL) { - dbgprintf("omprog: szParams = '%s'\n", es_str2cstr(estrParams, NULL) ); + if(Debug) { + char *params = es_str2cstr(estrParams, NULL); + dbgprintf("omprog: szParams = '%s'\n", params); + free(params); + } /* Count parameters if set */ c = es_getBufAddr(estrParams); /* Reset to beginning */ @@ -540,10 +543,10 @@ CODESTARTnewActInst iCnt = 0; while(iCnt < es_strlen(estrParams) ) { if (c[iCnt] == ' ' && c[iCnt-1] != '\\') - pData->iParams++; + pData->iParams++; iCnt++; } - dbgprintf("omprog: iParams = '%d'\n", pData->iParams); + DBGPRINTF("omprog: iParams = '%d'\n", pData->iParams); /* Create argv Array */ CHKmalloc(pData->aParams = malloc( (pData->iParams+1) * sizeof(char*))); /* One more for first param */ @@ -551,32 +554,32 @@ CODESTARTnewActInst /* Second Loop, create parameter array*/ c = es_getBufAddr(estrParams); /* Reset to beginning */ iCnt = iStr = iPrm = 0; - estrTmp = NULL; - bInQuotes = FALSE; + estrTmp = NULL; + bInQuotes = FALSE; /* Set first parameter to binary */ - pData->aParams[iPrm] = strdup((char*)pData->szBinary); - dbgprintf("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); - iPrm++; + pData->aParams[iPrm] = strdup((char*)pData->szBinary); + DBGPRINTF("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + iPrm++; while(iCnt < es_strlen(estrParams) ) { if ( c[iCnt] == ' ' && !bInQuotes ) { /* Copy into Param Array! */ - estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr); + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr); } else if ( iCnt+1 >= es_strlen(estrParams) ) { /* Copy rest of string into Param Array! */ - estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr+1); + estrTmp = es_newStrFromSubStr( estrParams, iStr, iCnt-iStr+1); } else if (c[iCnt] == '"') { /* switch inQuotes Mode */ - bInQuotes = !bInQuotes; + bInQuotes = !bInQuotes; } if ( estrTmp != NULL ) { - pData->aParams[iPrm] = es_str2cstr(estrTmp, NULL); + pData->aParams[iPrm] = es_str2cstr(estrTmp, NULL); iStr = iCnt+1; /* Set new start */ - dbgprintf("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); + DBGPRINTF("omprog: Param (%d): '%s'\n", iPrm, pData->aParams[iPrm]); es_deleteStr( estrTmp ); - estrTmp = NULL; + estrTmp = NULL; iPrm++; } @@ -584,7 +587,7 @@ CODESTARTnewActInst iCnt++; } /* NULL last parameter! */ - pData->aParams[iPrm] = NULL; + pData->aParams[iPrm] = NULL; } } else if(!strcmp(actpblk.descr[i].name, "output")) { @@ -592,28 +595,27 @@ CODESTARTnewActInst } else if(!strcmp(actpblk.descr[i].name, "forcesingleinstance")) { pData->bForceSingleInst = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "hup.signal")) { - const char *const signal = es_str2cstr(pvals[i].val.d.estr, NULL); - if(!strcmp(signal, "HUP")) + const char *const sig = es_str2cstr(pvals[i].val.d.estr, NULL); + if(!strcmp(sig, "HUP")) pData->iHUPForward = SIGHUP; - else if(!strcmp(signal, "USR1")) + else if(!strcmp(sig, "USR1")) pData->iHUPForward = SIGUSR1; - else if(!strcmp(signal, "USR2")) + else if(!strcmp(sig, "USR2")) pData->iHUPForward = SIGUSR2; - else if(!strcmp(signal, "INT")) + else if(!strcmp(sig, "INT")) pData->iHUPForward = SIGINT; - else if(!strcmp(signal, "TERM")) + else if(!strcmp(sig, "TERM")) pData->iHUPForward = SIGTERM; else { errmsg.LogError(0, RS_RET_CONF_PARAM_INVLD, - "omprog: hup.signal '%s' in hup.signal parameter", - signal); + "omprog: hup.signal '%s' in hup.signal parameter", sig); ABORT_FINALIZE(RS_RET_CONF_PARAM_INVLD); } - free((void*)signal); + free((void*)sig); } else if(!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else { - dbgprintf("omprog: program error, non-handled param '%s'\n", actpblk.descr[i].name); + DBGPRINTF("omprog: program error, non-handled param '%s'\n", actpblk.descr[i].name); } } diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index d58a04d47..1cf84f938 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -16,7 +16,7 @@ * * File begun on 2008-03-13 by RGerhards * - * Copyright 2008-2014 Adiscon GmbH. + * Copyright 2008-2016 Adiscon GmbH. * * This file is part of rsyslog. * @@ -259,8 +259,10 @@ CODESTARTfreeInstance free(pData->caCertFile); free(pData->myCertFile); free(pData->myPrivKeyFile); - for(i = 0 ; i < pData->permittedPeers.nmemb ; ++i) { - free(pData->permittedPeers.name[i]); + if(pData->permittedPeers.name != NULL) { + for(i = 0 ; i < pData->permittedPeers.nmemb ; ++i) { + free(pData->permittedPeers.name[i]); + } } ENDfreeInstance @@ -290,6 +292,7 @@ setInstParamDefaults(instanceData *pData) pData->caCertFile = NULL; pData->myCertFile = NULL; pData->myPrivKeyFile = NULL; + pData->permittedPeers.name = NULL; pData->permittedPeers.nmemb = 0; } @@ -340,7 +343,7 @@ CODESTARTnewActInst pData->permittedPeers.nmemb = pvals[i].val.d.ar->nmemb; CHKmalloc(pData->permittedPeers.name = malloc(sizeof(uchar*) * pData->permittedPeers.nmemb)); - for(j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) { + for(j = 0 ; j < pData->permittedPeers.nmemb ; ++j) { pData->permittedPeers.name[j] = (uchar*)es_str2cstr(pvals[i].val.d.ar->arr[j], NULL); } } else { @@ -442,7 +445,7 @@ finalize_it: BEGINbeginTransaction CODESTARTbeginTransaction -dbgprintf("omrelp: beginTransaction\n"); + DBGPRINTF("omrelp: beginTransaction\n"); if(!pWrkrData->bIsConnected) { CHKiRet(doConnect(pWrkrData)); } diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c index 770642cfc..0a0a96bea 100644 --- a/plugins/omruleset/omruleset.c +++ b/plugins/omruleset/omruleset.c @@ -10,7 +10,7 @@ * * File begun on 2009-11-02 by RGerhards * - * Copyright 2009-2013 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -125,10 +125,11 @@ ENDtryResume * rsyslog procesing we can not really slow down the producer any longer, as we already * work off a queue. So a delay would just block out execution for longer than needed. */ -BEGINdoAction +BEGINdoAction_NoStrings + msg_t **ppMsg = (msg_t **) pMsgData; msg_t *pMsg; CODESTARTdoAction - CHKmalloc(pMsg = MsgDup((msg_t*) ppString[0])); + CHKmalloc(pMsg = MsgDup(ppMsg[0])); DBGPRINTF(":omruleset: forwarding message %p to ruleset %s[%p]\n", pMsg, (char*) pWrkrData->pData->pszRulesetName, pWrkrData->pData->pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); diff --git a/plugins/pmciscoios/pmciscoios.c b/plugins/pmciscoios/pmciscoios.c index aec22fd1d..a656960e3 100644 --- a/plugins/pmciscoios/pmciscoios.c +++ b/plugins/pmciscoios/pmciscoios.c @@ -3,7 +3,7 @@ * * File begun on 2014-07-07 by RGerhards * - * Copyright 2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2014-2015 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -55,7 +55,8 @@ DEFobjCurrIf(datetime) /* parser instance parameters */ static struct cnfparamdescr parserpdescr[] = { - { "present.origin", eCmdHdlrBinary, 0 } + { "present.origin", eCmdHdlrBinary, 0 }, + { "present.xr", eCmdHdlrBinary, 0 } }; static struct cnfparamblk parserpblk = { CNFPARAMBLK_VERSION, @@ -65,6 +66,7 @@ static struct cnfparamblk parserpblk = struct instanceConf_s { int bOriginPresent; /* is ORIGIN field present? */ + int bXrPresent; /* is XR? */ }; BEGINisCompatibleWithFeature @@ -86,17 +88,26 @@ createInstance(instanceConf_t **pinst) DEFiRet; CHKmalloc(inst = MALLOC(sizeof(instanceConf_t))); inst->bOriginPresent = 0; + inst->bXrPresent = 0; *pinst = inst; finalize_it: RETiRet; } + +BEGINfreeParserInst +CODESTARTfreeParserInst + dbgprintf("pmciscoios: free parser instance %p\n", pInst); +ENDfreeParserInst + + BEGINnewParserInst - struct cnfparamvals *pvals; + struct cnfparamvals *pvals = NULL; int i; CODESTARTnewParserInst DBGPRINTF("newParserInst (pmciscoios)\n"); + inst = NULL; CHKiRet(createInstance(&inst)); if(lst == NULL) @@ -120,26 +131,28 @@ CODESTARTnewParserInst dbgprintf("pmciscoios: program error, non-handled " "param '%s'\n", parserpblk.descr[i].name); } + if(!strcmp(parserpblk.descr[i].name, "present.xr")) { + inst->bXrPresent = (int) pvals[i].val.d.n; + } else { + dbgprintf("pmciscoios: program error, non-handled " + "param '%s'\n", parserpblk.descr[i].name); + } } finalize_it: CODE_STD_FINALIZERnewParserInst if(lst != NULL) cnfparamvalsDestruct(pvals, &parserpblk); + if(iRet != RS_RET_OK) + freeParserInst(inst); ENDnewParserInst -BEGINfreeParserInst -CODESTARTfreeParserInst - dbgprintf("pmciscoios: free parser instance %p\n", pInst); -ENDfreeParserInst - - BEGINparse2 uchar *p2parse; long long msgcounter; int lenMsg; int i; - int iHostname; + int iHostname = 0; uchar bufParseTAG[512]; uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE]; /* used by origin */ CODESTARTparse2 @@ -184,7 +197,23 @@ CODESTARTparse2 p2parse += 2; } + /* XR RSP (optional) */ + if(pInst->bXrPresent) { + while( lenMsg > 1 + && !(*p2parse == ':')) { + --lenMsg; + ++p2parse; + } + /* delimiter check */ + if(lenMsg < 2) { + DBGPRINTF("pmciscoios: fail after XR: '%s'\n", p2parse); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + p2parse += 1; + } + /* TIMESTAMP */ + if(p2parse[0] == '*' || p2parse[0] == '.') p2parse++; if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg, PARSE3164_TZSTRING, NO_PERMIT_YEAR_AFTER_TIME) == RS_RET_OK) { if(pMsg->dfltTZ[0] != '\0') applyDfltTZ(&pMsg->tTIMESTAMP, pMsg->dfltTZ); @@ -194,6 +223,20 @@ CODESTARTparse2 } /* Note: date parser strips ": ", so we cannot do the delimiter check here */ + /* XR RSP (optional) */ + if(pInst->bXrPresent) { + while( lenMsg > 1 + && !(*p2parse == '%')) { + --lenMsg; + p2parse++; + } + /* delimiter check */ + if(lenMsg < 2) { + DBGPRINTF("pmciscoios: fail after XR tag search: '%s'\n", p2parse); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + } + /* parse SYSLOG TAG. must always start with '%', else we have a field mismatch */ if(lenMsg < 1 || *p2parse != '%') { DBGPRINTF("pmciscoios: fail at tag begin (no '%%'): '%s'\n", p2parse); @@ -206,6 +249,7 @@ CODESTARTparse2 --lenMsg; } /* delimiter check */ + if(pInst->bXrPresent) p2parse++; if(lenMsg < 2 || *p2parse != ':' || *(p2parse+1) != ' ') { DBGPRINTF("pmciscoios: fail after tag: '%s'\n", p2parse); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); @@ -217,6 +261,7 @@ CODESTARTparse2 /* if we reach this point, we have a wellformed message and can persist the values */ MsgSetTAG(pMsg, bufParseTAG, i); + /* if bOriginPresent !=0 iHostname gets initialized */ if(pInst->bOriginPresent) MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, iHostname); MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c index 61e2b061d..6bc279406 100644 --- a/plugins/pmlastmsg/pmlastmsg.c +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -10,7 +10,7 @@ * * File begun on 2010-07-13 by RGerhards * - * Copyright 2014-2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2014-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -91,16 +91,13 @@ CODESTARTparse --lenMsg; ++p2parse; } -dbgprintf("pmlastmsg: msg to look at: [%d]'%s'\n", lenMsg, p2parse); if((unsigned) lenMsg < sizeof(OpeningText)-1 + sizeof(ClosingText)-1 + 1) { /* too short, can not be "our" message */ -dbgprintf("msg too short!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { /* wrong opening text */ -dbgprintf("wrong opening text!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } lenMsg -= sizeof(OpeningText) - 1; @@ -119,9 +116,6 @@ dbgprintf("wrong opening text!\n"); if(strncasecmp((char*) p2parse, ClosingText, lenMsg) != 0) { /* wrong closing text */ -dbgprintf("strcasecmp: %d\n", strncasecmp((char*) p2parse, ClosingText, lenMsg)); -dbgprintf("pmlastmsg: closing msg to look at: [%d]'%s', (%s)\n", lenMsg, p2parse, ClosingText); -dbgprintf("wrong closing text!\n"); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); } diff --git a/runtime/Makefile.am b/runtime/Makefile.am index d02ed4f34..46f9e7b07 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -55,6 +55,9 @@ librsyslog_la_SOURCES = \ modules.h \ statsobj.c \ statsobj.h \ + dynstats.c \ + dynstats.h \ + statsobj.h \ stream.c \ stream.h \ var.c \ diff --git a/runtime/atomic.h b/runtime/atomic.h index d59858a12..cf80abce2 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -194,6 +194,7 @@ */ #ifdef HAVE_ATOMIC_BUILTINS64 # define ATOMIC_INC_uint64(data, phlpmut) ((void) __sync_fetch_and_add(data, 1)) +# define ATOMIC_ADD_uint64(data, phlpmut, value) ((void) __sync_fetch_and_add(data, value)) # define ATOMIC_DEC_unit64(data, phlpmut) ((void) __sync_sub_and_fetch(data, 1)) # define ATOMIC_INC_AND_FETCH_uint64(data, phlpmut) __sync_fetch_and_add(data, 1) @@ -206,6 +207,11 @@ ++(*(data)); \ pthread_mutex_unlock(phlpmut); \ } +# define ATOMIC_ADD_uint64(data, phlpmut, value) { \ + pthread_mutex_lock(phlpmut); \ + *data += value; \ + pthread_mutex_unlock(phlpmut); \ + } # define ATOMIC_DEC_uint64(data, phlpmut) { \ pthread_mutex_lock(phlpmut); \ --(*(data)); \ diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 68e898cd6..242f8611f 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -319,7 +319,7 @@ static int doParseOnOffOption(uchar **pp) pOptStart = *pp; skipWhiteSpace(pp); /* skip over any whitespace */ - if(getSubString(pp, (char*) szOpt, sizeof(szOpt) / sizeof(uchar), ' ') != 0) { + if(getSubString(pp, (char*) szOpt, sizeof(szOpt), ' ') != 0) { errmsg.LogError(0, NO_ERRCODE, "Invalid $-configline - could not extract on/off option"); return -1; } @@ -351,7 +351,7 @@ static rsRetVal doGetGID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p assert(pp != NULL); assert(*pp != NULL); - if(getSubString(pp, (char*) szName, sizeof(szName) / sizeof(uchar), ' ') != 0) { + if(getSubString(pp, (char*) szName, sizeof(szName), ' ') != 0) { errmsg.LogError(0, RS_RET_NOT_FOUND, "could not extract group name"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } @@ -407,7 +407,7 @@ static rsRetVal doGetUID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p assert(pp != NULL); assert(*pp != NULL); - if(getSubString(pp, (char*) szName, sizeof(szName) / sizeof(uchar), ' ') != 0) { + if(getSubString(pp, (char*) szName, sizeof(szName), ' ') != 0) { errmsg.LogError(0, RS_RET_NOT_FOUND, "could not extract user name"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } diff --git a/runtime/conf.c b/runtime/conf.c index d171f7665..2f97b08f6 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -107,7 +107,7 @@ doModLoad(uchar **pp, __attribute__((unused)) void* pVal) ASSERT(*pp != NULL); skipWhiteSpace(pp); /* skip over any whitespace */ - if(getSubString(pp, (char*) szName, sizeof(szName) / sizeof(uchar), ' ') != 0) { + if(getSubString(pp, (char*) szName, sizeof(szName), ' ') != 0) { errmsg.LogError(0, RS_RET_NOT_FOUND, "could not extract module name"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } @@ -168,7 +168,7 @@ doNameLine(uchar **pp, void* pVal) eDir = (enum eDirective) pVal; /* this time, it actually is NOT a pointer! */ - if(getSubString(&p, szName, sizeof(szName) / sizeof(char), ',') != 0) { + if(getSubString(&p, szName, sizeof(szName), ',') != 0) { errmsg.LogError(0, RS_RET_NOT_FOUND, "Invalid config line: could not extract name - line ignored"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } @@ -222,7 +222,7 @@ cfsysline(uchar *p) ASSERT(p != NULL); errno = 0; - if(getSubString(&p, (char*) szCmd, sizeof(szCmd) / sizeof(uchar), ' ') != 0) { + if(getSubString(&p, (char*) szCmd, sizeof(szCmd), ' ') != 0) { errmsg.LogError(0, RS_RET_NOT_FOUND, "Invalid $-configline - could not extract command - line ignored\n"); ABORT_FINALIZE(RS_RET_NOT_FOUND); } diff --git a/runtime/datetime.c b/runtime/datetime.c index 0ecccd64e..efb4c8156 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -90,7 +90,7 @@ static const time_t yearInSecs[] = { * Convert struct timeval to syslog_time */ void -timeval2syslogTime(struct timeval *tp, struct syslogTime *t) +timeval2syslogTime(struct timeval *tp, struct syslogTime *t, const int inUTC) { struct tm *tm; struct tm tmBuf; @@ -98,7 +98,10 @@ timeval2syslogTime(struct timeval *tp, struct syslogTime *t) time_t secs; secs = tp->tv_sec; - tm = localtime_r(&secs, &tmBuf); + if(inUTC) + tm = gmtime_r(&secs, &tmBuf); + else + tm = localtime_r(&secs, &tmBuf); t->year = tm->tm_year + 1900; t->month = tm->tm_mon + 1; @@ -109,24 +112,30 @@ timeval2syslogTime(struct timeval *tp, struct syslogTime *t) t->secfrac = tp->tv_usec; t->secfracPrecision = 6; -# if __sun - /* Solaris uses a different method of exporting the time zone. - * It is UTC - localtime, which is the opposite sign of mins east of GMT. - */ - lBias = -(tm->tm_isdst ? altzone : timezone); -# elif defined(__hpux) - lBias = tz.tz_dsttime ? - tz.tz_minuteswest : 0; -# else - lBias = tm->tm_gmtoff; -# endif - if(lBias < 0) { - t->OffsetMode = '-'; - lBias *= -1; - } else + if(inUTC) { t->OffsetMode = '+'; + lBias = 0; + } else { +# if __sun + /* Solaris uses a different method of exporting the time zone. + * It is UTC - localtime, which is the opposite sign of mins east of GMT. + */ + lBias = -(tm->tm_isdst ? altzone : timezone); +# elif defined(__hpux) + lBias = tz.tz_dsttime ? - tz.tz_minuteswest : 0; +# else + lBias = tm->tm_gmtoff; +# endif + if(lBias < 0) { + t->OffsetMode = '-'; + lBias *= -1; + } else + t->OffsetMode = '+'; + } t->OffsetHour = lBias / 3600; t->OffsetMinute = (lBias % 3600) / 60; t->timeType = TIME_TYPE_RFC5424; /* we have a high precision timestamp */ + t->inUTC = inUTC; } /** @@ -145,7 +154,7 @@ timeval2syslogTime(struct timeval *tp, struct syslogTime *t) * in some situations to minimize time() calls (namely when doing * output processing). This can be left NULL if not needed. */ -static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) +static void getCurrTime(struct syslogTime *t, time_t *ttSeconds, const int inUTC) { struct timeval tp; # if defined(__hpux) @@ -164,7 +173,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) if(ttSeconds != NULL) *ttSeconds = tp.tv_sec; - timeval2syslogTime(&tp, t); + timeval2syslogTime(&tp, t, inUTC); } diff --git a/runtime/datetime.h b/runtime/datetime.h index 3a05b1a7d..90ce941af 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -32,7 +32,7 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ - void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); + void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds, const int inUTC); rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS, int*); rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS, int*, const int bParseTZ, const int bDetectYearAfterTime); int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst); @@ -42,13 +42,13 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); /* v3, 2009-11-12 */ time_t (*GetTime)(time_t *ttSeconds); - /* v6, 2011-06-20 */ - void (*timeval2syslogTime)(struct timeval *tp, struct syslogTime *t); + /* v6, 2011-06-20 , v10, 2016-01-12*/ + void (*timeval2syslogTime)(struct timeval *tp, struct syslogTime *t, const int inUTC); /* v7, 2012-03-29 */ int (*formatTimestampUnix)(struct syslogTime *ts, char*pBuf); time_t (*syslogTime2time_t)(struct syslogTime *ts); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 10 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as @@ -60,6 +60,8 @@ ENDinterface(datetime) * 6 - see above * 8 - ParseTIMESTAMP3164 has addtl parameter to permit TZ string parsing * 9 - ParseTIMESTAMP3164 has addtl parameter to permit year parsing + * 10 - functions having addtl paramater inUTC to emit time in UTC: + * timeval2syslogTime, getCurrtime */ #define PARSE3164_TZSTRING 1 @@ -68,6 +70,12 @@ ENDinterface(datetime) #define PERMIT_YEAR_AFTER_TIME 1 #define NO_PERMIT_YEAR_AFTER_TIME 0 +/* two defines for functions that create timestamps either in local + * time or UTC. + */ +#define TIME_IN_UTC 1 +#define TIME_IN_LOCALTIME 0 + /* prototypes */ PROTOTYPEObj(datetime); void applyDfltTZ(struct syslogTime *pTime, char *tz); diff --git a/runtime/debug.c b/runtime/debug.c index bfdfa3414..d92043ea4 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -367,7 +367,7 @@ static void dbgMutLogPrintOne(dbgMutLog_t *pLog) strmutop = "owned"; break; default: - snprintf(buf, sizeof(buf)/sizeof(char), "unknown state %d - should not happen!", pLog->mutexOp); + snprintf(buf, sizeof(buf), "unknown state %d - should not happen!", pLog->mutexOp); strmutop = buf; break; } @@ -465,7 +465,7 @@ static inline void dbgMutexPreLockLog(pthread_mutex_t *pmut, dbgFuncDB_t *pFuncD pszHolder = "[NONE]"; else { dbgGetThrdName(pszHolderThrdName, sizeof(pszHolderThrdName), pHolder->thrd, 1); - snprintf(pszBuf, sizeof(pszBuf)/sizeof(char), "%s:%d [%s]", pHolder->pFuncDB->file, pHolder->lockLn, pszHolderThrdName); + snprintf(pszBuf, sizeof(pszBuf), "%s:%d [%s]", pHolder->pFuncDB->file, pHolder->lockLn, pszHolderThrdName); pszHolder = pszBuf; } @@ -512,7 +512,7 @@ static inline void dbgMutexPreTryLockLog(pthread_mutex_t *pmut, dbgFuncDB_t *pFu pszHolder = "[NONE]"; else { dbgGetThrdName(pszHolderThrdName, sizeof(pszHolderThrdName), pHolder->thrd, 1); - snprintf(pszBuf, sizeof(pszBuf)/sizeof(char), "%s:%d [%s]", pHolder->pFuncDB->file, pHolder->lockLn, pszHolderThrdName); + snprintf(pszBuf, sizeof(pszBuf), "%s:%d [%s]", pHolder->pFuncDB->file, pHolder->lockLn, pszHolderThrdName); pszHolder = pszBuf; } @@ -1226,7 +1226,7 @@ dbgGetRTOptNamVal(uchar **ppszOpt, uchar **ppOptName, uchar **ppOptVal) /* name - up until '=' or whitespace */ i = 0; - while(i < (sizeof(optname)/sizeof(uchar) - 1) && *p && *p != '=' && !isspace(*p)) { + while(i < (sizeof(optname) - 1) && *p && *p != '=' && !isspace(*p)) { optname[i++] = *p++; } @@ -1237,7 +1237,7 @@ dbgGetRTOptNamVal(uchar **ppszOpt, uchar **ppOptName, uchar **ppOptVal) /* we have a value, get it */ ++p; i = 0; - while(i < (sizeof(optval)/sizeof(uchar) - 1) && *p && !isspace(*p)) { + while(i < (sizeof(optval) - 1) && *p && !isspace(*p)) { optval[i++] = *p++; } optval[i] = '\0'; diff --git a/runtime/dnscache.c b/runtime/dnscache.c index e44813822..1d50311d3 100644 --- a/runtime/dnscache.c +++ b/runtime/dnscache.c @@ -305,7 +305,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) * is OK in any way. We do also log the error message. rgerhards, 2007-07-16 */ if(glbl.GetDropMalPTRMsgs() == 1) { - snprintf((char*)szErrMsg, sizeof(szErrMsg) / sizeof(uchar), + snprintf((char*)szErrMsg, sizeof(szErrMsg), "Malicious PTR record, message dropped " "IP = \"%s\" HOST = \"%s\"", szIP, fqdnBuf); @@ -320,7 +320,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) * (OK, I admit this is more or less impossible, but I am paranoid...) * rgerhards, 2007-07-16 */ - snprintf((char*)szErrMsg, sizeof(szErrMsg) / sizeof(uchar), + snprintf((char*)szErrMsg, sizeof(szErrMsg), "Malicious PTR record (message accepted, but used IP " "instead of PTR name: IP = \"%s\" HOST = \"%s\"", szIP, fqdnBuf); diff --git a/runtime/dynstats.c b/runtime/dynstats.c new file mode 100644 index 000000000..aba005aad --- /dev/null +++ b/runtime/dynstats.c @@ -0,0 +1,541 @@ +/* + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "srUtils.h" +#include "errmsg.h" +#include "rsconf.h" +#include "unicode-helper.h" + +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(statsobj) + +#define DYNSTATS_PARAM_NAME "name" +#define DYNSTATS_PARAM_RESETTABLE "resettable" +#define DYNSTATS_PARAM_MAX_CARDINALITY "maxCardinality" +#define DYNSTATS_PARAM_UNUSED_METRIC_LIFE "unusedMetricLife" /* in seconds */ + +#define DYNSTATS_DEFAULT_RESETTABILITY 1 +#define DYNSTATS_DEFAULT_MAX_CARDINALITY 2000 +#define DYNSTATS_DEFAULT_UNUSED_METRIC_LIFE 3600 /* seconds */ + +#define DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH 100 +#define DYNSTATS_METRIC_NAME_SEPARATOR '.' +#define DYNSTATS_HASHTABLE_SIZE_OVERPROVISIONING 1.25 + +static struct cnfparamdescr modpdescr[] = { + { DYNSTATS_PARAM_NAME, eCmdHdlrString, CNFPARAM_REQUIRED }, + { DYNSTATS_PARAM_RESETTABLE, eCmdHdlrBinary, 0 }, + { DYNSTATS_PARAM_MAX_CARDINALITY, eCmdHdlrPositiveInt, 0}, + { DYNSTATS_PARAM_UNUSED_METRIC_LIFE, eCmdHdlrPositiveInt, 0} /* in minutes */ +}; + +static struct cnfparamblk modpblk = +{ CNFPARAMBLK_VERSION, + sizeof(modpdescr)/sizeof(struct cnfparamdescr), + modpdescr +}; + +rsRetVal +dynstatsClassInit(void) { + DEFiRet; + CHKiRet(objGetObjInterface(&obj)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(statsobj, CORE_COMPONENT)); +finalize_it: + RETiRet; +} + +static inline void +dynstats_destroyCtr(dynstats_bucket_t *b, dynstats_ctr_t *ctr, uint8_t destructStatsCtr) { + if (destructStatsCtr) { + statsobj.DestructCounter(b->stats, ctr->pCtr); + } + free(ctr->metric); + free(ctr); +} + +static inline void /* assumes exclusive access to bucket */ +dynstats_destroyCounters(dynstats_bucket_t *b) { + dynstats_ctr_t *ctr; + + hashtable_destroy(b->table, 0); + statsobj.DestructAllCounters(b->stats); + while(1) { + ctr = b->ctrs; + if (ctr == NULL) { + break; + } else { + b->ctrs = ctr->next; + dynstats_destroyCtr(b, ctr, 0); + } + } + STATSCOUNTER_BUMP(b->ctrMetricsPurged, b->mutCtrMetricsPurged, b->metricCount); +} + +void +dynstats_destroyBucket(dynstats_bucket_t* b) { + dynstats_buckets_t *bkts; + + bkts = &loadConf->dynstats_buckets; + + pthread_rwlock_wrlock(&b->lock); + dynstats_destroyCounters(b); + statsobj.Destruct(&b->stats); + free(b->name); + pthread_rwlock_unlock(&b->lock); + pthread_rwlock_destroy(&b->lock); + pthread_mutex_destroy(&b->mutMetricCount); + statsobj.DestructCounter(bkts->global_stats, b->pOpsOverflowCtr); + statsobj.DestructCounter(bkts->global_stats, b->pNewMetricAddCtr); + statsobj.DestructCounter(bkts->global_stats, b->pNoMetricCtr); + statsobj.DestructCounter(bkts->global_stats, b->pMetricsPurgedCtr); + statsobj.DestructCounter(bkts->global_stats, b->pOpsIgnoredCtr); + free(b); +} + +static rsRetVal +dynstats_addBucketMetrics(dynstats_buckets_t *bkts, dynstats_bucket_t *b, const uchar* name) { + uchar *metric_name_buff, *metric_suffix; + const uchar *suffix_litteral; + int name_len; + DEFiRet; + + name_len = ustrlen(name); + CHKmalloc(metric_name_buff = malloc((name_len + DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH + 1) * sizeof(uchar))); + + ustrncpy(metric_name_buff, name, name_len); + metric_suffix = metric_name_buff + name_len; + *metric_suffix = DYNSTATS_METRIC_NAME_SEPARATOR; + metric_suffix++; + + suffix_litteral = UCHAR_CONSTANT("ops_overflow"); + ustrncpy(metric_suffix, suffix_litteral, DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH); + STATSCOUNTER_INIT(b->ctrOpsOverflow, b->mutCtrOpsOverflow); + CHKiRet(statsobj.AddManagedCounter(bkts->global_stats, metric_name_buff, ctrType_IntCtr, + CTR_FLAG_RESETTABLE, &(b->ctrOpsOverflow), &b->pOpsOverflowCtr)); + + suffix_litteral = UCHAR_CONSTANT("new_metric_add"); + ustrncpy(metric_suffix, suffix_litteral, DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH); + STATSCOUNTER_INIT(b->ctrNewMetricAdd, b->mutCtrNewMetricAdd); + CHKiRet(statsobj.AddManagedCounter(bkts->global_stats, metric_name_buff, ctrType_IntCtr, + CTR_FLAG_RESETTABLE, &(b->ctrNewMetricAdd), &b->pNewMetricAddCtr)); + + suffix_litteral = UCHAR_CONSTANT("no_metric"); + ustrncpy(metric_suffix, suffix_litteral, DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH); + STATSCOUNTER_INIT(b->ctrNoMetric, b->mutCtrNoMetric); + CHKiRet(statsobj.AddManagedCounter(bkts->global_stats, metric_name_buff, ctrType_IntCtr, + CTR_FLAG_RESETTABLE, &(b->ctrNoMetric), &b->pNoMetricCtr)); + + suffix_litteral = UCHAR_CONSTANT("metrics_purged"); + ustrncpy(metric_suffix, suffix_litteral, DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH); + STATSCOUNTER_INIT(b->ctrMetricsPurged, b->mutCtrMetricsPurged); + CHKiRet(statsobj.AddManagedCounter(bkts->global_stats, metric_name_buff, ctrType_IntCtr, + CTR_FLAG_RESETTABLE, &(b->ctrMetricsPurged), &b->pMetricsPurgedCtr)); + + suffix_litteral = UCHAR_CONSTANT("ops_ignored"); + ustrncpy(metric_suffix, suffix_litteral, DYNSTATS_MAX_BUCKET_NS_METRIC_LENGTH); + STATSCOUNTER_INIT(b->ctrOpsIgnored, b->mutCtrOpsIgnored); + CHKiRet(statsobj.AddManagedCounter(bkts->global_stats, metric_name_buff, ctrType_IntCtr, + CTR_FLAG_RESETTABLE, &(b->ctrOpsIgnored), &b->pOpsIgnoredCtr)); + +finalize_it: + free(metric_name_buff); + if (iRet != RS_RET_OK) { + if (b->pOpsOverflowCtr != NULL) { + statsobj.DestructCounter(bkts->global_stats, b->pOpsOverflowCtr); + } + if (b->pNewMetricAddCtr != NULL) { + statsobj.DestructCounter(bkts->global_stats, b->pNewMetricAddCtr); + } + if (b->pNoMetricCtr != NULL) { + statsobj.DestructCounter(bkts->global_stats, b->pNoMetricCtr); + } + if (b->pMetricsPurgedCtr != NULL) { + statsobj.DestructCounter(bkts->global_stats, b->pMetricsPurgedCtr); + } + if (b->pOpsIgnoredCtr != NULL) { + statsobj.DestructCounter(bkts->global_stats, b->pOpsIgnoredCtr); + } + } + RETiRet; +} + +static void +no_op_free(void __attribute__((unused)) *ignore) {} + +static rsRetVal +dynstats_resetBucket(dynstats_bucket_t *b, uint8_t do_purge) { + size_t htab_sz; + DEFiRet; + htab_sz = (size_t) (DYNSTATS_HASHTABLE_SIZE_OVERPROVISIONING * b->maxCardinality + 1); + pthread_rwlock_wrlock(&b->lock); + if (do_purge) { + dynstats_destroyCounters(b); + } + ATOMIC_STORE_0_TO_INT(&b->metricCount, &b->mutMetricCount); + b->ctrs = NULL; + if ((b->table = create_hashtable(htab_sz, hash_from_string, key_equals_string, no_op_free)) == NULL) { + errmsg.LogError(errno, RS_RET_INTERNAL_ERROR, "error trying to initialize hash-table for dyn-stats bucket named: %s", b->name); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } + + timeoutComp(&b->metricCleanupTimeout, b->unusedMetricLife); +finalize_it: + pthread_rwlock_unlock(&b->lock); + if (iRet != RS_RET_OK) { + statsobj.Destruct(&b->stats); + } + RETiRet; +} + +static inline void +dynstats_resetIfExpired(dynstats_bucket_t *b) { + long timeout; + pthread_rwlock_rdlock(&b->lock); + timeout = timeoutVal(&b->metricCleanupTimeout); + pthread_rwlock_unlock(&b->lock); + if (timeout == 0) { + errmsg.LogMsg(0, RS_RET_TIMED_OUT, LOG_INFO, "dynstats: bucket '%s' is being reset", b->name); + dynstats_resetBucket(b, 1); + } +} + +static void +dynstats_readCallback(statsobj_t __attribute__((unused)) *ignore, void *b) { + dynstats_buckets_t *bkts; + bkts = &loadConf->dynstats_buckets; + + pthread_rwlock_rdlock(&bkts->lock); + dynstats_resetIfExpired((dynstats_bucket_t *) b); + pthread_rwlock_unlock(&bkts->lock); +} + +static inline rsRetVal +dynstats_initNewBucketStats(dynstats_bucket_t *b) { + DEFiRet; + + CHKiRet(statsobj.Construct(&b->stats)); + CHKiRet(statsobj.SetOrigin(b->stats, UCHAR_CONSTANT("dynstats.bucket"))); + CHKiRet(statsobj.SetName(b->stats, b->name)); + statsobj.SetReadNotifier(b->stats, dynstats_readCallback, b); + CHKiRet(statsobj.ConstructFinalize(b->stats)); + +finalize_it: + RETiRet; +} + +static rsRetVal +dynstats_newBucket(const uchar* name, uint8_t resettable, uint32_t maxCardinality, uint32_t unusedMetricLife) { + dynstats_bucket_t *b; + dynstats_buckets_t *bkts; + uint8_t lock_initialized, metric_count_mutex_initialized; + pthread_rwlockattr_t bucket_lock_attr; + DEFiRet; + + lock_initialized = metric_count_mutex_initialized = 0; + b = NULL; + + bkts = &loadConf->dynstats_buckets; + + if (bkts->initialized) { + CHKmalloc(b = calloc(1, sizeof(dynstats_bucket_t))); + b->resettable = resettable; + b->maxCardinality = maxCardinality; + b->unusedMetricLife = 1000 * unusedMetricLife; + CHKmalloc(b->name = ustrdup(name)); + + pthread_rwlockattr_init(&bucket_lock_attr); +#ifdef HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP + pthread_rwlockattr_setkind_np(&bucket_lock_attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); +#endif + + pthread_rwlock_init(&b->lock, &bucket_lock_attr); + lock_initialized = 1; + pthread_mutex_init(&b->mutMetricCount, NULL); + metric_count_mutex_initialized = 1; + + CHKiRet(dynstats_initNewBucketStats(b)); + + CHKiRet(dynstats_resetBucket(b, 0)); + + CHKiRet(dynstats_addBucketMetrics(bkts, b, name)); + + pthread_rwlock_wrlock(&bkts->lock); + if (bkts->list == NULL) { + bkts->list = b; + } else { + b->next = bkts->list; + bkts->list = b; + } + pthread_rwlock_unlock(&bkts->lock); + } else { + errmsg.LogError(0, RS_RET_INTERNAL_ERROR, "dynstats: bucket creation failed, as global-initialization of buckets was unsuccessful"); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } +finalize_it: + if (iRet != RS_RET_OK) { + if (metric_count_mutex_initialized) { + pthread_mutex_destroy(&b->mutMetricCount); + } + if (lock_initialized) { + pthread_rwlock_destroy(&b->lock); + } + if (b != NULL) { + free(b->name); + free(b); + } + } + RETiRet; +} + +rsRetVal +dynstats_processCnf(struct cnfobj *o) { + struct cnfparamvals *pvals; + short i; + uchar *name = NULL; + uint8_t resettable = DYNSTATS_DEFAULT_RESETTABILITY; + uint32_t maxCardinality = DYNSTATS_DEFAULT_MAX_CARDINALITY; + uint32_t unusedMetricLife = DYNSTATS_DEFAULT_UNUSED_METRIC_LIFE; + DEFiRet; + + pvals = nvlstGetParams(o->nvlst, &modpblk, NULL); + if(pvals == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + + for(i = 0 ; i < modpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(modpblk.descr[i].name, DYNSTATS_PARAM_NAME)) { + CHKmalloc(name = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL)); + } else if (!strcmp(modpblk.descr[i].name, DYNSTATS_PARAM_RESETTABLE)) { + resettable = (pvals[i].val.d.n != 0); + } else if (!strcmp(modpblk.descr[i].name, DYNSTATS_PARAM_MAX_CARDINALITY)) { + maxCardinality = (uint32_t) pvals[i].val.d.n; + } else if (!strcmp(modpblk.descr[i].name, DYNSTATS_PARAM_UNUSED_METRIC_LIFE)) { + unusedMetricLife = (uint32_t) pvals[i].val.d.n; + } else { + dbgprintf("dyn_stats: program error, non-handled " + "param '%s'\n", modpblk.descr[i].name); + } + } + if (name != NULL) { + CHKiRet(dynstats_newBucket(name, resettable, maxCardinality, unusedMetricLife)); + } + +finalize_it: + free(name); + cnfparamvalsDestruct(pvals, &modpblk); + RETiRet; +} + +rsRetVal +dynstats_initCnf(dynstats_buckets_t *bkts) { + DEFiRet; + + bkts->initialized = 0; + + bkts->list = NULL; + CHKiRet(statsobj.Construct(&bkts->global_stats)); + CHKiRet(statsobj.SetOrigin(bkts->global_stats, UCHAR_CONSTANT("dynstats"))); + CHKiRet(statsobj.SetName(bkts->global_stats, UCHAR_CONSTANT("global"))); + CHKiRet(statsobj.ConstructFinalize(bkts->global_stats)); + pthread_rwlock_init(&bkts->lock, NULL); + + bkts->initialized = 1; + +finalize_it: + if (iRet != RS_RET_OK) { + statsobj.Destruct(&bkts->global_stats); + } + RETiRet; +} + +void +dynstats_destroyAllBuckets() { + dynstats_buckets_t *bkts; + dynstats_bucket_t *b; + bkts = &loadConf->dynstats_buckets; + if (bkts->initialized) { + pthread_rwlock_wrlock(&bkts->lock); + while(1) { + b = bkts->list; + if (b == NULL) { + break; + } else { + bkts->list = b->next; + dynstats_destroyBucket(b); + } + } + pthread_rwlock_unlock(&bkts->lock); + pthread_rwlock_destroy(&bkts->lock); + } +} + +dynstats_bucket_t * +dynstats_findBucket(const uchar* name) { + dynstats_buckets_t *bkts; + dynstats_bucket_t *b; + bkts = &loadConf->dynstats_buckets; + if (bkts->initialized) { + pthread_rwlock_rdlock(&bkts->lock); + b = bkts->list; + while(b != NULL) { + if (! ustrcmp(name, b->name)) { + break; + } + b = b->next; + } + pthread_rwlock_unlock(&bkts->lock); + } else { + b = NULL; + errmsg.LogError(0, RS_RET_INTERNAL_ERROR, "dynstats: bucket lookup failed, as global-initialization of buckets was unsuccessful"); + } + + return b; +} + +static rsRetVal +dynstats_createCtr(dynstats_bucket_t *b, const uchar* metric, dynstats_ctr_t **ctr) { + DEFiRet; + + CHKmalloc(*ctr = calloc(1, sizeof(dynstats_ctr_t))); + CHKmalloc((*ctr)->metric = ustrdup(metric)); + STATSCOUNTER_INIT((*ctr)->ctr, (*ctr)->mutCtr); + CHKiRet(statsobj.AddManagedCounter(b->stats, metric, ctrType_IntCtr, + b->resettable, &(*ctr)->ctr, &(*ctr)->pCtr)); +finalize_it: + if (iRet != RS_RET_OK) { + if ((*ctr) != NULL) { + free((*ctr)->metric); + free(*ctr); + *ctr = NULL; + } + } + RETiRet; +} + +static rsRetVal +dynstats_addNewCtr(dynstats_bucket_t *b, const uchar* metric, uint8_t doInitialIncrement) { + dynstats_ctr_t *ctr; + dynstats_ctr_t *found_ctr; + int created; + uchar *copy_of_key = NULL; + DEFiRet; + + created = 0; + ctr = NULL; + + if (ATOMIC_FETCH_32BIT(&b->metricCount, &b->mutMetricCount) >= b->maxCardinality) { + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + + CHKiRet(dynstats_createCtr(b, metric, &ctr)); + + pthread_rwlock_wrlock(&b->lock); + found_ctr = (dynstats_ctr_t*) hashtable_search(b->table, ctr->metric); + if (found_ctr != NULL) { + if (doInitialIncrement) { + STATSCOUNTER_INC(found_ctr->ctr, found_ctr->mutCtr); + } + } else { + copy_of_key = ustrdup(ctr->metric); + if (copy_of_key != NULL) { + created = hashtable_insert(b->table, copy_of_key, ctr); + } + if (created) { + if (b->ctrs == NULL) { + b->ctrs = ctr; + } else { + ctr->next = b->ctrs; + b->ctrs = ctr; + } + if (doInitialIncrement) { + STATSCOUNTER_INC(ctr->ctr, ctr->mutCtr); + } + } + } + pthread_rwlock_unlock(&b->lock); + + if (found_ctr != NULL) { + //ignore + } else if (created) { + ATOMIC_INC(&b->metricCount, &b->mutMetricCount); + STATSCOUNTER_INC(b->ctrNewMetricAdd, b->mutCtrNewMetricAdd); + } else { + if (copy_of_key != NULL) { + free(copy_of_key); + } + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + +finalize_it: + if ((! created) && (ctr != NULL)) { + dynstats_destroyCtr(b, ctr, 1); + } + RETiRet; +} + +rsRetVal +dynstats_inc(dynstats_bucket_t *b, uchar* metric) { + dynstats_ctr_t *ctr; + DEFiRet; + + if (! GatherStats) { + FINALIZE; + } + + if (ustrlen(metric) == 0) { + STATSCOUNTER_INC(b->ctrNoMetric, b->mutCtrNoMetric); + FINALIZE; + } + + if (pthread_rwlock_tryrdlock(&b->lock) == 0) { + ctr = (dynstats_ctr_t *) hashtable_search(b->table, metric); + if (ctr != NULL) { + STATSCOUNTER_INC(ctr->ctr, ctr->mutCtr); + } + pthread_rwlock_unlock(&b->lock); + } else { + ABORT_FINALIZE(RS_RET_NOENTRY); + } + + if (ctr == NULL) { + CHKiRet(dynstats_addNewCtr(b, metric, 1)); + } +finalize_it: + if (iRet != RS_RET_OK) { + if (iRet == RS_RET_NOENTRY) { + /* NOTE: this is not tested (because it requires very strong orchestration to gurantee contended lock for testing) */ + STATSCOUNTER_INC(b->ctrOpsIgnored, b->mutCtrOpsIgnored); + } else { + STATSCOUNTER_INC(b->ctrOpsOverflow, b->mutCtrOpsOverflow); + } + } + RETiRet; +} + diff --git a/runtime/dynstats.h b/runtime/dynstats.h new file mode 100644 index 000000000..98c12d1e1 --- /dev/null +++ b/runtime/dynstats.h @@ -0,0 +1,73 @@ +/* + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_DYNSTATS_H +#define INCLUDED_DYNSTATS_H + +#include "hashtable.h" + +typedef struct hashtable htable; + +struct dynstats_ctr_s { + STATSCOUNTER_DEF(ctr, mutCtr); + ctr_t *pCtr; + uchar *metric; + struct dynstats_ctr_s *next; /* linked list ptr */ +}; + +struct dynstats_bucket_s { + htable *table; + uchar *name; + pthread_rwlock_t lock; + statsobj_t *stats; + STATSCOUNTER_DEF(ctrOpsOverflow, mutCtrOpsOverflow); + ctr_t *pOpsOverflowCtr; + STATSCOUNTER_DEF(ctrNewMetricAdd, mutCtrNewMetricAdd); + ctr_t *pNewMetricAddCtr; + STATSCOUNTER_DEF(ctrNoMetric, mutCtrNoMetric); + ctr_t *pNoMetricCtr; + STATSCOUNTER_DEF(ctrMetricsPurged, mutCtrMetricsPurged); + ctr_t *pMetricsPurgedCtr; + STATSCOUNTER_DEF(ctrOpsIgnored, mutCtrOpsIgnored); + ctr_t *pOpsIgnoredCtr; + struct dynstats_bucket_s *next; /* linked list ptr */ + struct dynstats_ctr_s *ctrs; + uint32_t maxCardinality; + uint32_t metricCount; + pthread_mutex_t mutMetricCount; + uint32_t unusedMetricLife; + uint32_t lastResetTs; + struct timespec metricCleanupTimeout; + uint8_t resettable; +}; + +struct dynstats_buckets_s { + struct dynstats_bucket_s *list; + statsobj_t *global_stats; + pthread_rwlock_t lock; + uint8_t initialized; +}; + +rsRetVal dynstats_initCnf(dynstats_buckets_t *b); +rsRetVal dynstats_processCnf(struct cnfobj *o); +dynstats_bucket_t * dynstats_findBucket(const uchar* name); +rsRetVal dynstats_inc(dynstats_bucket_t *bucket, uchar* metric); +void dynstats_destroyAllBuckets(); +void dynstats_resetExpired(); +rsRetVal dynstatsClassInit(void); + +#endif /* #ifndef INCLUDED_DYNSTATS_H */ diff --git a/runtime/errmsg.c b/runtime/errmsg.c index 682687967..866397abb 100644 --- a/runtime/errmsg.c +++ b/runtime/errmsg.c @@ -96,7 +96,7 @@ doLogMsg(const int iErrno, const int iErrCode, const int severity, const char * snprintf(buf, sizeof(buf), "%s [v%s try http://www.rsyslog.com/e/%d ]", msg, VERSION, iErrCode * -1); } } - buf[sizeof(buf)/sizeof(char) - 1] = '\0'; /* just to be on the safe side... */ + buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ errno = 0; glblErrLogger(severity, iErrCode, (uchar*)buf); @@ -129,7 +129,7 @@ LogError(const int iErrno, const int iErrCode, const char *fmt, ... ) lenBuf--; } va_end(ap); - buf[sizeof(buf)/sizeof(char) - 1] = '\0'; /* just to be on the safe side... */ + buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ doLogMsg(iErrno, iErrCode, LOG_ERR, buf); } @@ -158,7 +158,7 @@ LogMsg(const int iErrno, const int iErrCode, const int severity, const char *fmt lenBuf--; } va_end(ap); - buf[sizeof(buf)/sizeof(char) - 1] = '\0'; /* just to be on the safe side... */ + buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ doLogMsg(iErrno, iErrCode, severity, buf); } diff --git a/runtime/glbl.c b/runtime/glbl.c index 01f82d4fe..2cb268046 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -7,18 +7,18 @@ * * Module begun 2008-04-16 by Rainer Gerhards * - * Copyright 2008-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -111,6 +111,10 @@ static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ static int bParserEscapeCCCStyle = 0; /* escape control characters in c style: 0 - no, 1 - yes */ short janitorInterval = 10; /* interval (in minutes) at which the janitor runs */ +int glblReportNewSenders = 0; +int glblReportGoneAwaySenders = 0; +int glblSenderStatsTimeout = 12 * 60 * 60; /* 12 hr timeout for senders */ +int glblSenderKeepTrack = 0; /* keep track of known senders? */ pid_t glbl_ourpid; #ifndef HAVE_ATOMIC_BUILTINS @@ -150,6 +154,10 @@ static struct cnfparamdescr cnfparamdescr[] = { { "parser.parsehostnameandtag", eCmdHdlrBinary, 0 }, { "stdlog.channelspec", eCmdHdlrString, 0 }, { "janitor.interval", eCmdHdlrPositiveInt, 0 }, + { "senders.reportnew", eCmdHdlrBinary, 0 }, + { "senders.reportgoneaway", eCmdHdlrBinary, 0 }, + { "senders.timeoutafter", eCmdHdlrPositiveInt, 0 }, + { "senders.keeptrack", eCmdHdlrBinary, 0 }, { "net.ipprotocol", eCmdHdlrGetWord, 0 }, { "net.acladdhostnameonfail", eCmdHdlrBinary, 0 }, { "net.aclresolvehostname", eCmdHdlrBinary, 0 }, @@ -164,8 +172,8 @@ static struct cnfparamblk paramblk = }; static struct cnfparamdescr timezonecnfparamdescr[] = { - { "id", eCmdHdlrString, 0 }, - { "offset", eCmdHdlrGetWord, 0 } + { "id", eCmdHdlrString, CNFPARAM_REQUIRED}, + { "offset", eCmdHdlrGetWord, CNFPARAM_REQUIRED } }; static struct cnfparamblk timezonepblk = { CNFPARAMBLK_VERSION, @@ -826,7 +834,11 @@ addTimezoneInfo(uchar *tzid, char offsMode, int8_t offsHour, int8_t offsMin) DEFiRet; tzinfo_t *newti; CHKmalloc(newti = realloc(tzinfos, (ntzinfos+1)*sizeof(tzinfo_t))); - CHKmalloc(newti[ntzinfos].id = strdup((char*)tzid)); + if((newti[ntzinfos].id = strdup((char*)tzid)) == NULL) { + free(newti); + DBGPRINTF("addTimezoneInfo: strdup failed with OOM\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } newti[ntzinfos].offsMode = offsMode; newti[ntzinfos].offsHour = offsHour; newti[ntzinfos].offsMin = offsMin; @@ -863,8 +875,10 @@ glblProcessTimezone(struct cnfobj *o) int i; pvals = nvlstGetParams(o->nvlst, &timezonepblk, NULL); - dbgprintf("timezone param blk after glblProcessTimezone:\n"); - cnfparamsPrint(&timezonepblk, pvals); + if(Debug) { + dbgprintf("timezone param blk after glblProcessTimezone:\n"); + cnfparamsPrint(&timezonepblk, pvals); + } for(i = 0 ; i < timezonepblk.nParams ; ++i) { if(!pvals[i].bUsed) @@ -879,6 +893,20 @@ glblProcessTimezone(struct cnfobj *o) } } + /* note: the following two checks for NULL are not strictly necessary + * as these are required parameters for the config block. But we keep + * them to make the clang static analyzer happy, which also helps + * guard against logic errors. + */ + if(offset == NULL) { + parser_errmsg("offset parameter missing (logic error?), timezone config ignored"); + goto done; + } + if(id == NULL) { + parser_errmsg("id parameter missing (logic error?), timezone config ignored"); + goto done; + } + if( strlen((char*)offset) != 6 || !(offset[0] == '-' || offset[0] == '+') || !(isdigit(offset[1]) && isdigit(offset[2])) @@ -917,8 +945,10 @@ glblProcessCnf(struct cnfobj *o) int i; cnfparamvals = nvlstGetParams(o->nvlst, ¶mblk, cnfparamvals); - dbgprintf("glbl param blk after glblProcessCnf:\n"); - cnfparamsPrint(¶mblk, cnfparamvals); + if(Debug) { + dbgprintf("glbl param blk after glblProcessCnf:\n"); + cnfparamsPrint(¶mblk, cnfparamvals); + } /* The next thing is a bit hackish and should be changed in higher * versions. There are a select few parameters which we need to @@ -1088,6 +1118,14 @@ glblDoneLoadCnf(void) "parameter '%s' -- ignored", proto); } free(proto); + } else if(!strcmp(paramblk.descr[i].name, "senders.reportnew")) { + glblReportNewSenders = (int) cnfparamvals[i].val.d.n; + } else if(!strcmp(paramblk.descr[i].name, "senders.reportgoneaway")) { + glblReportGoneAwaySenders = (int) cnfparamvals[i].val.d.n; + } else if(!strcmp(paramblk.descr[i].name, "senders.timeoutafter")) { + glblSenderStatsTimeout = (int) cnfparamvals[i].val.d.n; + } else if(!strcmp(paramblk.descr[i].name, "senders.keeptrack")) { + glblSenderKeepTrack = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "net.acladdhostnameonfail")) { *(net.pACLAddHostnameOnFail) = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "net.aclresolvehostname")) { diff --git a/runtime/glbl.h b/runtime/glbl.h index 5399b462a..466a51964 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -8,18 +8,18 @@ * Please note that there currently is no glbl.c file as we do not yet * have any implementations. * - * Copyright 2008-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -109,6 +109,10 @@ ENDinterface(glbl) PROTOTYPEObj(glbl); extern int glblDebugOnShutdown; /* start debug log when we are shut down */ +extern int glblReportNewSenders; +extern int glblReportGoneAwaySenders; +extern int glblSenderStatsTimeout; +extern int glblSenderKeepTrack; extern short janitorInterval; static inline pid_t glblGetOurPid(void) { return glbl_ourpid; } diff --git a/runtime/gss-misc.c b/runtime/gss-misc.c index 6f4fea944..a8ca249fe 100644 --- a/runtime/gss-misc.c +++ b/runtime/gss-misc.c @@ -76,7 +76,7 @@ static void display_status_(char *m, OM_uint32 code, int type) } else { char buf[1024]; snprintf(buf, sizeof(buf), "GSS-API error %s: %s\n", m, (char *) msg.value); - buf[sizeof(buf)/sizeof(char) - 1] = '\0'; + buf[sizeof(buf) - 1] = '\0'; errmsg.LogError(0, NO_ERRCODE, "%s", buf); } if (msg.length != 0) diff --git a/runtime/janitor.c b/runtime/janitor.c index b4471a8a0..7b3e50426 100644 --- a/runtime/janitor.c +++ b/runtime/janitor.c @@ -7,7 +7,7 @@ * * Module begun 2014-05-15 by Rainer Gerhards * - * Copyright (C) 2014 by Rainer Gerhards and Adiscon GmbH. + * Copyright (C) 2014-2015 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -41,7 +41,7 @@ static pthread_mutex_t janitorMut = PTHREAD_MUTEX_INITIALIZER; rsRetVal janitorAddEtry(void (*cb)(void*), const char *id, void *pUsr) { - struct janitorEtry *etry; + struct janitorEtry *etry = NULL; DEFiRet; CHKmalloc(etry = malloc(sizeof(struct janitorEtry))); CHKmalloc(etry->id = strdup(id)); @@ -53,6 +53,8 @@ janitorAddEtry(void (*cb)(void*), const char *id, void *pUsr) pthread_mutex_unlock(&janitorMut); DBGPRINTF("janitor: entry %p, id '%s' added\n", etry, id); finalize_it: + if(iRet != RS_RET_OK && etry != NULL) + free(etry); RETiRet; } diff --git a/runtime/libgcry.c b/runtime/libgcry.c index 9dabf5fc5..ca4c86e6e 100644 --- a/runtime/libgcry.c +++ b/runtime/libgcry.c @@ -1,6 +1,6 @@ /* gcry.c - rsyslog's libgcrypt based crypto provider * - * Copyright 2013 Adiscon GmbH. + * Copyright 2013-2016 Adiscon GmbH. * * We need to store some additional information in support of encryption. * For this, we create a side-file, which is named like the actual log @@ -25,11 +25,11 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -285,8 +285,8 @@ finalize_it: RETiRet; } -static rsRetVal -eiWriteIV(gcryfile gf, uchar *iv) +static rsRetVal __attribute__((nonnull(2))) +eiWriteIV(gcryfile gf, const uchar *const iv) { static const char hexchars[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; @@ -520,7 +520,17 @@ seedIV(gcryfile gf, uchar **iv) { int fd; + #ifdef __clang_analyzer__ + *iv = calloc(1, gf->blkLength); /* do NOT use this code! */ + /* this execution branch is only present to prevent a + * "garbagge value used" warning by the static analyzer. + * In fact, that is exactly what we want to and need to + * use. Using calloc here keeps that analyzer happy, but would + * cause a security issue if used in practice. + */ + #else *iv = malloc(gf->blkLength); /* do NOT zero-out! */ + #endif /* if we cannot obtain data from /dev/urandom, we use whatever * is present at the current memory location as random data. Of * course, this is very weak and we should consider a different @@ -595,6 +605,7 @@ rsgcryBlkBegin(gcryfile gf) gcry_error_t gcryError; uchar *iv = NULL; DEFiRet; + const char openMode = gf->openMode; gcryError = gcry_cipher_open(&gf->chd, gf->ctx->algo, gf->ctx->mode, 0); if (gcryError) { @@ -610,7 +621,7 @@ rsgcryBlkBegin(gcryfile gf) ABORT_FINALIZE(RS_RET_ERR); } - if(gf->openMode == 'r') { + if(openMode == 'r') { readIV(gf, &iv); readBlkEnd(gf); } else { @@ -624,7 +635,7 @@ rsgcryBlkBegin(gcryfile gf) ABORT_FINALIZE(RS_RET_ERR); } - if(gf->openMode == 'w') { + if(openMode == 'w') { CHKiRet(eiOpenAppend(gf)); CHKiRet(eiWriteIV(gf, iv)); } diff --git a/runtime/librsgt.c b/runtime/librsgt.c index 68248496a..f78d8b563 100644 --- a/runtime/librsgt.c +++ b/runtime/librsgt.c @@ -17,7 +17,7 @@ * information (most importantly last block hash) and sigblkConstruct * reads (or initilizes if not present) it. * - * Copyright 2013 Adiscon GmbH. + * Copyright 2013-2016 Adiscon GmbH. * * This file is part of rsyslog. * @@ -146,7 +146,7 @@ rsgtExit(void) static inline gtfile rsgtfileConstruct(gtctx ctx) { - gtfile gf; + gtfile gf = NULL; if((gf = calloc(1, sizeof(struct gtfile_s))) == NULL) goto done; gf->ctx = ctx; @@ -342,25 +342,18 @@ tlvWriteBlockHdr(gtfile gf) { 2 + hashOutputLengthOctets(gf->hashAlg) /* iv */ + 2 + 1 + gf->x_prev->len /* last hash */; /* write top-level TLV object block-hdr */ - r = tlv16Write(gf, 0x00, 0x0901, tlvlen); + CHKr(tlv16Write(gf, 0x00, 0x0901, tlvlen)); /* and now write the children */ /* hash-algo */ - r = tlv8Write(gf, 0x00, 0x01, 1); - if(r != 0) goto done; - r = tlvbufAddOctet(gf, hashIdentifier(gf->hashAlg)); - if(r != 0) goto done; + CHKr(tlv8Write(gf, 0x00, 0x01, 1)); + CHKr(tlvbufAddOctet(gf, hashIdentifier(gf->hashAlg))); /* block-iv */ - r = tlv8Write(gf, 0x00, 0x02, hashOutputLengthOctets(gf->hashAlg)); - if(r != 0) goto done; - r = tlvbufAddOctetString(gf, gf->IV, hashOutputLengthOctets(gf->hashAlg)); - if(r != 0) goto done; + CHKr(tlv8Write(gf, 0x00, 0x02, hashOutputLengthOctets(gf->hashAlg))); + CHKr(tlvbufAddOctetString(gf, gf->IV, hashOutputLengthOctets(gf->hashAlg))); /* last-hash */ - r = tlv8Write(gf, 0x00, 0x03, gf->x_prev->len + 1); - if(r != 0) goto done; - r = tlvbufAddOctet(gf, gf->x_prev->hashID); - if(r != 0) goto done; - r = tlvbufAddOctetString(gf, gf->x_prev->data, gf->x_prev->len); - if(r != 0) goto done; + CHKr(tlv8Write(gf, 0x00, 0x03, gf->x_prev->len + 1)); + CHKr(tlvbufAddOctet(gf, gf->x_prev->hashID)); + CHKr(tlvbufAddOctetString(gf, gf->x_prev->data, gf->x_prev->len)); done: return r; } @@ -420,6 +413,7 @@ readStateFile(gtfile gf) if (gf->x_prev->data == NULL) { free(gf->x_prev); gf->x_prev = NULL; + goto err; } if(read(fd, gf->x_prev->data, gf->x_prev->len) @@ -432,7 +426,6 @@ readStateFile(gtfile gf) return; err: - gf->x_prev = malloc(sizeof(imprint_t)); gf->x_prev->hashID = hashIdentifier(gf->hashAlg); gf->x_prev->len = hashOutputLengthOctets(gf->hashAlg); @@ -567,6 +560,8 @@ rsgtCtxOpenFile(gtctx ctx, unsigned char *logfn) gf->statefilename = (uchar*) strdup(fn); if(tlvOpen(gf, LOGSIGHDR, sizeof(LOGSIGHDR)-1) != 0) { reportErr(ctx, "signature file open failed"); + /* Free memory */ + free(gf); gf = NULL; } done: return gf; @@ -631,7 +626,7 @@ sigblkInit(gtfile gf) { if(gf == NULL) goto done; seedIV(gf); - memset(gf->roots_valid, 0, sizeof(gf->roots_valid)/sizeof(char)); + memset(gf->roots_valid, 0, sizeof(gf->roots_valid)); gf->nRoots = 0; gf->nRecords = 0; gf->bInBlk = 1; @@ -654,7 +649,7 @@ bufAddIV(gtfile gf, uchar *buf, size_t *len) /* concat: add imprint to buffer */ static inline void -bufAddImprint(gtfile gf, uchar *buf, size_t *len, imprint_t *imp) +bufAddImprint(uchar *buf, size_t *len, imprint_t *imp) { buf[*len] = imp->hashID; ++(*len); @@ -688,7 +683,7 @@ hash_m(gtfile gf, GTDataHash **m) size_t len = 0; int r = 0; - bufAddImprint(gf, concatBuf, &len, gf->x_prev); + bufAddImprint(concatBuf, &len, gf->x_prev); bufAddIV(gf, concatBuf, &len); rgt = GTDataHash_create(gf->hashAlg, concatBuf, len, m); if(rgt != GT_OK) { diff --git a/runtime/librsgt.h b/runtime/librsgt.h index debf3e51a..8bb43e772 100644 --- a/runtime/librsgt.h +++ b/runtime/librsgt.h @@ -332,7 +332,7 @@ int rsgt_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t int rsgt_chkFileHdr(FILE *fp, char *expect); gtfile rsgt_vrfyConstruct_gf(void); void rsgt_vrfyBlkInit(gtfile gf, block_hdr_t *bh, uint8_t bHasRecHashes, uint8_t bHasIntermedHashes); -int rsgt_vrfy_nextRec(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, gterrctx_t *ectx); +int rsgt_vrfy_nextRec(gtfile gf, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, gterrctx_t *ectx); int verifyBLOCK_HDR(FILE *sigfp, FILE *nsigfp); int verifyBLOCK_SIG(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, uint8_t bExtend, gterrctx_t *ectx); void rsgt_errctxInit(gterrctx_t *ectx); @@ -341,13 +341,16 @@ void rsgt_errctxSetErrRec(gterrctx_t *ectx, char *rec); void rsgt_errctxFrstRecInBlk(gterrctx_t *ectx, char *rec); void rsgt_objfree(uint16_t tlvtype, void *obj); void rsgt_set_debug(int iDebug); -int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose); +int rsgt_ConvertSigFile(FILE *oldsigfp, FILE *newsigfp, int verbose); /* TODO: replace these? */ int hash_m(gtfile gf, GTDataHash **m); int hash_r(gtfile gf, GTDataHash **r, const unsigned char *rec, const size_t len); int hash_node(gtfile gf, GTDataHash **node, GTDataHash *m, GTDataHash *r, uint8_t level); -extern char *rsgt_read_puburl; /**< url of publication server */ +extern char *rsgt_read_puburl; /**< url of publication server */ +extern char *rsgt_extend_puburl; /**< url of extension server */ +extern char *rsgt_userid; /**< userid for extension server */ +extern char *rsgt_userkey; /**< userkey for extension server */ extern uint8_t rsgt_read_showVerified; extern int RSGT_FLAG_TLV16_RUNTIME; extern int RSGT_FLAG_NONCRIT_RUNTIME; diff --git a/runtime/librsgt_common.h b/runtime/librsgt_common.h index a303f7fa5..412da66e2 100644 --- a/runtime/librsgt_common.h +++ b/runtime/librsgt_common.h @@ -36,6 +36,8 @@ typedef struct imprint_s imprint_t; typedef struct block_hdr_s block_hdr_t; typedef struct block_sig_s block_sig_t; typedef struct tlvrecord_s tlvrecord_t; +typedef struct block_hashchain_s block_hashchain_t; +typedef struct block_hashstep_s block_hashstep_t; struct tlvrecord_s { uint16_t tlvtype; @@ -47,7 +49,7 @@ struct tlvrecord_s { struct imprint_s { uint8_t hashID; - int len; + size_t len; uint8_t *data; }; @@ -69,6 +71,23 @@ struct block_sig_s { } sig; }; +struct block_hashstep_s { + uint8_t direction; /* left-link or right-link */ + uint8_t level_corr; + imprint_t sib_hash; +}; + +struct block_hashchain_s { + imprint_t rec_hash; + uint64_t stepCount; /* Helper to count left & right links */ + block_hashstep_t *hashsteps[MAX_ROOTS]; /* Using MAX_ROOTS here as well for the moment! */ + uint8_t direction; /* left-link or right-link */ + uint8_t level; /* default 0 */ +// block_hashstep_t left_link; +// block_hashstep_t right_link; +}; + + static inline char * sigTypeName(uint8_t sigID) { @@ -85,4 +104,9 @@ sigTypeName(uint8_t sigID) #define RSGT_TYPE_MASK 0x1f #define RSGT_FLAG_TLV16 0x80 +/* check return state of operation and abort, if non-OK */ +#define CHKr(code) if((r = code) != 0) goto done +/* check return state of operation and jump to donedecode, if non-OK */ +#define CHKrDecode(code) if((r = code) != 0) goto donedecode + #endif /* #ifndef INCLUDED_LIBRSGTCM_H */ \ No newline at end of file diff --git a/runtime/librsgt_read.c b/runtime/librsgt_read.c index 76ddda9ca..f7df39269 100644 --- a/runtime/librsgt_read.c +++ b/runtime/librsgt_read.c @@ -53,6 +53,8 @@ typedef unsigned char uchar; static int rsgt_read_debug = 0; char *rsgt_read_puburl = "http://verify.guardtime.com/gt-controlpublications.bin"; char *rsgt_extend_puburl = "http://verifier.guardtime.net/gt-extendingservice"; +char *rsgt_userid = ""; +char *rsgt_userkey = ""; uint8_t rsgt_read_showVerified = 0; /* macro to obtain next char from file including error tracking */ @@ -61,16 +63,6 @@ uint8_t rsgt_read_showVerified = 0; goto done; \ } -/* check return state of operation and abort, if non-OK */ -#define CHKr(code) if((r = code) != 0) goto done - -static void -errfunc(__attribute__((unused)) void *usrptr, uchar *emsg) -{ - if (rsgt_read_debug) - printf("Internal Error: %s \n", emsg); -} - /* if verbose==0, only the first and last two octets are shown, * otherwise everything. */ @@ -233,7 +225,7 @@ static inline int rsgt_tlvfileAddOctet(FILE *newsigfp, int8_t octet) int r = 0; if ( fputc(octet, newsigfp) == EOF ) r = RSGTE_IO; -done: return r; + return r; } static inline int rsgt_tlvfileAddOctetString(FILE *newsigfp, uint8_t *octet, int size) { @@ -389,7 +381,8 @@ rsgt_tlvRecRead(FILE *fp, tlvrecord_t *rec) r = 0; done: - if(rsgt_read_debug) + if(r == 0 && rsgt_read_debug) + /* Only show debug if no fail */ printf("debug: rsgt_tlvRecRead tlvtype %4.4x, len %u, r = %d\n", (unsigned) rec->tlvtype, (unsigned) rec->tlvlen, r); return r; @@ -450,7 +443,7 @@ static int rsgt_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint) { int r = 1; - imprint_t *imp; + imprint_t *imp = NULL; if((imp = calloc(1, sizeof(imprint_t))) == NULL) { r = RSGTE_OOM; @@ -468,8 +461,14 @@ rsgt_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint) *imprint = imp; r = 0; done: - if(rsgt_read_debug) - printf("debug: read tlvDecodeIMPRINT returned %d TLVLen=%d, HashID=%d\n", r, rec->tlvlen, imp->hashID); + if(r == 0) { + if(rsgt_read_debug) + printf("debug: read tlvDecodeIMPRINT returned %d TLVLen=%d, HashID=%d\n", r, rec->tlvlen, imp->hashID); + } else { + /* Free memory on FAIL!*/ + if (imp != NULL) + rsgt_objfree(rec->tlvtype, imp); + } return r; } @@ -574,7 +573,7 @@ rsgt_tlvDecodeBLOCK_HDR(tlvrecord_t *rec, block_hdr_t **blockhdr) { int r = 1; uint16_t strtidx = 0; - block_hdr_t *bh; + block_hdr_t *bh = NULL; if((bh = calloc(1, sizeof(block_hdr_t))) == NULL) { r = RSGTE_OOM; goto done; @@ -589,8 +588,14 @@ rsgt_tlvDecodeBLOCK_HDR(tlvrecord_t *rec, block_hdr_t **blockhdr) *blockhdr = bh; r = 0; done: - if(rsgt_read_debug) - printf("debug: rsgt_tlvDecodeBLOCK_HDR returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + if (r == 0) { + if(rsgt_read_debug) + printf("debug: tlvDecodeBLOCK_HDR returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + } else { + /* Free memory on FAIL!*/ + if (bh != NULL) + rsgt_objfree(rec->tlvtype, bh); + } return r; } @@ -599,7 +604,7 @@ rsgt_tlvDecodeBLOCK_SIG(tlvrecord_t *rec, block_sig_t **blocksig) { int r = 1; uint16_t strtidx = 0; - block_sig_t *bs; + block_sig_t *bs = NULL; if((bs = calloc(1, sizeof(block_sig_t))) == NULL) { r = RSGTE_OOM; goto done; @@ -613,8 +618,14 @@ rsgt_tlvDecodeBLOCK_SIG(tlvrecord_t *rec, block_sig_t **blocksig) *blocksig = bs; r = 0; done: - if(rsgt_read_debug) - printf("debug: rsgt_tlvDecodeBLOCK_SIG returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + if(r == 0) { + if (rsgt_read_debug) + printf("debug: rsgt_tlvDecodeBLOCK_SIG returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + } else { + /* Free memory on FAIL!*/ + if (bs != NULL) + rsgt_objfree(rec->tlvtype, bs); + } return r; } static int @@ -637,8 +648,8 @@ rsgt_tlvRecDecode(tlvrecord_t *rec, void *obj) break; } done: - if(rsgt_read_debug) - printf("debug: rsgt_tlvRecDecode returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + if(r == 0 && rsgt_read_debug) + printf("debug: tlvRecDecode returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); return r; } @@ -652,14 +663,15 @@ rsgt_tlvrdRecHash(FILE *fp, FILE *outfp, imprint_t **imp) if(rec.tlvtype != 0x0902) { r = RSGTE_MISS_REC_HASH; rsgt_objfree(rec.tlvtype, *imp); + *imp = NULL; goto done; } if(outfp != NULL) if((r = rsgt_tlvwrite(outfp, &rec)) != 0) goto done; r = 0; done: - if(rsgt_read_debug) - printf("debug: rsgt_tlvrdRecHash returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); + if(r == 0 && rsgt_read_debug) + printf("debug: tlvrdRecHash returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); return r; } @@ -673,14 +685,15 @@ rsgt_tlvrdTreeHash(FILE *fp, FILE *outfp, imprint_t **imp) if(rec.tlvtype != 0x0903) { r = RSGTE_MISS_TREE_HASH; rsgt_objfree(rec.tlvtype, *imp); + *imp = NULL; goto done; } if(outfp != NULL) if((r = rsgt_tlvwrite(outfp, &rec)) != 0) goto done; r = 0; done: - if(rsgt_read_debug) - printf("debug: rsgt_tlvrdTreeHash returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); + if(r == 0 && rsgt_read_debug) + printf("debug: tlvrdTreeHash returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); return r; } @@ -847,6 +860,10 @@ rsgt_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose) void rsgt_objfree(uint16_t tlvtype, void *obj) { + // check if obj is valid + if (obj == NULL ) + return; + switch(tlvtype) { case 0x0901: free(((block_hdr_t*)obj)->iv); @@ -1061,15 +1078,17 @@ rsgt_vrfy_chkTreeHash(gtfile gf, FILE *sigfp, FILE *nsigfp, } r = 0; done: - if(rsgt_read_debug) - printf("debug: rsgt_vrfy_chkTreeHash returned %d, hashID=%d, Length=%d\n", r, imp->hashID, hashOutputLengthOctets(imp->hashID)); - if(imp != NULL) + if(imp != NULL) { + if(rsgt_read_debug) + printf("debug: rsgt_vrfy_chkTreeHash returned %d, hashID=%d, Length=%d\n", r, imp->hashID, hashOutputLengthOctets(imp->hashID)); + /* Free memory */ rsgt_objfree(0x0903, imp); + } return r; } int -rsgt_vrfy_nextRec(block_sig_t *bs, gtfile gf, FILE *sigfp, FILE *nsigfp, +rsgt_vrfy_nextRec(gtfile gf, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, gterrctx_t *ectx) { int r = 0; @@ -1176,6 +1195,7 @@ done: iWr += subrec.lenHdr; \ memcpy(newrec.data+iWr, subrec.data, subrec.tlvlen); \ iWr += subrec.tlvlen; + static inline int rsgt_extendSig(GTTimestamp *timestamp, tlvrecord_t *rec, gterrctx_t *ectx) { @@ -1204,19 +1224,19 @@ rsgt_extendSig(GTTimestamp *timestamp, tlvrecord_t *rec, gterrctx_t *ectx) */ iRd = iWr = 0; // TODO; check tlvtypes at comment places below! - if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* HASH_ALGO */ COPY_SUBREC_TO_NEWREC - if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* BLOCK_IV */ COPY_SUBREC_TO_NEWREC - if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* LAST_HASH */ COPY_SUBREC_TO_NEWREC - if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* REC_COUNT */ COPY_SUBREC_TO_NEWREC - if ((r = rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsgt_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* actual sig! */ newrec.data[iWr++] = 0x09 | RSGT_FLAG_TLV16_RUNTIME; newrec.data[iWr++] = 0x06; @@ -1253,7 +1273,12 @@ verifyBLOCK_HDR(FILE *sigfp, FILE *nsigfp) } if (nsigfp != NULL) if ((r = rsgt_tlvwrite(nsigfp, &rec)) != 0) goto done; -done: rsgt_objfree(rec.tlvtype, bh); +done: +/* if (r == 0 || r == RSGTE_IO)*/ { + /* Only free memory if return is OK or error was RSGTE_IO was (happened in rsksi_tlvwrite) */ + if (bh != NULL) + rsgt_objfree(rec.tlvtype, bh); + } if(rsgt_read_debug) printf("debug: verifyBLOCK_HDR returned %d\n", r); return r; @@ -1325,7 +1350,7 @@ void rsgt_set_debug(int iDebug) } /* Helper function to convert an old V10 signature file into V11 */ -int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) +int rsgt_ConvertSigFile(FILE *oldsigfp, FILE *newsigfp, int verbose) { int r = 0, rRead = 0; imprint_t *imp = NULL; @@ -1373,6 +1398,7 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) /* Free mem*/ free(imp->data); free(imp); + imp = NULL; break; case 0x0902: /* Split Data into HEADER and BLOCK */ @@ -1389,7 +1415,7 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) } /* Check OLD encoded HASH ALGO */ - CHKr(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x00 && subrec.tlvlen == 1)) { r = RSGTE_FMT; goto donedecode; @@ -1397,7 +1423,7 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) bh->hashID = subrec.data[0]; /* Check OLD encoded BLOCK_IV */ - CHKr(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x01)) { r = RSGTE_INVLTYP; goto donedecode; @@ -1406,7 +1432,7 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) memcpy(bh->iv, subrec.data, subrec.tlvlen); /* Check OLD encoded LAST HASH */ - CHKr(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x02)) { r = RSGTE_INVLTYP; goto donedecode; } bh->lastHash.hashID = subrec.data[0]; if(subrec.tlvlen != 1 + hashOutputLengthOctets(bh->lastHash.hashID)) { @@ -1421,7 +1447,7 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) rsgt_printBLOCK_HDR(stdout, bh, verbose); /* Check OLD encoded COUNT */ - CHKr(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x03 && subrec.tlvlen <= 8)) { r = RSGTE_INVLTYP; goto donedecode; } bs->recCount = 0; for(i = 0 ; i < subrec.tlvlen ; ++i) { @@ -1429,13 +1455,12 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) } /* Check OLD encoded SIG */ - CHKr(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsgt_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x0906)) { r = RSGTE_INVLTYP; goto donedecode; } bs->sig.der.len = subrec.tlvlen; bs->sigID = SIGID_RFC3161; if((bs->sig.der.data = (uint8_t*)malloc(bs->sig.der.len)) == NULL) {r=RSGTE_OOM;goto donedecode;} memcpy(bs->sig.der.data, subrec.data, bs->sig.der.len); - r = 0; /* Debug output */ rsgt_printBLOCK_SIG(stdout, bs, verbose); @@ -1453,48 +1478,37 @@ int rsgt_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose) 2 + hashOutputLengthOctets(bh->hashID) /* iv */ + 2 + 1 + bh->lastHash.len /* last hash */; /* write top-level TLV object block-hdr */ - r = rsgt_tlv16Write(newsigfp, 0x00, 0x0901, tlvlen); + CHKrDecode(rsgt_tlv16Write(newsigfp, 0x00, 0x0901, tlvlen)); /* and now write the children */ /* hash-algo */ - r = rsgt_tlv8Write(newsigfp, 0x00, 0x01, 1); - if(r != 0) goto done; - r = rsgt_tlvfileAddOctet(newsigfp, hashIdentifier(bh->hashID)); - if(r != 0) goto done; + CHKrDecode(rsgt_tlv8Write(newsigfp, 0x00, 0x01, 1)); + CHKrDecode(rsgt_tlvfileAddOctet(newsigfp, hashIdentifier(bh->hashID))); /* block-iv */ - r = rsgt_tlv8Write(newsigfp, 0x00, 0x02, hashOutputLengthOctets(bh->hashID)); - if(r != 0) goto done; - r = rsgt_tlvfileAddOctetString(newsigfp, bh->iv, hashOutputLengthOctets(bh->hashID)); - if(r != 0) goto done; + CHKrDecode(rsgt_tlv8Write(newsigfp, 0x00, 0x02, hashOutputLengthOctets(bh->hashID))); + CHKrDecode(rsgt_tlvfileAddOctetString(newsigfp, bh->iv, hashOutputLengthOctets(bh->hashID))); /* last-hash */ - r = rsgt_tlv8Write(newsigfp, 0x00, 0x03, bh->lastHash.len + 1); - if(r != 0) goto done; - r = rsgt_tlvfileAddOctet(newsigfp, bh->lastHash.hashID); - if(r != 0) goto done; - r = rsgt_tlvfileAddOctetString(newsigfp, bh->lastHash.data, bh->lastHash.len); - if(r != 0) goto done; + CHKrDecode(rsgt_tlv8Write(newsigfp, 0x00, 0x03, bh->lastHash.len + 1)); + CHKrDecode(rsgt_tlvfileAddOctet(newsigfp, bh->lastHash.hashID)); + CHKrDecode(rsgt_tlvfileAddOctetString(newsigfp, bh->lastHash.data, bh->lastHash.len)); /* Create Block Signature */ tlvlenRecords = rsgt_tlvGetInt64OctetSize(bs->recCount); tlvlen = 2 + tlvlenRecords /* rec-count */ + 4 + bs->sig.der.len /* rfc-3161 */; /* write top-level TLV object (block-sig */ - r = rsgt_tlv16Write(newsigfp, 0x00, 0x0904, tlvlen); - if(r != 0) goto done; + CHKrDecode(rsgt_tlv16Write(newsigfp, 0x00, 0x0904, tlvlen)); /* and now write the children */ /* rec-count */ - r = rsgt_tlv8Write(newsigfp, 0x00, 0x01, tlvlenRecords); - if(r != 0) goto done; - r = rsgt_tlvfileAddInt64(newsigfp, bs->recCount); - if(r != 0) goto done; + CHKrDecode(rsgt_tlv8Write(newsigfp, 0x00, 0x01, tlvlenRecords)); + CHKrDecode(rsgt_tlvfileAddInt64(newsigfp, bs->recCount)); /* rfc-3161 */ - r = rsgt_tlv16Write(newsigfp, 0x00, 0x906, bs->sig.der.len); - if(r != 0) goto done; - r = rsgt_tlvfileAddOctetString(newsigfp, bs->sig.der.data, bs->sig.der.len); + CHKrDecode(rsgt_tlv16Write(newsigfp, 0x00, 0x906, bs->sig.der.len)); + CHKrDecode(rsgt_tlvfileAddOctetString(newsigfp, bs->sig.der.data, bs->sig.der.len)); +donedecode: /* Set back to OLD default */ RSGT_FLAG_TLV16_RUNTIME = 0x20; -donedecode: /* Free mem*/ if (bh != NULL) { free(bh->iv); diff --git a/runtime/librsksi.c b/runtime/librsksi.c index d6b39b8ee..82f3e0469 100644 --- a/runtime/librsksi.c +++ b/runtime/librsksi.c @@ -71,7 +71,7 @@ reportErr(rsksictx ctx, char *errmsg) done: return; } -static void +void reportKSIAPIErr(rsksictx ctx, ksifile ksi, char *apiname, int ecode) { char errbuf[4096]; @@ -89,31 +89,51 @@ rsksisetErrFunc(rsksictx ctx, void (*func)(void*, uchar *), void *usrptr) ctx->errFunc = func; } -imprint_t * -rsksiImprintFromKSI_DataHash(ksifile ksi, KSI_DataHash *hash) +int +rsksiIntoImprintFromKSI_DataHash(imprint_t* imp, ksifile ksi, KSI_DataHash *hash) { - int r; - imprint_t *imp; + int r = RSGTE_SUCCESS; const unsigned char *digest; - unsigned digest_len; + size_t digest_len; - if((imp = calloc(1, sizeof(imprint_t))) == NULL) { - goto done; - } - int hashID; + KSI_HashAlgorithm hashID; r = KSI_DataHash_extract(hash, &hashID, &digest, &digest_len); if (r != KSI_OK){ reportKSIAPIErr(ksi->ctx, ksi, "KSI_DataHash_extract", r); - free(imp); imp = NULL; goto done; + r = RSGTE_IO; + goto done; } imp->hashID = hashID; imp->len = digest_len; if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) { - free(imp); imp = NULL; goto done; + r = RSGTE_OOM; + goto done; } memcpy(imp->data, digest, digest_len); -done: return imp; +done: + return r; +} + + +imprint_t * +rsksiImprintFromKSI_DataHash(ksifile ksi, KSI_DataHash *hash) +{ + int r; + imprint_t *imp; + + if((imp = calloc(1, sizeof(imprint_t))) == NULL) { + goto done; + } + + r = rsksiIntoImprintFromKSI_DataHash(imp, ksi, hash); + if (r != RSGTE_SUCCESS) { + free(imp); + imp = NULL; + goto done; + } +done: + return imp; } void @@ -126,25 +146,9 @@ rsksiimprintDel(imprint_t *imp) } int -rsksiInit(char *usragent) +rsksiInit(__attribute__((unused)) char *usragent) { - int r = 0; - int ret = KSI_OK; - -/* - ret = GT_init(); - if(ret != KSI_OK) { - r = 1; - goto done; - } - ret = GTHTTP_init(usragent, 1); - if(ret != KSI_OK) { - r = 1; - goto done; - } -*/ - -done: return r; + return 0; } void @@ -156,7 +160,7 @@ rsksiExit(void) static inline ksifile rsksifileConstruct(rsksictx ctx) { - ksifile ksi; + ksifile ksi = NULL; if((ksi = calloc(1, sizeof(struct ksifile_s))) == NULL) goto done; ksi->ctx = ctx; @@ -169,14 +173,14 @@ rsksifileConstruct(rsksictx ctx) done: return ksi; } -static inline int +static inline size_t tlvbufPhysWrite(ksifile ksi) { ssize_t lenBuf; ssize_t iTotalWritten; ssize_t iWritten; char *pWriteBuf; - int r = 0; + size_t r = 0; lenBuf = ksi->tlvIdx; pWriteBuf = ksi->tlvBuf; @@ -204,7 +208,7 @@ finalize_it: return r; } -static inline int +static inline size_t tlvbufChkWrite(ksifile ksi) { if(ksi->tlvIdx == sizeof(ksi->tlvBuf)) { @@ -217,19 +221,19 @@ tlvbufChkWrite(ksifile ksi) /* write to TLV file buffer. If buffer is full, an actual call occurs. Else * output is written only on flush or close. */ -static inline int +static inline size_t tlvbufAddOctet(ksifile ksi, int8_t octet) { - int r; + size_t r; r = tlvbufChkWrite(ksi); if(r != 0) goto done; ksi->tlvBuf[ksi->tlvIdx++] = octet; done: return r; } -static inline int -tlvbufAddOctetString(ksifile ksi, uint8_t *octet, int size) +static inline size_t +tlvbufAddOctetString(ksifile ksi, uint8_t *octet, size_t size) { - int i, r = 0; + size_t i, r = 0; for(i = 0 ; i < size ; ++i) { r = tlvbufAddOctet(ksi, octet[i]); if(r != 0) goto done; @@ -336,7 +340,7 @@ tlvWriteHashKSI(ksifile ksi, uint16_t tlvtype, KSI_DataHash *rec) unsigned tlvlen; int r; const unsigned char *digest; - unsigned digest_len; + size_t digest_len; r = KSI_DataHash_extract(rec, NULL, &digest, &digest_len); if (r != KSI_OK){ reportKSIAPIErr(ksi->ctx, ksi, "KSI_DataHash_extract", r); @@ -359,25 +363,18 @@ tlvWriteBlockHdrKSI(ksifile ksi) { 2 + hashOutputLengthOctetsKSI(ksi->hashAlg) /* iv */ + 2 + 1 + ksi->x_prev->len /* last hash */; /* write top-level TLV object block-hdr */ - r = tlv16WriteKSI(ksi, 0x00, 0x0901, tlvlen); + CHKr(tlv16WriteKSI(ksi, 0x00, 0x0901, tlvlen)); /* and now write the children */ /* hash-algo */ - r = tlv8WriteKSI(ksi, 0x00, 0x01, 1); - if(r != 0) goto done; - r = tlvbufAddOctet(ksi, hashIdentifierKSI(ksi->hashAlg)); - if(r != 0) goto done; + CHKr(tlv8WriteKSI(ksi, 0x00, 0x01, 1)); + CHKr(tlvbufAddOctet(ksi, hashIdentifierKSI(ksi->hashAlg))); /* block-iv */ - r = tlv8WriteKSI(ksi, 0x00, 0x02, hashOutputLengthOctetsKSI(ksi->hashAlg)); - if(r != 0) goto done; - r = tlvbufAddOctetString(ksi, ksi->IV, hashOutputLengthOctetsKSI(ksi->hashAlg)); - if(r != 0) goto done; + CHKr(tlv8WriteKSI(ksi, 0x00, 0x02, hashOutputLengthOctetsKSI(ksi->hashAlg))); + CHKr(tlvbufAddOctetString(ksi, ksi->IV, hashOutputLengthOctetsKSI(ksi->hashAlg))); /* last-hash */ - r = tlv8WriteKSI(ksi, 0x00, 0x03, ksi->x_prev->len + 1); - if(r != 0) goto done; - r = tlvbufAddOctet(ksi, ksi->x_prev->hashID); - if(r != 0) goto done; - r = tlvbufAddOctetString(ksi, ksi->x_prev->data, ksi->x_prev->len); - if(r != 0) goto done; + CHKr(tlv8WriteKSI(ksi, 0x00, 0x03, ksi->x_prev->len + 1)); + CHKr(tlvbufAddOctet(ksi, ksi->x_prev->hashID)); + CHKr(tlvbufAddOctetString(ksi, ksi->x_prev->data, ksi->x_prev->len)); done: return r; } @@ -401,8 +398,8 @@ tlvWriteBlockSigKSI(ksifile ksi, uchar *der, uint16_t lenDer) if(r != 0) goto done; r = tlvbufAddInt64(ksi, ksi->nRecords); if(r != 0) goto done; - /* rfc-3161 */ - r = tlv16WriteKSI(ksi, 0x00, 0x906, lenDer); + /* Open-KSI signature */ + r = tlv16WriteKSI(ksi, 0x00, 0x0905, lenDer); if(r != 0) goto done; r = tlvbufAddOctetString(ksi, der, lenDer); done: return r; @@ -439,10 +436,11 @@ readStateFile(ksifile ksi) if (ksi->x_prev->data == NULL) { free(ksi->x_prev); ksi->x_prev = NULL; + goto err; } if(read(fd, ksi->x_prev->data, ksi->x_prev->len) - != ksi->x_prev->len) { + != (ssize_t) ksi->x_prev->len) { rsksiimprintDel(ksi->x_prev); ksi->x_prev = NULL; goto err; @@ -451,7 +449,6 @@ readStateFile(ksifile ksi) return; err: - ksi->x_prev = malloc(sizeof(imprint_t)); ksi->x_prev->hashID = hashIdentifierKSI(ksi->hashAlg); ksi->x_prev->len = hashOutputLengthOctetsKSI(ksi->hashAlg); @@ -587,6 +584,8 @@ rsksiCtxOpenFile(rsksictx ctx, unsigned char *logfn) ksi->statefilename = (uchar*) strdup(fn); if(tlvOpenKSI(ksi, LOGSIGHDR, sizeof(LOGSIGHDR)-1) != 0) { reportErr(ctx, "signature file open failed"); + /* Free memory */ + free(ksi); ksi = NULL; } done: return ksi; @@ -610,8 +609,6 @@ rsksiSetHashFunction(rsksictx ctx, char *algName) ctx->hashAlg = KSI_HASHALG_SHA2_384; else if(!strcmp(algName, "SHA2-512")) ctx->hashAlg = KSI_HASHALG_SHA2_512; - else if(!strcmp(algName, "RIPEMD-256")) - ctx->hashAlg = KSI_HASHALG_RIPEMD_256; else if(!strcmp(algName, "SHA3-244")) ctx->hashAlg = KSI_HASHALG_SHA3_244; else if(!strcmp(algName, "SHA3-256")) @@ -664,7 +661,7 @@ sigblkInitKSI(ksifile ksi) { if(ksi == NULL) goto done; seedIVKSI(ksi); - memset(ksi->roots_valid, 0, sizeof(ksi->roots_valid)/sizeof(char)); + memset(ksi->roots_valid, 0, sizeof(ksi->roots_valid)); ksi->nRoots = 0; ksi->nRecords = 0; ksi->bInBlk = 1; @@ -687,7 +684,7 @@ bufAddIV(ksifile ksi, uchar *buf, size_t *len) /* concat: add imprint to buffer */ static inline void -bufAddImprint(ksifile ksi, uchar *buf, size_t *len, imprint_t *imp) +bufAddImprint(uchar *buf, size_t *len, imprint_t *imp) { buf[*len] = imp->hashID; ++(*len); @@ -701,7 +698,7 @@ bufAddHash(ksifile ksi, uchar *buf, size_t *len, KSI_DataHash *hash) { int r; const unsigned char *digest; - unsigned digest_len; + size_t digest_len; r = KSI_DataHash_extract(hash, NULL, &digest, &digest_len); // TODO: error check if (r != KSI_OK){ reportKSIAPIErr(ksi->ctx, ksi, "KSI_DataHash_extract", r); @@ -730,7 +727,7 @@ hash_m_ksi(ksifile ksi, KSI_DataHash **m) size_t len = 0; int r = 0; - bufAddImprint(ksi, concatBuf, &len, ksi->x_prev); + bufAddImprint(concatBuf, &len, ksi->x_prev); bufAddIV(ksi, concatBuf, &len); rgt = KSI_DataHash_create(ksi->ctx->ksi_ctx, concatBuf, len, ksi->hashAlg, m); if(rgt != KSI_OK) { @@ -847,7 +844,7 @@ static int signIt(ksifile ksi, KSI_DataHash *hash) { unsigned char *der = NULL; - unsigned lenDer; + size_t lenDer; int r = KSI_OK; int ret = 0; KSI_Signature *sig = NULL; diff --git a/runtime/librsksi.h b/runtime/librsksi.h index bf25c1d29..ce996c71d 100644 --- a/runtime/librsksi.h +++ b/runtime/librsksi.h @@ -61,7 +61,7 @@ struct ksifile_s { uint8_t disabled; /* permits to disable this file --> set to 1 */ uint64_t blockSizeLimit; uint8_t *IV; /* initial value for blinding masks */ - imprint_t *x_prev; /* last leaf hash (maybe of previous block) --> preserve on term */ + imprint_t *x_prev; /* last leaf hash (maybe of previous block) --> preserve on term */ unsigned char *sigfilename; unsigned char *statefilename; int fd; @@ -119,6 +119,7 @@ struct rsksistatefile { }; /* error states */ +#define RSGTE_SUCCESS 0 /* Success state */ #define RSGTE_IO 1 /* any kind of io error */ #define RSGTE_FMT 2 /* data fromat error */ #define RSGTE_INVLTYP 3 /* invalid TLV type record (unexcpected at this point) */ @@ -142,6 +143,7 @@ struct rsksistatefile { #define RSGTE_HASH_CREATE 20 /* error creating a hash */ #define RSGTE_END_OF_SIG 21 /* unexpected end of signature - more log line exist */ #define RSGTE_END_OF_LOG 22 /* unexpected end of log file - more signatures exist */ +#define RSGTE_EXTRACT_HASH 23 /* error extracting hashes for record */ /* the following function maps RSGTE_* state to a string - must be updated * whenever a new state is added. @@ -152,7 +154,7 @@ static inline char * RSKSIE2String(int err) { switch(err) { - case 0: + case RSGTE_SUCCESS: return "success"; case RSGTE_IO: return "i/o error"; @@ -198,6 +200,8 @@ RSKSIE2String(int err) return "unexpected end of signature"; case RSGTE_END_OF_LOG: return "unexpected end of log"; + case RSGTE_EXTRACT_HASH: + return "either record-hash, left-hash or right-hash was empty"; default: return "unknown error"; } @@ -220,8 +224,6 @@ hashOutputLengthOctetsKSI(uint8_t hashID) return 48; case KSI_HASHALG_SHA2_512: /** The SHA-512 algorithm. */ return 64; - case KSI_HASHALG_RIPEMD_256: /** The RIPEMD-256 algorithm. */ - return 32; case KSI_HASHALG_SHA3_244: /** The SHA3-244 algorithm. */ return 28; case KSI_HASHALG_SHA3_256: /** The SHA3-256 algorithm. */ @@ -252,8 +254,6 @@ hashIdentifierKSI(KSI_HashAlgorithm hashID) return 0x04; case KSI_HASHALG_SHA2_512: /** The SHA-512 algorithm. */ return 0x05; - case KSI_HASHALG_RIPEMD_256: /** The RIPEMD-256 algorithm. */ - return 0x06; case KSI_HASHALG_SHA3_244: /** The SHA3-244 algorithm. */ return 0x07; case KSI_HASHALG_SHA3_256: /** The SHA3-256 algorithm. */ @@ -283,8 +283,6 @@ hashAlgNameKSI(uint8_t hashID) return "SHA2-384"; case KSI_HASHALG_SHA2_512: return "SHA2-512"; - case KSI_HASHALG_RIPEMD_256: - return "RIPEMD-256"; case KSI_HASHALG_SHA3_244: return "SHA3-224"; case KSI_HASHALG_SHA3_256: @@ -314,8 +312,6 @@ hashID2AlgKSI(uint8_t hashID) return KSI_HASHALG_SHA2_384; case 0x05: return KSI_HASHALG_SHA2_512; - case 0x06: - return KSI_HASHALG_RIPEMD_256; case 0x07: return KSI_HASHALG_SHA3_244; case 0x08: @@ -357,13 +353,15 @@ int rsksiInit(char *usragent); void rsksiExit(void); rsksictx rsksiCtxNew(void); void rsksisetErrFunc(rsksictx ctx, void (*func)(void*, unsigned char *), void *usrptr); +void reportKSIAPIErr(rsksictx ctx, ksifile ksi, char *apiname, int ecode); ksifile rsksiCtxOpenFile(rsksictx ctx, unsigned char *logfn); int rsksifileDestruct(ksifile ksi); void rsksiCtxDel(rsksictx ctx); void sigblkInitKSI(ksifile ksi); int sigblkAddRecordKSI(ksifile ksi, const unsigned char *rec, const size_t len); int sigblkFinishKSI(ksifile ksi); -imprint_t * rsksiImprintFromKSI_DataHash(ksifile ksi, KSI_DataHash *hash); +int rsksiIntoImprintFromKSI_DataHash(imprint_t* imp, ksifile ksi, KSI_DataHash *hash); +imprint_t* rsksiImprintFromKSI_DataHash(ksifile ksi, KSI_DataHash *hash); void rsksiimprintDel(imprint_t *imp); /* reader functions */ int rsksi_tlvrdHeader(FILE *fp, unsigned char *hdr); @@ -371,13 +369,16 @@ int rsksi_tlvrd(FILE *fp, tlvrecord_t *rec, void *obj); void rsksi_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose); void rsksi_printBLOCK_HDR(FILE *fp, block_hdr_t *bh, uint8_t verbose); void rsksi_printBLOCK_SIG(FILE *fp, block_sig_t *bs, uint8_t verbose); -int rsksi_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t **bh, uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes); -int rsksi_chkFileHdr(FILE *fp, char *expect); +int rsksi_getBlockParams(ksifile ksi, FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t **bh, uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes); +int rsksi_getExcerptBlockParams(ksifile ksi, FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t **bh); +int rsksi_chkFileHdr(FILE *fp, char *expect, uint8_t verbose); ksifile rsksi_vrfyConstruct_gf(void); void rsksi_vrfyBlkInit(ksifile ksi, block_hdr_t *bh, uint8_t bHasRecHashes, uint8_t bHasIntermedHashes); int rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx); -int verifyBLOCK_HDRKSI(FILE *sigfp, FILE *nsigfp); -int verifyBLOCK_SIGKSI(block_sig_t *bs, ksifile ksi, FILE *sigfp, FILE *nsigfp, uint8_t bExtend, ksierrctx_t *ectx); + int rsksi_vrfy_nextRecExtract(ksifile ksi, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx, block_hashchain_t *hashchain, int storehashchain); + int rsksi_vrfy_nextHashChain(ksifile ksi, block_sig_t *bs, FILE *sigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx); +int verifyBLOCK_HDRKSI(ksifile ksi, FILE *sigfp, FILE *nsigfp, tlvrecord_t* tlvrec); +int verifyBLOCK_SIGKSI(block_sig_t *bs, ksifile ksi, FILE *sigfp, FILE *nsigfp, uint8_t bExtend, KSI_DataHash *ksiHash, ksierrctx_t *ectx); void rsksi_errctxInit(ksierrctx_t *ectx); void rsksi_errctxExit(ksierrctx_t *ectx); void rsksi_errctxSetErrRec(ksierrctx_t *ectx, char *rec); @@ -385,12 +386,27 @@ void rsksi_errctxFrstRecInBlk(ksierrctx_t *ectx, char *rec); void rsksi_objfree(uint16_t tlvtype, void *obj); void rsksi_set_debug(int iDebug); int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose); + + int rsksi_WriteHashChain(FILE *newsigfp, block_hashchain_t *hashchain, block_sig_t *bsIn, int verbose); + int rsksi_ExtractBlockSignature(FILE *newsigfp, ksifile ksi, block_sig_t *bsIn, ksierrctx_t *ectx, int verbose); + int rsksi_tlvwrite(FILE *fp, tlvrecord_t *rec); + int rsksi_tlvRecDecode(tlvrecord_t *rec, void *obj); + int rsksi_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint); + int rsksi_tlvDecodeHASHCHAIN(tlvrecord_t *rec, block_hashchain_t **blhashchain); + int verifySigblkFinish(ksifile ksi, KSI_DataHash **pRoot); + int verifySigblkFinishChain(ksifile ksi, block_hashchain_t *hashchain, KSI_DataHash **pRoot, ksierrctx_t *ectx); + + void outputHash(FILE *fp, const char *hdr, const uint8_t *data, const uint16_t len, const uint8_t verbose); + void outputKSIHash(FILE *fp, char *hdr, const KSI_DataHash *const __restrict__ hash, const uint8_t verbose); /* TODO: replace these? */ int hash_m_ksi(ksifile ksi, KSI_DataHash **m); int hash_r_ksi(ksifile ksi, KSI_DataHash **r, const unsigned char *rec, const size_t len); int hash_node_ksi(ksifile ksi, KSI_DataHash **node, KSI_DataHash *m, KSI_DataHash *r, uint8_t level); -extern char *rsksi_read_puburl; /**< url of publication server */ +extern char *rsksi_read_puburl; /**< url of publication server */ +extern char *rsksi_extend_puburl; /**< url of extension server */ +extern char *rsksi_userid; /**< userid for extension server */ +extern char *rsksi_userkey; /**< userkey for extension server */ extern uint8_t rsksi_read_showVerified; extern int RSKSI_FLAG_TLV16_RUNTIME; extern int RSKSI_FLAG_NONCRIT_RUNTIME; diff --git a/runtime/librsksi_read.c b/runtime/librsksi_read.c index 5bd525284..1fd3fe692 100644 --- a/runtime/librsksi_read.c +++ b/runtime/librsksi_read.c @@ -7,7 +7,7 @@ * This part of the library uses C stdio and expects that the * caller will open and close the file to be read itself. * - * Copyright 2013-2015 Adiscon GmbH. + * Copyright 2013-2016 Adiscon GmbH. * * This file is part of rsyslog. * @@ -44,6 +44,11 @@ #include "librsgt_common.h" #include "librsksi.h" +/* TODO: FIX Warnings! */ +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wunused-label" +#pragma GCC diagnostic ignored "-Wunused-parameter" + typedef unsigned char uchar; #ifndef VERSION #define VERSION "no-version" @@ -63,10 +68,6 @@ uint8_t rsksi_read_showVerified = 0; goto done; \ } -/* check return state of operation and abort, if non-OK */ -#define CHKr(code) if((r = code) != 0) goto done - - /* if verbose==0, only the first and last two octets are shown, * otherwise everything. */ @@ -84,11 +85,11 @@ outputHexBlob(FILE *fp, const uint8_t *blob, const uint16_t len, const uint8_t v } } -static inline void +void outputKSIHash(FILE *fp, char *hdr, const KSI_DataHash *const __restrict__ hash, const uint8_t verbose) { const unsigned char *digest; - unsigned digest_len; + size_t digest_len; KSI_DataHash_extract(hash, NULL, &digest, &digest_len); // TODO: error check fprintf(fp, "%s", hdr); @@ -96,9 +97,8 @@ outputKSIHash(FILE *fp, char *hdr, const KSI_DataHash *const __restrict__ hash, fputc('\n', fp); } -static inline void -outputHash(FILE *fp, const char *hdr, const uint8_t *data, - const uint16_t len, const uint8_t verbose) +void +outputHash(FILE *fp, const char *hdr, const uint8_t *data, const uint16_t len, const uint8_t verbose) { fprintf(fp, "%s", hdr); outputHexBlob(fp, data, len, verbose); @@ -198,23 +198,19 @@ reportError(const int errcode, ksierrctx_t *ectx) /* obviously, this is not an error-reporting function. We still use * ectx, as it has most information we need. */ - static void reportVerifySuccess(ksierrctx_t *ectx) /*OLD CODE , GTVerificationInfo *vrfyInf)*/ { - if(ectx->fp != NULL) { - fprintf(ectx->fp, "%s[%llu:%llu:%llu]: block signature successfully verified\n", - ectx->filename, - (long long unsigned) ectx->blkNum, (long long unsigned) ectx->recNum, - (long long unsigned) ectx->recNumInFile); - if(ectx->frstRecInBlk != NULL) - fprintf(ectx->fp, "\tBlock Start Record.: '%s'\n", ectx->frstRecInBlk); - if(ectx->errRec != NULL) - fprintf(ectx->fp, "\tBlock End Record...: '%s'\n", ectx->errRec); - fprintf(ectx->fp, "\tKSI Verify Signature: [%u]%s\n", - ectx->ksistate, KSI_getErrorString(ectx->ksistate)); - /* OLDCODE: NOT NEEDED ANYMORE GTVerificationInfo_print(ectx->fp, 0, vrfyInf);*/ - } + fprintf(stdout, "%s[%llu:%llu:%llu]: block signature successfully verified\n", + ectx->filename, + (long long unsigned) ectx->blkNum, (long long unsigned) ectx->recNum, + (long long unsigned) ectx->recNumInFile); + if(ectx->frstRecInBlk != NULL) + fprintf(stdout, "\tBlock Start Record.: '%s'\n", ectx->frstRecInBlk); + if(ectx->errRec != NULL) + fprintf(stdout, "\tBlock End Record...: '%s'\n", ectx->errRec); + fprintf(stdout, "\tKSI Verify Signature: [%u]%s\n", + ectx->ksistate, KSI_getErrorString(ectx->ksistate)); } /* return the actual length in to-be-written octets of an integer */ @@ -328,7 +324,7 @@ done: return r; * * @returns 0 if ok, something else otherwise */ -static int +int rsksi_tlvwrite(FILE *fp, tlvrecord_t *rec) { int r = RSGTE_IO; @@ -337,6 +333,28 @@ rsksi_tlvwrite(FILE *fp, tlvrecord_t *rec) r = 0; done: return r; } +/* +int +rsksi_tlvWriteHashKSI(FILE *fp, ksifile ksi, uint16_t tlvtype, KSI_DataHash *rec) +{ + unsigned tlvlen; + int r; + const unsigned char *digest; + size_t digest_len; + r = KSI_DataHash_extract(rec, NULL, &digest, &digest_len); + if (r != KSI_OK){ + reportKSIAPIErr(ksi->ctx, ksi, "KSI_DataHash_extract", r); + goto done; + } + tlvlen = 1 + digest_len; + r = rsksi_tlv16Write(fp, 0x00, tlvtype, tlvlen); + if(r != 0) goto done; + r = rsksi_tlvfileAddOctet(fp, hashIdentifierKSI(ksi->hashAlg)); + if(r != 0) goto done; + r = rsksi_tlvfileAddOctetString(fp, (unsigned char*)digest, digest_len); +done: return r; +} +*/ /** * Read a header from a binary file. @@ -365,6 +383,9 @@ rsksi_tlvRecRead(FILE *fp, tlvrecord_t *rec) { int r = 1; int c; + /* Init record variables */ + rec->tlvtype = 0; + rec->tlvlen = 0; NEXTC; rec->hdr[0] = c; @@ -386,16 +407,17 @@ rsksi_tlvRecRead(FILE *fp, tlvrecord_t *rec) rec->hdr[1] = c; rec->tlvlen = c; } - if(fread(rec->data, (size_t) rec->tlvlen, 1, fp) != 1) { + if((r = fread(rec->data, (size_t) rec->tlvlen, 1, fp)) != 1) { r = feof(fp) ? RSGTE_EOF : RSGTE_IO; goto done; } - if(rsksi_read_debug) - printf("debug: read tlvtype %4.4x, len %u\n", (unsigned) rec->tlvtype, - (unsigned) rec->tlvlen); r = 0; -done: return r; +done: + /* Only show debug if no fail */ + if(rsksi_read_debug && r != 0 && r != RSGTE_EOF) + printf("debug: rsksi_tlvRecRead:\t read tlvtype %4.4x, len %u r=%d\n", (unsigned) rec->tlvtype, (unsigned) rec->tlvlen, r); + return r; } /* decode a sub-tlv record from an existing record's memory buffer @@ -406,49 +428,51 @@ rsksi_tlvDecodeSUBREC(tlvrecord_t *rec, uint16_t *stridx, tlvrecord_t *newrec) int r = 1; int c; - if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break #1\n"); goto done;} c = rec->data[(*stridx)++]; newrec->hdr[0] = c; newrec->tlvtype = c & 0x1f; if(c & RSKSI_FLAG_TLV16_RUNTIME) { /* tlv16? */ newrec->lenHdr = 4; - if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break #2\n"); goto done;} c = rec->data[(*stridx)++]; newrec->hdr[1] = c; newrec->tlvtype = (newrec->tlvtype << 8) | c; - if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break #3\n"); goto done;} c = rec->data[(*stridx)++]; newrec->hdr[2] = c; newrec->tlvlen = c << 8; - if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break #4\n"); goto done;} c = rec->data[(*stridx)++]; newrec->hdr[3] = c; newrec->tlvlen |= c; } else { - if(rec->tlvlen == *stridx) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen == *stridx) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break #5\n"); goto done;} c = rec->data[(*stridx)++]; newrec->lenHdr = 2; newrec->hdr[1] = c; newrec->tlvlen = c; } - if(rec->tlvlen < *stridx + newrec->tlvlen) {r=RSGTE_LEN; goto done;} + if(rec->tlvlen < *stridx + newrec->tlvlen) {r=RSGTE_LEN; if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSUBREC:\t\t break rec->tlvlen=%d newrec->tlvlen=%d stridx=%d #6\n", rec->tlvlen, newrec->tlvlen, *stridx); goto done;} memcpy(newrec->data, (rec->data)+(*stridx), newrec->tlvlen); *stridx += newrec->tlvlen; if(rsksi_read_debug) - printf("debug: read sub-tlv: tlvtype %4.4x, len %u\n", + printf("debug: rsksi_tlvDecodeSUBREC:\t\t Read subtlv: tlvtype %4.4x, len %u\n", (unsigned) newrec->tlvtype, (unsigned) newrec->tlvlen); r = 0; -done: return r; +done: + if(r != 0) /* Only on FAIL! */ + printf("debug: rsksi_tlvDecodeSUBREC:\t\t Failed, tlv record %4.4x with error %d\n", rec->tlvtype, r); + return r; } - -static int +int rsksi_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint) { int r = 1; - imprint_t *imp; + imprint_t *imp = NULL; if((imp = calloc(1, sizeof(imprint_t))) == NULL) { r = RSGTE_OOM; @@ -466,8 +490,164 @@ rsksi_tlvDecodeIMPRINT(tlvrecord_t *rec, imprint_t **imprint) *imprint = imp; r = 0; done: - if(rsksi_read_debug) - printf("debug: read tlvDecodeIMPRINT returned %d TLVLen=%d, HashID=%d\n", r, rec->tlvlen, imp->hashID); + if(r == 0) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeIMPRINT:\t\t returned %d TLVType=%4.4x, TLVLen=%d, HashID=%d\n", r, rec->tlvtype, rec->tlvlen, imp->hashID); + if (rsksi_read_debug) outputHash(stdout, "debug: rsksi_tlvDecodeIMPRINT:\t\t hash: ", imp->data, imp->len, 1); + } else { + /* Free memory on FAIL!*/ + printf("debug: rsksi_tlvDecodeIMPRINT:\t\t Failed, tlv record %4.4x with error %d\n", rec->tlvtype, r); + if (imp != NULL) + rsksi_objfree(rec->tlvtype, imp); + } + return r; +} +static int +rsksi_tlvDecodeSIB_HASH(tlvrecord_t *rec, uint16_t *strtidx, imprint_t *imp) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsksi_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x02)) { r = RSGTE_INVLTYP; goto done; } + imp->hashID = subrec.data[0]; + if(subrec.tlvlen != 1 + hashOutputLengthOctetsKSI(imp->hashID)) { + r = RSGTE_LEN; + goto done; + } + imp->len = subrec.tlvlen - 1; + if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(imp->data, subrec.data+1, subrec.tlvlen-1); + r = 0; +done: return r; +} +static int +rsksi_tlvDecodeREC_HASH(tlvrecord_t *rec, uint16_t *strtidx, imprint_t *imp) +{ + int r = 1; + tlvrecord_t subrec; + CHKr(rsksi_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x01)) { r = RSGTE_INVLTYP; goto done; } + imp->hashID = subrec.data[0]; + + if(subrec.tlvlen != 1 + hashOutputLengthOctetsKSI(imp->hashID)) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeREC_HASH:\t\t FAIL on subrec.tlvtype %4.4x subrec.tlvlen = %d\n", subrec.tlvtype, subrec.tlvlen); + r = RSGTE_LEN; + goto done; + } + imp->len = subrec.tlvlen - 1; + if((imp->data = (uint8_t*)malloc(imp->len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(imp->data, subrec.data+1, subrec.tlvlen-1); + r = 0; +done: + if(r == 0) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeREC_HASH:\t\t returned %d TLVType=%4.4x, TLVLen=%d\n", r, rec->tlvtype, rec->tlvlen); + } else + printf("debug: rsksi_tlvDecodeREC_HASH:\t\t Failed, TLVType=%4.4x, TLVLen=%d with error %d\n", rec->tlvtype, rec->tlvlen, r); + + return r; +} +static int +rsksi_tlvDecodeLEVEL_CORR(tlvrecord_t *rec, uint16_t *strtidx, uint8_t *levelcorr) +{ + int r = 1; + tlvrecord_t subrec; + + CHKr(rsksi_tlvDecodeSUBREC(rec, strtidx, &subrec)); + if(!(subrec.tlvtype == 0x01 && subrec.tlvlen == 1)) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeLEVEL_CORR:\t FAIL on subrec.tlvtype %4.4x subrec.tlvlen = %d\n", subrec.tlvtype, subrec.tlvlen); + r = RSGTE_FMT; + goto done; + } + *levelcorr = subrec.data[0]; + r = 0; +done: + if(r == 0) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeLEVEL_CORR:\t returned %d TLVType=%4.4x, TLVLen=%d\n", r, rec->tlvtype, rec->tlvlen); + } else + printf("debug: rsksi_tlvDecodeLEVEL_CORR:\t Failed, tlv record %4.4x with error %d\n", rec->tlvtype, r); + return r; +} + +static int +rsksi_tlvDecodeHASH_STEP(tlvrecord_t *rec, uint16_t *pstrtidx, block_hashstep_t **blhashstep) +{ + int r = 1; + uint16_t strtidx = 0; + tlvrecord_t subrec; + *blhashstep = NULL; /* Set to NULL by default first */ + + /* Init HashStep */ + block_hashstep_t *hashstep = NULL; + if((hashstep = calloc(1, sizeof(block_hashstep_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + hashstep->sib_hash.data = NULL; + + /* Get Haststep Subrecord now */ + CHKr(rsksi_tlvDecodeSUBREC(rec, pstrtidx, &subrec)); /* Add to external counter */ + hashstep->direction = subrec.tlvtype; /* TLVType is also the DIRECTION! */ + + /* Extract HASH and LEVEL Correction!*/ + CHKr(rsksi_tlvDecodeLEVEL_CORR(&subrec, &strtidx, &(hashstep->level_corr))); + CHKr(rsksi_tlvDecodeSIB_HASH(&subrec, &strtidx, &(hashstep->sib_hash))); + + if(strtidx != subrec.tlvlen) { + r = RSGTE_LEN; + goto done; + } + + *blhashstep = hashstep; + r = 0; +done: + if (r == 0) { + if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeHASH_STEP:\t returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + } else { + /* Free memory on FAIL!*/ + printf("debug: rsksi_tlvDecodeHASH_STEP:\t Failed, tlv record %4.4x with error %d\n", rec->tlvtype, r); + if (hashstep != NULL) { + if (hashstep->sib_hash.data != NULL) + free(hashstep->sib_hash.data); + free(hashstep); + } + } + return r; +} +int +rsksi_tlvDecodeHASH_CHAIN(tlvrecord_t *rec, block_hashchain_t **blhashchain) +{ + int r = 1; + uint16_t strtidx = 0; + /* Init HashChain Object */ + block_hashchain_t *hashchain = NULL; + if((hashchain = calloc(1, sizeof(block_hashchain_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + hashchain->rec_hash.data = NULL; + hashchain->stepCount = 0; + hashchain->level = 0; + + /* Extract hash chain */ + CHKr(rsksi_tlvDecodeREC_HASH(rec, &strtidx, &(hashchain->rec_hash))); + + /* Loop until all Steps have been processed */ + while(rec->tlvlen > strtidx) { + CHKr(rsksi_tlvDecodeHASH_STEP(rec, &strtidx, &(hashchain->hashsteps[hashchain->stepCount++]))); + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeHASH_CHAIN:\t tlvlen=%d strtidx=%d\n", rec->tlvlen, strtidx); + } + + *blhashchain = hashchain; + r = 0; +done: + if(r == 0) { + if (rsksi_read_debug) printf("debug: rsksi_tlvDecodeHASH_CHAIN:\t returned %d TLVType=%4.4x, TLVLen=%d\n", r, rec->tlvtype, rec->tlvlen); + } else { + /* Free memory on FAIL!*/ + printf("debug: rsksi_tlvDecodeHASH_CHAIN:\t Failed, TLVType=%4.4x, TLVLen=%d with error %d\n", rec->tlvtype, rec->tlvlen, r); + if (hashchain != NULL) + rsksi_objfree(rec->tlvtype, hashchain); + } return r; } @@ -546,13 +726,15 @@ rsksi_tlvDecodeSIG(tlvrecord_t *rec, uint16_t *strtidx, block_sig_t *bs) tlvrecord_t subrec; CHKr(rsksi_tlvDecodeSUBREC(rec, strtidx, &subrec)); - if(!(subrec.tlvtype == 0x0906)) { r = RSGTE_INVLTYP; goto done; } + if(!(subrec.tlvtype == 0x0905)) { r = RSGTE_INVLTYP; goto done; } bs->sig.der.len = subrec.tlvlen; bs->sigID = SIGID_RFC3161; if((bs->sig.der.data = (uint8_t*)malloc(bs->sig.der.len)) == NULL) {r=RSGTE_OOM;goto done;} memcpy(bs->sig.der.data, subrec.data, bs->sig.der.len); r = 0; -done: return r; +done: + if(rsksi_read_debug) printf("debug: rsksi_tlvDecodeSIG:\t\t returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + return r; } static int @@ -560,7 +742,7 @@ rsksi_tlvDecodeBLOCK_HDR(tlvrecord_t *rec, block_hdr_t **blockhdr) { int r = 1; uint16_t strtidx = 0; - block_hdr_t *bh; + block_hdr_t *bh = NULL; if((bh = calloc(1, sizeof(block_hdr_t))) == NULL) { r = RSGTE_OOM; goto done; @@ -574,15 +756,53 @@ rsksi_tlvDecodeBLOCK_HDR(tlvrecord_t *rec, block_hdr_t **blockhdr) } *blockhdr = bh; r = 0; -done: return r; +done: + if (r == 0) { + if(rsksi_read_debug) printf("debug: tlvDecodeBLOCK_HDR:\t\t returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + } else { + /* Free memory on FAIL!*/ + if (bh != NULL) + rsksi_objfree(rec->tlvtype, bh); + } + return r; } +static int +rsksi_tlvDecodeEXCERPT_SIG(tlvrecord_t *rec, block_sig_t **blocksig) +{ + int r = 1; + block_sig_t *bs = NULL; + if((bs = calloc(1, sizeof(block_sig_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + + /* Read signature now */ + if(!(rec->tlvtype == 0x0905)) { r = RSGTE_INVLTYP; goto done; } + bs->recCount = 0; + bs->sig.der.len = rec->tlvlen; + bs->sigID = SIGID_RFC3161; + if((bs->sig.der.data = (uint8_t*)malloc(bs->sig.der.len)) == NULL) {r=RSGTE_OOM;goto done;} + memcpy(bs->sig.der.data, rec->data, bs->sig.der.len); + + *blocksig = bs; + r = 0; +done: + if(r == 0) { + if (rsksi_read_debug) printf("debug: tlvDecodeEXCERPT_SIG:\t returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); + } else { + /* Free memory on FAIL!*/ + if (bs != NULL) + rsksi_objfree(rec->tlvtype, bs); + } + return r; +} static int rsksi_tlvDecodeBLOCK_SIG(tlvrecord_t *rec, block_sig_t **blocksig) { int r = 1; uint16_t strtidx = 0; - block_sig_t *bs; + block_sig_t *bs = NULL; if((bs = calloc(1, sizeof(block_sig_t))) == NULL) { r = RSGTE_OOM; goto done; @@ -595,9 +815,17 @@ rsksi_tlvDecodeBLOCK_SIG(tlvrecord_t *rec, block_sig_t **blocksig) } *blocksig = bs; r = 0; -done: return r; +done: + if(r == 0) { + if (rsksi_read_debug) printf("debug: tlvDecodeBLOCK_SIG:\t\t returned %d, tlvtype %4.4x, recCount %ju\n", r, (unsigned) rec->tlvtype, bs->recCount); + } else { + /* Free memory on FAIL!*/ + if (bs != NULL) + rsksi_objfree(rec->tlvtype, bs); + } + return r; } -static int +int rsksi_tlvRecDecode(tlvrecord_t *rec, void *obj) { int r = 1; @@ -615,10 +843,17 @@ rsksi_tlvRecDecode(tlvrecord_t *rec, void *obj) r = rsksi_tlvDecodeBLOCK_SIG(rec, obj); if(r != 0) goto done; break; + case 0x0905: + r = rsksi_tlvDecodeEXCERPT_SIG(rec, obj); + if(r != 0) goto done; + break; + case 0x0907: + r = rsksi_tlvDecodeHASH_CHAIN(rec, obj); + if(r != 0) goto done; + break; } done: - if(rsksi_read_debug) - printf("debug: read tlvRecDecode returned %d \n", r); + if(rsksi_read_debug) printf("debug: rsksi_tlvRecDecode:\t\t returned %d, tlvtype %4.4x\n", r, (unsigned) rec->tlvtype); return r; } @@ -630,14 +865,20 @@ rsksi_tlvrdRecHash(FILE *fp, FILE *outfp, imprint_t **imp) if((r = rsksi_tlvrd(fp, &rec, imp)) != 0) goto done; if(rec.tlvtype != 0x0902) { + if(rsksi_read_debug) printf("debug: rsksi_tlvrdRecHash:\t\t\t expected tlvtype 0x0902, but was %4.4x\n", rec.tlvtype); r = RSGTE_MISS_REC_HASH; rsksi_objfree(rec.tlvtype, *imp); + *imp = NULL; goto done; } - if(outfp != NULL) + if(outfp != NULL) { if((r = rsksi_tlvwrite(outfp, &rec)) != 0) goto done; + } r = 0; -done: return r; +done: + if(r == 0 && rsksi_read_debug) + printf("debug: tlvrdRecHash:\t\t\t returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); + return r; } static int @@ -648,14 +889,19 @@ rsksi_tlvrdTreeHash(FILE *fp, FILE *outfp, imprint_t **imp) if((r = rsksi_tlvrd(fp, &rec, imp)) != 0) goto done; if(rec.tlvtype != 0x0903) { + if(rsksi_read_debug) printf("debug: rsksi_tlvrdTreeHash:\t\t expected tlvtype 0x0903, but was %4.4x\n", rec.tlvtype); r = RSGTE_MISS_TREE_HASH; rsksi_objfree(rec.tlvtype, *imp); + *imp = NULL; goto done; } - if(outfp != NULL) + if(outfp != NULL) { if((r = rsksi_tlvwrite(outfp, &rec)) != 0) goto done; + } r = 0; -done: return r; +done: + if(r == 0 && rsksi_read_debug) printf("debug: rsksi_tlvrdTreeHash:\t\t returned %d, rec->tlvtype %4.4x\n", r, (unsigned) rec.tlvtype); + return r; } /* read BLOCK_SIG during verification phase */ @@ -666,8 +912,9 @@ rsksi_tlvrdVrfyBlockSig(FILE *fp, block_sig_t **bs, tlvrecord_t *rec) if((r = rsksi_tlvrd(fp, rec, bs)) != 0) goto done; if(rec->tlvtype != 0x0904) { + if(rsksi_read_debug) printf("debug: rsksi_tlvrdVrfyBlockSig:\t expected tlvtype 0x0904, but was %4.4x\n", rec->tlvtype); r = RSGTE_MISS_BLOCKSIG; - rsksi_objfree(rec->tlvtype, *bs); + /* NOT HERE, done above ! rsksi_objfree(rec->tlvtype, *bs); */ goto done; } r = 0; @@ -698,7 +945,9 @@ rsksi_tlvrd(FILE *fp, tlvrecord_t *rec, void *obj) int r; if((r = rsksi_tlvRecRead(fp, rec)) != 0) goto done; r = rsksi_tlvRecDecode(rec, obj); -done: return r; +done: + if(rsksi_read_debug && r != RSGTE_SUCCESS && r != RSGTE_EOF) printf("debug: rsksi_tlvrd:\t failed with error %d\n", r); + return r; } @@ -807,7 +1056,7 @@ rsksi_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose) case 0x0904: rsksi_printBLOCK_SIG(fp, obj, verbose); break; - default:fprintf(fp, "unknown tlv record %4.4x\n", tlvtype); + default:fprintf(fp, "rsksi_tlvprint :\t unknown tlv record %4.4x\n", tlvtype); break; } } @@ -821,25 +1070,80 @@ rsksi_tlvprint(FILE *fp, uint16_t tlvtype, void *obj, uint8_t verbose) void rsksi_objfree(uint16_t tlvtype, void *obj) { + int j; + // check if obj is valid + if (obj == NULL ) + return; + switch(tlvtype) { case 0x0901: - free(((block_hdr_t*)obj)->iv); - free(((block_hdr_t*)obj)->lastHash.data); + if ( ((block_hdr_t*)obj)->iv != NULL) + free(((block_hdr_t*)obj)->iv); + if ( ((block_hdr_t*)obj)->lastHash.data != NULL) + free(((block_hdr_t*)obj)->lastHash.data); break; case 0x0902: case 0x0903: free(((imprint_t*)obj)->data); break; - case 0x0904: - free(((block_sig_t*)obj)->sig.der.data); + case 0x0904: /* signature data for a log block */ + case 0x0905: /* signature data for a log block */ + if ( ((block_sig_t*)obj)->sig.der.data != NULL) { + free(((block_sig_t*)obj)->sig.der.data); + } break; - default:fprintf(stderr, "rsksi_objfree: unknown tlv record %4.4x\n", - tlvtype); + case 0x0907: /* Free Hash Chain */ + if ( ((block_hashchain_t*)obj)->rec_hash.data != NULL) { + free(((block_hashchain_t*)obj)->rec_hash.data); + } + /* Loop through Step Objects and delete mem */ + if (((block_hashchain_t*)obj)->stepCount > 0) { + for(j = 0 ; j < ((block_hashchain_t*)obj)->stepCount ; ++j) { + if (((block_hashchain_t*)obj)->hashsteps[j]->sib_hash.data != NULL) { + free(((block_hashchain_t*)obj)->hashsteps[j]->sib_hash.data); + } + } + } + break; + default:fprintf(stderr, "rsksi_objfree:\t unknown tlv record %4.4x\n", tlvtype); break; } free(obj); } +block_hashstep_t* +rsksiHashstepFromKSI_DataHash(ksifile ksi, KSI_DataHash *hash) +{ + int r; + const unsigned char *digest; + size_t digest_len; + block_hashstep_t* hashstep; + + if((hashstep = calloc(1, sizeof(block_hashstep_t))) == NULL) { + goto done; + } + + /* Get imprint from KSI_Hash */ + KSI_HashAlgorithm hashID; + r = KSI_DataHash_extract(hash, &hashID, &digest, &digest_len); + if (r != KSI_OK){ + reportKSIAPIErr(ksi->ctx, ksi, "KSI_DataHash_extract", r); + free(hashstep); hashstep = NULL; + goto done; + } + + /* Fill Hashstep object */ + hashstep->sib_hash.hashID = hashID; + hashstep->sib_hash.len = digest_len; + if((hashstep->sib_hash.data = (uint8_t*)malloc(hashstep->sib_hash.len)) == NULL) { + free(hashstep); hashstep = NULL; + goto done; + } + memcpy(hashstep->sib_hash.data, digest, digest_len); +done: + return hashstep; +} + /** * Read block parameters. This detects if the block contains the * individual log hashes, the intermediate hashes and the overall @@ -864,10 +1168,10 @@ rsksi_objfree(uint16_t tlvtype, void *obj) * @returns 0 if ok, something else otherwise */ int -rsksi_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t **bh, - uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes) +rsksi_getBlockParams(ksifile ksi, FILE *fp, uint8_t bRewind, block_sig_t **bs, + block_hdr_t **bh, uint8_t *bHasRecHashes, uint8_t *bHasIntermedHashes) { - int r; + int r = RSGTE_SUCCESS; uint64_t nRecs = 0; uint8_t bDone = 0; uint8_t bHdr = 0; @@ -920,9 +1224,100 @@ rsksi_getBlockParams(FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t ** } } done: + if(rsksi_read_debug && r != RSGTE_EOF && r != RSGTE_SUCCESS) printf("debug: rsksi_getBlockParams:\t returned %d\n", r); return r; } +/** + * Read Excerpt block parameters. This detects if the block contains + * hash chains for log records. + * If a caller intends to verify a log file based on the parameters, + * he must re-read the file from the begining (we could keep things + * in memory, but this is impractical for large blocks). In order + * to facitate this, the function permits to rewind to the original + * read location when it is done. + * + * @param[in] fp file pointer of tlv file + * @param[in] bRewind 0 - do not rewind at end of procesing, 1 - do so + * @param[out] bs block signature record + * + * @returns 0 if ok, something else otherwise + */ +int +rsksi_getExcerptBlockParams(ksifile ksi, FILE *fp, uint8_t bRewind, block_sig_t **bs, block_hdr_t **bh) +{ + int r = RSGTE_SUCCESS; + uint64_t nRecs = 0; + uint8_t bSig = 0; + off_t rewindPos = 0; + void *obj; + tlvrecord_t rec; + + /* Initial RewindPos */ + if(bRewind) rewindPos = ftello(fp); + *bs = NULL; + + /* Init Blockheader */ + if((*bh = calloc(1, sizeof(block_hdr_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + (*bh)->iv = NULL; + (*bh)->lastHash.data = NULL; + + while(r == RSGTE_SUCCESS && bSig == 0) { /* we will err out on EOF */ + if((r = rsksi_tlvrd(fp, &rec, &obj)) != 0) goto done; + switch(rec.tlvtype) { + case 0x0905: /* OpenKSI signature | Excerpt File */ + if (*bs == NULL ) { + *bs = (block_sig_t*) obj; + + /* Save NEW RewindPos */ + if(bRewind) rewindPos = ftello(fp); + } else { + /* Previous Block finished */ + bSig = 1; + } + break; + case 0x0907: /* hash chain for one log record | Excerpt File */ + if (*bs != NULL) { + if (nRecs == 0) /* Copy HASHID from record hash */ + (*bh)->hashID = ((block_hashchain_t*)obj)->rec_hash.hashID; + /* Increment hash chain count */ + nRecs++; + } + break; + default:fprintf(fp, "unknown tlv record %4.4x\n", rec.tlvtype); + break; + } + + /* Free second Signatur object if set! */ + if(bSig == 1 && obj != NULL) rsksi_objfree(rec.tlvtype, obj); + } +done: + if (*bs != NULL) { + if (r == RSGTE_EOF) { + if(rsksi_read_debug) printf("debug: rsksi_getExcerptBlockParams:\t Reached END of FILE\n"); + r = RSGTE_SUCCESS; + } + } else { + goto done2; + } + + /* Copy Count back! */ + (*bs)->recCount = nRecs; + + /* Rewind file back */ + if(bRewind) { + if(fseeko(fp, rewindPos, SEEK_SET) != 0) { + r = RSGTE_IO; + goto done2; + } + } +done2: + if(rsksi_read_debug) printf("debug: rsksi_getExcerptBlockParams:\t Found %lld records, returned %d\n", (long long unsigned)nRecs, r); + return r; +} /** * Read the file header and compare it to the expected value. @@ -932,17 +1327,22 @@ done: * @returns 0 if ok, something else otherwise */ int -rsksi_chkFileHdr(FILE *fp, char *expect) +rsksi_chkFileHdr(FILE *fp, char *expect, uint8_t verbose) { int r; char hdr[9]; + off_t rewindPos = ftello(fp); if((r = rsksi_tlvrdHeader(fp, (uchar*)hdr)) != 0) goto done; - if(strcmp(hdr, expect)) + if(strcmp(hdr, expect)) { r = RSGTE_INVLHDR; + fseeko(fp, rewindPos, SEEK_SET); /* Reset Filepointer on failure for additional checks*/ + } else r = 0; done: + if(r != RSGTE_SUCCESS && verbose) + printf("rsksi_chkFileHdr:\t\t failed expected '%s' but was '%s'\n", expect, hdr); return r; } @@ -959,12 +1359,25 @@ rsksi_vrfyConstruct_gf(void) rsksictx ctx = rsksiCtxNew(); ksi->ctx = ctx; /* assign context to ksifile */ - /* Setting KSI Extender! */ - ksistate = KSI_CTX_setExtender(ksi->ctx->ksi_ctx, rsksi_read_puburl, rsksi_userid, rsksi_userkey); + /* Setting KSI Publication URL ! */ + ksistate = KSI_CTX_setPublicationUrl(ksi->ctx->ksi_ctx, rsksi_read_puburl); if(ksistate != KSI_OK) { - fprintf(stderr, "Error %d setting KSI Extender: \n", ksistate); - return NULL; + fprintf(stderr, "Failed setting KSI Publication URL '%s' with error (%d): %s\n", rsksi_read_puburl, ksistate, KSI_getErrorString(ksistate)); + free(ksi); + return NULL; } + if(rsksi_read_debug) + fprintf(stdout, "PublicationUrl set to: '%s'\n", rsksi_read_puburl); + + /* Setting KSI Extender! */ + ksistate = KSI_CTX_setExtender(ksi->ctx->ksi_ctx, rsksi_extend_puburl, rsksi_userid, rsksi_userkey); + if(ksistate != KSI_OK) { + fprintf(stderr, "Failed setting KSIExtender URL '%s' with error (%d): %s\n", rsksi_extend_puburl, ksistate, KSI_getErrorString(ksistate)); + free(ksi); + return NULL; + } + if(rsksi_read_debug) + fprintf(stdout, "ExtenderUrl set to: '%s'\n", rsksi_extend_puburl); done: return ksi; } @@ -975,14 +1388,23 @@ rsksi_vrfyBlkInit(ksifile ksi, block_hdr_t *bh, uint8_t bHasRecHashes, uint8_t b ksi->hashAlg = hashID2AlgKSI(bh->hashID); ksi->bKeepRecordHashes = bHasRecHashes; ksi->bKeepTreeHashes = bHasIntermedHashes; - free(ksi->IV); - ksi->IV = malloc(getIVLenKSI(bh)); - memcpy(ksi->IV, bh->iv, getIVLenKSI(bh)); - ksi->x_prev = malloc(sizeof(imprint_t)); - ksi->x_prev->len=bh->lastHash.len; - ksi->x_prev->hashID = bh->lastHash.hashID; - ksi->x_prev->data = malloc(ksi->x_prev->len); - memcpy(ksi->x_prev->data, bh->lastHash.data, ksi->x_prev->len); + if (ksi->IV != NULL ) { + free(ksi->IV); + ksi->IV = NULL; + } + if (bh->iv != NULL) { + ksi->IV = malloc(getIVLenKSI(bh)); + memcpy(ksi->IV, bh->iv, getIVLenKSI(bh)); + } + if (bh->lastHash.data != NULL ) { + ksi->x_prev = malloc(sizeof(imprint_t)); + ksi->x_prev->len=bh->lastHash.len; + ksi->x_prev->hashID = bh->lastHash.hashID; + ksi->x_prev->data = malloc(ksi->x_prev->len); + memcpy(ksi->x_prev->data, bh->lastHash.data, ksi->x_prev->len); + } else { + ksi->x_prev = NULL; + } } static int @@ -1038,25 +1460,27 @@ rsksi_vrfy_chkTreeHash(ksifile ksi, FILE *sigfp, FILE *nsigfp, r = RSGTE_INVLD_TREE_HASHID; goto done; } - if(memcmp(imp->data, digest, - hashOutputLengthOctetsKSI(imp->hashID))) { + if(memcmp(imp->data, digest, hashOutputLengthOctetsKSI(imp->hashID))) { r = RSGTE_INVLD_TREE_HASH; ectx->computedHash = hash; ectx->fileHash = imp; reportError(r, ectx); ectx->computedHash = NULL, ectx->fileHash = NULL; goto done; - } + } r = 0; done: - if(imp != NULL) + if(imp != NULL) { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_chkTreeHash:\t\t returned %d, hashID=%d, Length=%d\n", r, imp->hashID, hashOutputLengthOctetsKSI(imp->hashID)); + /* Free memory */ rsksi_objfree(0x0903, imp); + } return r; } +/* Helper function to verifiy the next record in the signature file */ int -rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, - unsigned char *rec, size_t len, ksierrctx_t *ectx) +rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx) { int r = 0; KSI_DataHash *x; /* current hash */ @@ -1065,6 +1489,7 @@ rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, hash_m_ksi(ksi, &m); hash_r_ksi(ksi, &recHash, rec, len); + if(ksi->bKeepRecordHashes) { r = rsksi_vrfy_chkRecHash(ksi, sigfp, nsigfp, recHash, ectx); if(r != 0) goto done; @@ -1077,8 +1502,11 @@ rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, r = rsksi_vrfy_chkTreeHash(ksi, sigfp, nsigfp, x, ectx); if(r != 0) goto done; } + + /* Store Current Hash for later use */ rsksiimprintDel(ksi->x_prev); ksi->x_prev = rsksiImprintFromKSI_DataHash(ksi, x); + /* add x to the forest as new leaf, update roots list */ t = x; for(j = 0 ; j < ksi->nRoots ; ++j) { @@ -1116,26 +1544,362 @@ rsksi_vrfy_nextRec(ksifile ksi, FILE *sigfp, FILE *nsigfp, /* cleanup */ KSI_DataHash_free(m); done: - if(recHash != NULL) - KSI_DataHash_free(recHash); + if(recHash != NULL) KSI_DataHash_free(recHash); + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextRec:\t\t returned %d\n", r); return r; } +/* Helper function to verifiy the next record in the signature file */ +int +rsksi_vrfy_nextRecExtract(ksifile ksi, FILE *sigfp, FILE *nsigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx, block_hashchain_t *hashchain, int storehashchain) +{ + int r = 0; + KSI_DataHash *x; /* current hash */ + KSI_DataHash *m, *recHash = NULL, *t, *t_del; + uint8_t j; + block_hashstep_t *hashstep = NULL; + + hash_m_ksi(ksi, &m); + hash_r_ksi(ksi, &recHash, rec, len); + + if(ksi->bKeepRecordHashes) { + r = rsksi_vrfy_chkRecHash(ksi, sigfp, nsigfp, recHash, ectx); + if(r != 0) goto done; + } + hash_node_ksi(ksi, &x, m, recHash, 1); /* hash leaf */ + if(ksi->bKeepTreeHashes) { + ectx->treeLevel = 0; + ectx->lefthash = m; + ectx->righthash = recHash; + r = rsksi_vrfy_chkTreeHash(ksi, sigfp, nsigfp, x, ectx); + if(r != 0) goto done; + } + +/* EXTRA DEBUG !!!! */ +if(rsksi_read_debug) { + outputKSIHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t Tree Left Hash.....: ", m, ectx->verbose); + outputKSIHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t Tree Right Hash....: ", recHash, ectx->verbose); + outputKSIHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t Tree Current Hash..: ", x, ectx->verbose); +} + + /* Store Current Hash for later use */ + rsksiimprintDel(ksi->x_prev); + ksi->x_prev = rsksiImprintFromKSI_DataHash(ksi, x); + + if (storehashchain == 1) { + /* Store record hash for HashChain */ + if (hashchain->rec_hash.data == NULL) { + /* Extract and copy record imprint*/ + rsksiIntoImprintFromKSI_DataHash( &(hashchain->rec_hash), ksi, x ); + if(rsksi_read_debug) outputHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t RECORD Hash: \t\t", hashchain->rec_hash.data, hashchain->rec_hash.len, ectx->verbose); + hashchain->direction = 0x03; /* RIGHT */ + hashstep = rsksiHashstepFromKSI_DataHash(ksi, m); + if (hashstep == NULL ) { r = RSGTE_IO; goto done; } + hashstep->direction = hashchain->direction; + hashstep->level_corr = 0; /* Level Correction 0 */ + if(rsksi_read_debug) outputHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t RIGHT Hash: \t\t", hashstep->sib_hash.data, hashstep->sib_hash.len, ectx->verbose); + + /* Attach to HashChain */ + hashchain->hashsteps[hashchain->stepCount] = hashstep; + hashchain->stepCount++; + + hashchain->direction = 0x03; /* RIGHT */ + hashchain->level = 1; + } + } + + /* add x to the forest as new leaf, update roots list */ + t = x; + + for(j = 0 ; j < ksi->nRoots ; ++j) { + if(ksi->roots_valid[j] == 0) { + if ((j+1) == hashchain->level) { +/* NOT SURE ABOUT j+1 ! */ + hashchain->direction = 0x02; /* LEFT */ + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextRecExtract:\t LEFT DIRECTION %d!!!\n", j); + } + ksi->roots_hash[j] = t; + ksi->roots_valid[j] = 1; + t = NULL; + break; + } else if(t != NULL) { + if ((j+1) == hashchain->level) { +/* NOT SURE ABOUT j+1 ! */ + if (hashchain->direction == 0x03) { /*RIGHT*/ + hashstep = rsksiHashstepFromKSI_DataHash(ksi, ksi->roots_hash[j]); + if(hashstep == NULL ) { r = RSGTE_IO; goto done; } + if(rsksi_read_debug) { + printf("debug: rsksi_vrfy_nextRecExtract:\t RIGHT DIRECTION Level %d!!!\n", j); + outputHash(stdout, "debug: rsksi_vrfy_nextRecExtract: \t RIGHT Hash: \t\t", hashstep->sib_hash.data, hashstep->sib_hash.len, ectx->verbose); + } + } else { /*LEFT*/ + hashstep = rsksiHashstepFromKSI_DataHash(ksi, t); + if(hashstep == NULL ) { r = RSGTE_IO; goto done; } + if(rsksi_read_debug) { + printf("debug: rsksi_vrfy_nextRecExtract:\t LEFT DIRECTION Level %d!!!\n", j); + outputHash(stdout, "debug: rsksi_vrfy_nextRecExtract: \t LEFT Hash: \t\t", hashstep->sib_hash.data, hashstep->sib_hash.len, ectx->verbose); + } + } + hashstep->direction = hashchain->direction; + hashstep->level_corr = j; + + /* Attach to HashChain */ + hashchain->hashsteps[hashchain->stepCount] = hashstep; + hashchain->stepCount++; + + /* Set Direction and Chainlevel */ + hashchain->direction = 0x03; /*RIGHT*/ + hashchain->level = j+1; + } + + /* hash interim node */ + ectx->treeLevel = j+1; + ectx->righthash = t; + t_del = t; + hash_node_ksi(ksi, &t, ksi->roots_hash[j], t_del, j+2); + ksi->roots_valid[j] = 0; + if(ksi->bKeepTreeHashes) { + ectx->lefthash = ksi->roots_hash[j]; + r = rsksi_vrfy_chkTreeHash(ksi, sigfp, nsigfp, t, ectx); + if(r != 0) goto done; /* mem leak ok, we terminate! */ + } + KSI_DataHash_free(ksi->roots_hash[j]); + KSI_DataHash_free(t_del); + } + } + + if(t != NULL) { + if(rsksi_read_debug) outputKSIHash(stdout, "debug: rsksi_vrfy_nextRecExtract:\t New Root Hash....: ", t, ectx->verbose); + if (ksi->nRoots < hashchain->level) + hashchain->direction = 0x02; /*LEFT*/ + + /* new level, append "at the top" */ + ksi->roots_hash[ksi->nRoots] = t; + ksi->roots_valid[ksi->nRoots] = 1; + ++ksi->nRoots; + assert(ksi->nRoots < MAX_ROOTS); + t = NULL; + } + ++ksi->nRecords; + + /* cleanup */ + KSI_DataHash_free(m); +done: + if(recHash != NULL) KSI_DataHash_free(recHash); +if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextRecExtract:\t returned %d\n", r); + return r; +} + +/* Helper function to verifiy the next hash chain record in the signature file */ +int +rsksi_vrfy_nextHashChain(ksifile ksi, block_sig_t *bs, FILE *sigfp, unsigned char *rec, size_t len, ksierrctx_t *ectx) +{ + unsigned int j; + int r = 0; + int ksistate; + KSI_Signature *sig = NULL; + KSI_DataHash *line_hash = NULL, *root_hash = NULL, *root_tmp = NULL; + KSI_DataHash *rec_hash = NULL, *sibling_hash = NULL; /* left_hash = NULL, *right_hash = NULL; */ + int bCheckLineHash = 0; /* Line Hash will be checked after first record !*/ + void *obj; + tlvrecord_t tlvrec; + block_hashchain_t *hashchain = NULL; + uint8_t uiLevelCorr = 0; + + /* Check for next valid tlvrecord */ + if ((r = rsksi_tlvrd(sigfp, &tlvrec, &obj)) != 0) goto done; + if (tlvrec.tlvtype != 0x0907) { + r = RSGTE_INVLTYP; + goto done; + } + + /* Convert Pointer to block_hashchain_t*/ + hashchain = (block_hashchain_t*)obj; + + /* Verify Hash Alg */ + if(hashchain->rec_hash.hashID != hashIdentifierKSI(ksi->hashAlg)) { + reportError(r, ectx); + r = RSGTE_INVLD_REC_HASHID; + goto done; + } + + /* Create Hash from Line */ + hash_r_ksi(ksi, &line_hash, rec, len); + if(rsksi_read_debug) outputKSIHash(stdout, "debug: rsksi_vrfy_nextHashChain:\t Line Hash.:.............: ", line_hash, ectx->verbose); + + /* Convert Record hash from HashChain into KSI_DataHash */ + KSI_DataHash_fromDigest (ksi->ctx->ksi_ctx, hashchain->rec_hash.hashID, hashchain->rec_hash.data, hashchain->rec_hash.len, &rec_hash); + if(rsksi_read_debug) outputKSIHash(stdout, "debug: rsksi_vrfy_nextHashChain:\t HashChain Record Hash...: ", rec_hash, ectx->verbose); + + /* Clone Line Hash for first time*/ + if ( KSI_DataHash_clone(line_hash, &root_tmp) != KSI_OK ) { + r = RSGTE_IO; + goto done; + } + + /* Loop through hashchain now */ + for(j = 0 ; j < hashchain->stepCount ; ++j) { + /* Set Level correction */ + uiLevelCorr += hashchain->hashsteps[j]->level_corr + 1; + + /* Convert Sibling hash from HashChain into KSI_DataHash */ + KSI_DataHash_fromDigest (ksi->ctx->ksi_ctx, hashchain->hashsteps[j]->sib_hash.hashID, hashchain->hashsteps[j]->sib_hash.data, hashchain->hashsteps[j]->sib_hash.len, &sibling_hash); + if(rsksi_read_debug) outputKSIHash(stdout, "debug: rsksi_vrfy_nextHashChain:\t HashChain Sibling Hash..: ", sibling_hash, ectx->verbose); + + if (hashchain->hashsteps[j]->direction == 0x02 /* LEFT */){ + /* Combine Root Hash with LEFT Sibling */ + hash_node_ksi(ksi, &root_hash, root_tmp, sibling_hash, uiLevelCorr); + } else /* RIGHT */ { + /* Combine Root Hash with RIGHT Sibling */ + hash_node_ksi(ksi, &root_hash, sibling_hash, root_tmp, uiLevelCorr); + } + KSI_DataHash_free(root_tmp); /* Free tmp hash*/ + if(rsksi_read_debug) outputKSIHash(stdout, "debug: rsksi_vrfy_nextHashChain:\t HashChain New Root Hash.: ", root_hash, ectx->verbose); + + /* First Sibling, check */ + if (bCheckLineHash == 0) { + /* Compare root_hash vs rec_hash */ + if ( KSI_DataHash_equals (root_hash, rec_hash) != 1 ) { + r = RSGTE_INVLD_REC_HASH; + ectx->computedHash = root_hash; + ectx->fileHash = &(hashchain->rec_hash); + reportError(r, ectx); + ectx->computedHash = NULL, ectx->fileHash = NULL; + goto done; + } else { + if(rsksi_read_showVerified) printf("Successfully compared line hash against record hash\n"); + } + bCheckLineHash = 1; + } + + /* Store into TMP for next LOOP */ + root_tmp = root_hash; + + /* Free memory */ + if(sibling_hash != NULL) KSI_DataHash_free(sibling_hash); + } + + /* Parse KSI Signature */ + ksistate = KSI_Signature_parse(ksi->ctx->ksi_ctx, bs->sig.der.data, bs->sig.der.len, &sig); + if(ksistate != KSI_OK) { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_parse failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); + r = RSGTE_INVLD_SIGNATURE; + ectx->ksistate = ksistate; + goto done; + } else { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_parse was successfull\n"); + } + + /* Verify KSI Signature */ + ksistate = KSI_Signature_verify(sig, ksi->ctx->ksi_ctx); + if(ksistate != KSI_OK) { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_verify failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); + r = RSGTE_INVLD_SIGNATURE; + ectx->ksistate = ksistate; + goto done; + } else { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_verify was successfull\n"); + } + + /* Verify Roothash against Signature */ + ksistate = KSI_Signature_verifyDataHash(sig, ksi->ctx->ksi_ctx, root_hash); + if (ksistate != KSI_OK) { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_verifyDataHash failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); + r = RSGTE_INVLD_SIGNATURE; + ectx->ksistate = ksistate; + goto done; + } else { + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t KSI_Signature_parse was successfull\n"); + if(rsksi_read_showVerified) + reportVerifySuccess(ectx); + } +done: + /* Free Memory */ + if(root_hash != NULL) KSI_DataHash_free(root_hash); + if(line_hash != NULL) KSI_DataHash_free(line_hash); + if(rec_hash != NULL) KSI_DataHash_free(rec_hash); + if(hashchain != NULL) rsksi_objfree(0x0907, hashchain); + + if(rsksi_read_debug) printf("debug: rsksi_vrfy_nextHashChain:\t returned %d\n", r); + return r; +} + +/* Finish Verify Sigblock with additional HashChain handling */ +int +verifySigblkFinishChain(ksifile ksi, block_hashchain_t *hashchain, KSI_DataHash **pRoot, ksierrctx_t *ectx) +{ + KSI_DataHash *root, *rootDel; + int8_t j; + int r = 0; + root = NULL; + block_hashstep_t *hashstep = NULL; + + if(ksi->nRecords == 0) { + if(rsksi_read_debug) printf("debug: verifySigblkFinishChain:\t\t Error, No records found! %d\n", r); + goto done; + } + + for(j = 0 ; j < ksi->nRoots ; ++j) { + if(root == NULL) { + if (j == hashchain->level ) { + hashchain->direction = 0x03; /*RIGHT*/ + } + root = ksi->roots_valid[j] ? ksi->roots_hash[j] : NULL; + ksi->roots_valid[j] = 0; /* guess this is redundant with init, maybe del */ + } else if(ksi->roots_valid[j]) { + if (j >= hashchain->level ) { + if (hashchain->direction == 0x03) { /*RIGHT*/ + hashstep = rsksiHashstepFromKSI_DataHash(ksi, ksi->roots_hash[j]); + if(hashstep == NULL ) { r = RSGTE_IO; goto done; } + if(rsksi_read_debug) outputHash(stdout, "debug: verifySigblkFinishChain:\t\t RIGHT Hash: \t\t", hashstep->sib_hash.data, hashstep->sib_hash.len, ectx->verbose); + } else { /*LEFT*/ + hashstep = rsksiHashstepFromKSI_DataHash(ksi, root); + if(hashstep == NULL ) { r = RSGTE_IO; goto done; } + if(rsksi_read_debug) outputHash(stdout, "debug: verifySigblkFinishChain:\t\t LEFT Hash: \t\t", hashstep->sib_hash.data, hashstep->sib_hash.len, ectx->verbose); + } + hashstep->direction = hashchain->direction; + hashstep->level_corr = j - hashchain->level; + + /* Attach to HashChain */ + hashchain->hashsteps[hashchain->stepCount] = hashstep; + hashchain->stepCount++; + + /* Set Direction and Chainlevel */ + hashchain->direction = 0x03; /*RIGHT*/ + hashchain->level = j+1; + } + rootDel = root; + hash_node_ksi(ksi, &root, ksi->roots_hash[j], root, j+2); + ksi->roots_valid[j] = 0; /* guess this is redundant with init, maybe del */ + KSI_DataHash_free(rootDel); + } + } + + *pRoot = root; + r = 0; +done: + ksi->bInBlk = 0; + if (rsksi_read_debug && root != NULL) outputKSIHash(stdout, "debug: verifySigblkFinishChain:\t\t ROOTHash: \t", root, 0); +return r; +} + /* TODO: think about merging this with the writer. The * same applies to the other computation algos. */ -static int +int verifySigblkFinish(ksifile ksi, KSI_DataHash **pRoot) { KSI_DataHash *root, *rootDel; int8_t j; - int r; - - if(ksi->nRecords == 0) - goto done; - + int r = 0; root = NULL; + + if(ksi->nRecords == 0) { + if(rsksi_read_debug) printf("debug: verifySigblkFinish:\t\t no records!!!%d\n", r); + goto done; + } + for(j = 0 ; j < ksi->nRoots ; ++j) { if(root == NULL) { root = ksi->roots_valid[j] ? ksi->roots_hash[j] : NULL; @@ -1152,7 +1916,8 @@ verifySigblkFinish(ksifile ksi, KSI_DataHash **pRoot) r = 0; done: ksi->bInBlk = 0; - return r; + if (rsksi_read_debug && root != NULL) outputKSIHash(stdout, "debug: verifySigblkFinish:\t\t Root hash: \t", root, 1); +return r; } /* helper for rsksi_extendSig: */ @@ -1166,8 +1931,7 @@ static inline int rsksi_extendSig(KSI_Signature *sig, ksifile ksi, tlvrecord_t *rec, ksierrctx_t *ectx) { KSI_Signature *extended = NULL; - /*OLDCODE GTTimestamp *out_timestamp;*/ - uint8_t *der; + uint8_t *der = NULL; size_t lenDer; int r, rgt; tlvrecord_t newrec, subrec; @@ -1180,20 +1944,14 @@ rsksi_extendSig(KSI_Signature *sig, ksifile ksi, tlvrecord_t *rec, ksierrctx_t * r = RSGTE_SIG_EXTEND; goto done; } - /*OLD CODE - rgt = GTHTTP_extendTimestamp(timestamp, rsksi_extend_puburl, &out_timestamp); + + /* Serialize Signature. */ + rgt = KSI_Signature_serialize(extended, &der, &lenDer); if(rgt != KSI_OK) { - ectx->gtstate = rgt; - r = RSGTE_TS_EXTEND; + ectx->ksistate = rgt; + r = RSGTE_SIG_EXTEND; goto done; } - r = GTTimestamp_getDEREncoded(out_timestamp, &der, &lenDer); - if(r != KSI_OK) { - r = RSGTE_TS_DERENCODE; - ectx->gtstate = rgt; - goto done; - } - */ /* update block_sig tlv record with new extended timestamp */ /* we now need to copy all tlv records before the actual der @@ -1201,19 +1959,19 @@ rsksi_extendSig(KSI_Signature *sig, ksifile ksi, tlvrecord_t *rec, ksierrctx_t * */ iRd = iWr = 0; // TODO; check tlvtypes at comment places below! - if ((r = rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* HASH_ALGO */ COPY_SUBREC_TO_NEWREC - if ((r = rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* BLOCK_IV */ COPY_SUBREC_TO_NEWREC - if ((r = rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* LAST_HASH */ COPY_SUBREC_TO_NEWREC - if ((r = rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* REC_COUNT */ COPY_SUBREC_TO_NEWREC - if ((r = rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)) != 0) goto done; + CHKr(rsksi_tlvDecodeSUBREC(rec, &iRd, &subrec)); /* actual sig! */ newrec.data[iWr++] = 0x09 | RSKSI_FLAG_TLV16_RUNTIME; newrec.data[iWr++] = 0x06; @@ -1234,25 +1992,30 @@ rsksi_extendSig(KSI_Signature *sig, ksifile ksi, tlvrecord_t *rec, ksierrctx_t * done: if(extended != NULL) KSI_Signature_free(extended); + if (der != NULL) + KSI_free(der); return r; } /* Verify the existance of the header. */ int -verifyBLOCK_HDRKSI(FILE *sigfp, FILE *nsigfp) +verifyBLOCK_HDRKSI(ksifile ksi, FILE *sigfp, FILE *nsigfp, tlvrecord_t* tlvrec) { int r; - tlvrecord_t rec; block_hdr_t *bh = NULL; - if ((r = rsksi_tlvrd(sigfp, &rec, &bh)) != 0) goto done; - if (rec.tlvtype != 0x0901) { + if ((r = rsksi_tlvrd(sigfp, tlvrec, &bh)) != 0) goto done; + if (tlvrec->tlvtype != 0x0901) { + if(rsksi_read_debug) printf("debug: verifyBLOCK_HDRKSI:\t\t expected tlvtype 0x0901, but was %4.4x\n", tlvrec->tlvtype); r = RSGTE_MISS_BLOCKSIG; goto done; } if (nsigfp != NULL) - if ((r = rsksi_tlvwrite(nsigfp, &rec)) != 0) goto done; -done: rsksi_objfree(rec.tlvtype, bh); + if ((r = rsksi_tlvwrite(nsigfp, tlvrec)) != 0) goto done; +done: + if (bh != NULL) + rsksi_objfree(tlvrec->tlvtype, bh); + if(rsksi_read_debug) printf("debug: verifyBLOCK_HDRKSI:\t\t returned %d\n", r); return r; } @@ -1261,17 +2024,18 @@ done: rsksi_objfree(rec.tlvtype, bh); */ int verifyBLOCK_SIGKSI(block_sig_t *bs, ksifile ksi, FILE *sigfp, FILE *nsigfp, - uint8_t bExtend, ksierrctx_t *ectx) + uint8_t bExtend, KSI_DataHash *ksiHash, ksierrctx_t *ectx) { int r; int ksistate; block_sig_t *file_bs = NULL; KSI_Signature *sig = NULL; - KSI_DataHash *ksiHash = NULL; tlvrecord_t rec; - - if((r = verifySigblkFinish(ksi, &ksiHash)) != 0) - goto done; + if (ksiHash == NULL) { + if((r = verifySigblkFinish(ksi, &ksiHash)) != 0) + goto done; + } + if(rsksi_read_debug) outputKSIHash(stdout, "debug: verifyBLOCK_SIGKSI:\t\t SigBlock Finish Hash....: ", ksiHash, ectx->verbose); if((r = rsksi_tlvrdVrfyBlockSig(sigfp, &file_bs, &rec)) != 0) goto done; if(ectx->recNum != bs->recCount) { @@ -1282,57 +2046,49 @@ verifyBLOCK_SIGKSI(block_sig_t *bs, ksifile ksi, FILE *sigfp, FILE *nsigfp, /* Parse KSI Signature */ ksistate = KSI_Signature_parse(ksi->ctx->ksi_ctx, file_bs->sig.der.data, file_bs->sig.der.len, &sig); if(ksistate != KSI_OK) { - if(rsksi_read_debug) - printf("debug: KSI_Signature_parse failed with error %d\n", ksistate); + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_parse failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); r = RSGTE_INVLD_SIGNATURE; ectx->ksistate = ksistate; goto done; + } else { + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_parse was successfull\n"); } - -/* OLDCODE - gtstate = GTTimestamp_DERDecode(file_bs->sig.der.data, - file_bs->sig.der.len, ×tamp); - if(gtstate != KSI_OK) { - r = RSGTE_TS_DERDECODE; - ectx->gtstate = gtstate; + /* Verify KSI Signature */ + ksistate = KSI_Signature_verify(sig, ksi->ctx->ksi_ctx); + if(ksistate != KSI_OK) { + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_verify failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); + r = RSGTE_INVLD_SIGNATURE; + ectx->ksistate = ksistate; goto done; + } else { + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_verify was successfull\n"); } -*/ ksistate = KSI_Signature_verifyDataHash(sig, ksi->ctx->ksi_ctx, ksiHash); if (ksistate != KSI_OK) { - if(rsksi_read_debug) - printf("debug: KSI_Signature_verifyDataHash faile with error %d\n", ksistate); + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_verifyDataHash failed with error: %s (%d)\n", KSI_getErrorString(ksistate), ksistate); r = RSGTE_INVLD_SIGNATURE; ectx->ksistate = ksistate; goto done; - /* TODO proberly additional verify with KSI_Signature_verify*/ + } else { + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t KSI_Signature_verifyDataHash was successfull\n"); } -/* OLD CODE - gtstate = GTHTTP_verifyTimestampHash(timestamp, root, NULL, - NULL, NULL, rsksi_read_puburl, 0, &vrfyInf); - if(! (gtstate == KSI_OK - && vrfyInf->verification_errors == GT_NO_FAILURES) ) { - r = RSGTE_INVLD_TIMESTAMP; - ectx->gtstate = gtstate; - goto done; - } -*/ - if(rsksi_read_debug) - printf("debug: verifyBLOCK_SIGKSI processed without error's\n"); + + if(rsksi_read_debug) printf("debug: verifyBLOCK_SIGKSI:\t\t processed without error's\n"); if(rsksi_read_showVerified) - reportVerifySuccess(ectx); /*OLDCODE, vrfyInf);*/ + reportVerifySuccess(ectx); if(bExtend) if((r = rsksi_extendSig(sig, ksi, &rec, ectx)) != 0) goto done; - if(nsigfp != NULL) + if(nsigfp != NULL) { if((r = rsksi_tlvwrite(nsigfp, &rec)) != 0) goto done; + } r = 0; done: if(file_bs != NULL) rsksi_objfree(0x0904, file_bs); if(r != 0) reportError(r, ectx); - if(ksiHash != NULL) + if(ksiHash != NULL) KSI_DataHash_free(ksiHash); if(sig != NULL) KSI_Signature_free(sig); @@ -1394,6 +2150,7 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose /* Free mem*/ free(imp->data); free(imp); + imp = NULL; break; case 0x0902: /* Split Data into HEADER and BLOCK */ @@ -1410,7 +2167,7 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose } /* Check OLD encoded HASH ALGO */ - CHKr(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x00 && subrec.tlvlen == 1)) { r = RSGTE_FMT; goto donedecode; @@ -1418,7 +2175,7 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose bh->hashID = subrec.data[0]; /* Check OLD encoded BLOCK_IV */ - CHKr(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x01)) { r = RSGTE_INVLTYP; goto donedecode; @@ -1427,7 +2184,7 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose memcpy(bh->iv, subrec.data, subrec.tlvlen); /* Check OLD encoded LAST HASH */ - CHKr(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x02)) { r = RSGTE_INVLTYP; goto donedecode; } bh->lastHash.hashID = subrec.data[0]; if(subrec.tlvlen != 1 + hashOutputLengthOctetsKSI(bh->lastHash.hashID)) { @@ -1442,7 +2199,7 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose rsksi_printBLOCK_HDR(stdout, bh, verbose); /* Check OLD encoded COUNT */ - CHKr(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + CHKrDecode(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); if(!(subrec.tlvtype == 0x03 && subrec.tlvlen <= 8)) { r = RSGTE_INVLTYP; goto donedecode; } bs->recCount = 0; for(i = 0 ; i < subrec.tlvlen ; ++i) { @@ -1450,13 +2207,12 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose } /* Check OLD encoded SIG */ - CHKr(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); - if(!(subrec.tlvtype == 0x0906)) { r = RSGTE_INVLTYP; goto donedecode; } + CHKrDecode(rsksi_tlvDecodeSUBREC(&rec, &strtidx, &subrec)); + if(!(subrec.tlvtype == 0x0905)) { r = RSGTE_INVLTYP; goto donedecode; } bs->sig.der.len = subrec.tlvlen; bs->sigID = SIGID_RFC3161; if((bs->sig.der.data = (uint8_t*)malloc(bs->sig.der.len)) == NULL) {r=RSGTE_OOM;goto donedecode;} memcpy(bs->sig.der.data, subrec.data, bs->sig.der.len); - r = 0; /* Debug output */ rsksi_printBLOCK_SIG(stdout, bs, verbose); @@ -1474,48 +2230,37 @@ int rsksi_ConvertSigFile(char* name, FILE *oldsigfp, FILE *newsigfp, int verbose 2 + hashOutputLengthOctetsKSI(bh->hashID) /* iv */ + 2 + 1 + bh->lastHash.len /* last hash */; /* write top-level TLV object block-hdr */ - r = rsksi_tlv16Write(newsigfp, 0x00, 0x0901, tlvlen); + CHKrDecode(rsksi_tlv16Write(newsigfp, 0x00, 0x0901, tlvlen)); /* and now write the children */ /* hash-algo */ - r = rsksi_tlv8Write(newsigfp, 0x00, 0x01, 1); - if(r != 0) goto done; - r = rsksi_tlvfileAddOctet(newsigfp, hashIdentifierKSI(bh->hashID)); - if(r != 0) goto done; + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x01, 1)); + CHKrDecode(rsksi_tlvfileAddOctet(newsigfp, hashIdentifierKSI(bh->hashID))); /* block-iv */ - r = rsksi_tlv8Write(newsigfp, 0x00, 0x02, hashOutputLengthOctetsKSI(bh->hashID)); - if(r != 0) goto done; - r = rsksi_tlvfileAddOctetString(newsigfp, bh->iv, hashOutputLengthOctetsKSI(bh->hashID)); - if(r != 0) goto done; + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x02, hashOutputLengthOctetsKSI(bh->hashID))); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, bh->iv, hashOutputLengthOctetsKSI(bh->hashID))); /* last-hash */ - r = rsksi_tlv8Write(newsigfp, 0x00, 0x03, bh->lastHash.len + 1); - if(r != 0) goto done; - r = rsksi_tlvfileAddOctet(newsigfp, bh->lastHash.hashID); - if(r != 0) goto done; - r = rsksi_tlvfileAddOctetString(newsigfp, bh->lastHash.data, bh->lastHash.len); - if(r != 0) goto done; + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x03, bh->lastHash.len + 1)); + CHKrDecode(rsksi_tlvfileAddOctet(newsigfp, bh->lastHash.hashID)); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, bh->lastHash.data, bh->lastHash.len)); /* Create Block Signature */ tlvlenRecords = rsksi_tlvGetInt64OctetSize(bs->recCount); tlvlen = 2 + tlvlenRecords /* rec-count */ + - 4 + bs->sig.der.len /* rfc-3161 */; + 4 + bs->sig.der.len /* open-ksi */; /* write top-level TLV object (block-sig */ - r = rsksi_tlv16Write(newsigfp, 0x00, 0x0904, tlvlen); - if(r != 0) goto done; + CHKrDecode(rsksi_tlv16Write(newsigfp, 0x00, 0x0904, tlvlen)); /* and now write the children */ /* rec-count */ - r = rsksi_tlv8Write(newsigfp, 0x00, 0x01, tlvlenRecords); - if(r != 0) goto done; - r = rsksi_tlvfileAddInt64(newsigfp, bs->recCount); - if(r != 0) goto done; - /* rfc-3161 */ - r = rsksi_tlv16Write(newsigfp, 0x00, 0x906, bs->sig.der.len); - if(r != 0) goto done; - r = rsksi_tlvfileAddOctetString(newsigfp, bs->sig.der.data, bs->sig.der.len); + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x01, tlvlenRecords)); + CHKrDecode(rsksi_tlvfileAddInt64(newsigfp, bs->recCount)); + /* open-ksi */ + CHKrDecode(rsksi_tlv16Write(newsigfp, 0x00, 0x905, bs->sig.der.len)); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, bs->sig.der.data, bs->sig.der.len)); +donedecode: /* Set back to OLD default */ RSKSI_FLAG_TLV16_RUNTIME = 0x20; -donedecode: /* Free mem*/ if (bh != NULL) { free(bh->iv); @@ -1531,7 +2276,7 @@ donedecode: if(r != 0) goto done; break; default: - fprintf(stdout, "unknown tlv record %4.4x\n", rec.tlvtype); + printf("debug: rsksi_ConvertSigFile:\t unknown tlv record %4.4x\n", rec.tlvtype); break; } } else { @@ -1542,7 +2287,7 @@ donedecode: if(r == RSGTE_EOF) r = 0; /* Successfully finished file */ else if(rsksi_read_debug) - printf("debug: rsksi_ConvertSigFile failed to read with error %d\n", r); + printf("debug: rsksi_ConvertSigFile:\t failed to read with error %d\n", r); goto done; } @@ -1552,6 +2297,92 @@ donedecode: } done: if(rsksi_read_debug) - printf("debug: rsksi_ConvertSigFile returned %d\n", r); + printf("debug: rsksi_ConvertSigFile:\t returned %d\n", r); + return r; +} + + + +/* Helper function to write full hashchain! */ +int rsksi_WriteHashChain(FILE *newsigfp, block_hashchain_t *hashchain, block_sig_t *bsIn, int verbose) +{ + uint8_t j; + int r = 0; + unsigned tlvlen; + uint8_t tlvlenLevelCorr; + uint8_t uiLevelCorr = 0; + + if(rsksi_read_debug) printf("debug: rsksi_WriteHashChain:\t\t NEW HashChain started with %lld Chains\n", (long long unsigned)hashchain->stepCount); + + /* Error Check */ + if (hashchain == NULL || + hashchain->rec_hash.data == NULL || + hashchain->stepCount == 0) { + r = RSGTE_EXTRACT_HASH; + goto done; + } + + /* Total Length of Hash Chain */ + tlvlenLevelCorr = rsksi_tlvGetInt64OctetSize(uiLevelCorr); + + /* Total Length of Hash Chain */ + tlvlen = 4 + /* ???? */ + 2 + 1 + hashchain->rec_hash.len /* rec-hash */ + + ((2 + tlvlenLevelCorr + 2 + 1 + hashchain->hashsteps[0]->sib_hash.len)*hashchain->stepCount); /* Count of all left/right chains */ + if(rsksi_read_debug) printf("debug: rsksi_WriteHashChain:\t\t tlvlen=%d \n", tlvlen); + + /* Start hash chain for one log record */ + CHKrDecode(rsksi_tlv16Write(newsigfp, 0x00, 0x0907, tlvlen)); + + /* rec-hash */ + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x01, 1 + hashchain->rec_hash.len)); + CHKrDecode(rsksi_tlvfileAddOctet(newsigfp, hashchain->rec_hash.hashID)); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, hashchain->rec_hash.data, hashchain->rec_hash.len)); + if(rsksi_read_debug) { + printf("debug: rsksi_WriteHashChain:\t\t Write Record tlvlen=%zu \n", 1 + hashchain->rec_hash.len); + outputHash(stdout, "debug: rsksi_WriteHashChain:\t\t RECORD Hash: \t\t", hashchain->rec_hash.data, hashchain->rec_hash.len, verbose); + } + + /* Process Chains */ + for(j = 0 ; j < hashchain->stepCount ; ++j) { + tlvlen = 2 + tlvlenLevelCorr + 2 + 1 + hashchain->hashsteps[j]->sib_hash.len; + + /* Step in the hash chain*/ + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, hashchain->hashsteps[j]->direction, tlvlen)); + /* level correction value */ + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x01, tlvlenLevelCorr)); + CHKrDecode(rsksi_tlvfileAddInt64(newsigfp, hashchain->hashsteps[j]->level_corr)); + /* sibling hash value */ + CHKrDecode(rsksi_tlv8Write(newsigfp, 0x00, 0x02, 1 + hashchain->hashsteps[j]->sib_hash.len)); + CHKrDecode(rsksi_tlvfileAddOctet(newsigfp, hashchain->hashsteps[j]->sib_hash.hashID)); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, hashchain->hashsteps[j]->sib_hash.data, hashchain->hashsteps[j]->sib_hash.len)); + + if(rsksi_read_debug) { + printf("debug: rsksi_WriteHashChain:\t\t Write Chain tlvlen=%d \n", tlvlen); + printf("debug: rsksi_WriteHashChain:\t\t Direction %d, level_corr %lld\n", hashchain->hashsteps[j]->direction, (long long unsigned) uiLevelCorr); + outputHash(stdout, "debug: rsksi_WriteHashChain:\t\t Chain Hash: \t\t", hashchain->hashsteps[j]->sib_hash.data, hashchain->hashsteps[j]->sib_hash.len, verbose); + } + } +donedecode: + if(r != 0) printf("debug: rsksi_WriteHashChain:\t\t failed to write with error %d\n", r); +done: + if(rsksi_read_debug) printf("debug: rsksi_WriteHashChain:\t\t returned %d\n", r); + return r; +} + +/* Helper function to Extract Block Signature */ +int rsksi_ExtractBlockSignature(FILE *newsigfp, ksifile ksi, block_sig_t *bsIn, ksierrctx_t *ectx, int verbose) +{ + int r = 0; + + /* WRITE BLOCK Signature */ + /* open-ksi */ + CHKrDecode(rsksi_tlv16Write(newsigfp, 0x00, 0x905, bsIn->sig.der.len)); + CHKrDecode(rsksi_tlvfileAddOctetString(newsigfp, bsIn->sig.der.data, bsIn->sig.der.len)); + +donedecode: + if(r != 0) printf("debug: rsksi_ExtractBlockSignature:\t\t failed to write with error %d\n", r); +done: + if(rsksi_read_debug) printf("debug: ExtractBlockSignature:\t\t returned %d\n", r); return r; } diff --git a/runtime/lmcry_gcry.c b/runtime/lmcry_gcry.c index 9a0c0072e..3459818ab 100644 --- a/runtime/lmcry_gcry.c +++ b/runtime/lmcry_gcry.c @@ -102,7 +102,7 @@ SetCnfParam(void *pT, struct nvlst *lst, int paramType) { lmcry_gcry_t *pThis = (lmcry_gcry_t*) pT; int i, r; - unsigned keylen; + unsigned keylen = 0; uchar *key = NULL; uchar *keyfile = NULL; uchar *keyprogram = NULL; @@ -198,14 +198,23 @@ SetCnfParam(void *pT, struct nvlst *lst, int paramType) } cnfparamvalsDestruct(pvals, pblk); - if(key != NULL) { - memset(key, 0, strlen((char*)key)); - free(key); - } - free(keyfile); - free(algo); - free(mode); + finalize_it: + if (key != NULL) + free(key); + + if (keyfile != NULL) + free(keyfile); + + if (algo != NULL) + free(algo); + + if (keyprogram != NULL) + free(keyprogram); + + if (mode != NULL) + free(mode); + RETiRet; } diff --git a/runtime/lookup.c b/runtime/lookup.c index 0413f4b3c..2a8d86432 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -1,18 +1,18 @@ /* lookup.c * Support for lookup tables in RainerScript. * - * Copyright 2013 Adiscon GmbH. + * Copyright 2013-2016 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,6 +38,8 @@ #include "dirty.h" #include "unicode-helper.h" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + /* definitions for objects we access */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) diff --git a/runtime/module-template.h b/runtime/module-template.h index febcfadbe..71b843e73 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -276,8 +276,9 @@ static rsRetVal endTransaction(wrkrInstanceData_t __attribute__((unused)) *pWrkr /* doAction() */ #define BEGINdoAction \ -static rsRetVal doAction(uchar __attribute__((unused)) **ppString, wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ +static rsRetVal doAction(void * pMsgData, wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ {\ + uchar **ppString = (uchar **) pMsgData; \ DEFiRet; #define CODESTARTdoAction \ @@ -287,6 +288,14 @@ static rsRetVal doAction(uchar __attribute__((unused)) **ppString, wrkrInstanceD RETiRet;\ } +/* below is a variant of doAction where the passed-in data is not the common + * case of string. + */ +#define BEGINdoAction_NoStrings \ +static rsRetVal doAction(void * pMsgData, wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ +{\ + DEFiRet; + /* dbgPrintInstInfo() * Extra comments: diff --git a/runtime/modules.c b/runtime/modules.c index ca6db2801..21d4959b0 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -11,7 +11,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -247,7 +247,8 @@ static void moduleDestruct(modInfo_t *pThis) free(pThis->cnfName); if(pThis->pModHdlr != NULL) { # ifdef VALGRIND -# warning "dlclose disabled for valgrind" + DBGPRINTF("moduleDestruct: compiled with valgrind, do " + "not unload module\n"); # else if (pThis->eKeepType == eMOD_NOKEEP) { dlclose(pThis->pModHdlr); @@ -414,33 +415,43 @@ finalize_it: * module list. Needed to prevent mem leaks. */ static inline void -abortCnfUse(cfgmodules_etry_t *pNew) +abortCnfUse(cfgmodules_etry_t **pNew) { - free(pNew); + if(pNew != NULL) { + free(*pNew); + *pNew = NULL; + } } /* Add a module to the config module list for current loadConf. * Requires last pointer obtained by readyModForCnf(). + * The module pointer is handed over to this function. It is no + * longer available to caller one we are called. */ rsRetVal -addModToCnfList(cfgmodules_etry_t *pNew, cfgmodules_etry_t *pLast) +addModToCnfList(cfgmodules_etry_t **pNew, cfgmodules_etry_t *pLast) { DEFiRet; - assert(pNew != NULL); + assert(*pNew != NULL); + if(pNew == NULL) + ABORT_FINALIZE(RS_RET_ERR); if(loadConf == NULL) { + abortCnfUse(pNew); FINALIZE; /* we are in an early init state */ } if(pLast == NULL) { - loadConf->modules.root = pNew; + loadConf->modules.root = *pNew; } else { /* there already exist entries */ - pLast->next = pNew; + pLast->next = *pNew; } finalize_it: + if(pNew != NULL) + *pNew = NULL; RETiRet; } @@ -1053,8 +1064,8 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) int bHasExtension; void *pModHdlr, *pModInit; modInfo_t *pModInfo; - cfgmodules_etry_t *pNew; - cfgmodules_etry_t *pLast; + cfgmodules_etry_t *pNew = NULL; + cfgmodules_etry_t *pLast = NULL; uchar *pModDirCurr, *pModDirNext; int iLoadCnt; struct dlhandle_s *pHandle = NULL; @@ -1110,7 +1121,7 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) /* regular modules need to be added to conf list (for * builtins, this happend during initial load). */ - addModToCnfList(pNew, pLast); + addModToCnfList(&pNew, pLast); } } } @@ -1129,7 +1140,7 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) free(pPathBuf); /* we always alloc enough memory for everything we potentiall need to add */ lenPathBuf = PATHBUF_OVERHEAD; - CHKmalloc(pPathBuf = malloc(sizeof(uchar)*lenPathBuf)); + CHKmalloc(pPathBuf = malloc(lenPathBuf)); } *pPathBuf = '\0'; /* we do not need to append the path - its already in the module name */ iPathLen = 0; @@ -1152,7 +1163,7 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) free(pPathBuf); /* we always alloc enough memory for everything we potentiall need to add */ lenPathBuf = iPathLen + PATHBUF_OVERHEAD; - CHKmalloc(pPathBuf = malloc(sizeof(uchar)*lenPathBuf)); + CHKmalloc(pPathBuf = malloc(lenPathBuf)); } memcpy((char *) pPathBuf, (char *)pModDirCurr, iPathLen); @@ -1230,18 +1241,19 @@ Load(uchar *pModName, sbool bConfLoad, struct nvlst *lst) errmsg.LogError(0, localRet, "module '%s', failed processing config parameters", pPathBuf); - abortCnfUse(pNew); ABORT_FINALIZE(localRet); } } pModInfo->bSetModCnfCalled = 1; } - addModToCnfList(pNew, pLast); + addModToCnfList(&pNew, pLast); } finalize_it: if(pPathBuf != pathBuf) /* used malloc()ed memory? */ free(pPathBuf); + if(iRet != RS_RET_OK) + abortCnfUse(&pNew); pthread_mutex_unlock(&mutObjGlobalOp); RETiRet; } diff --git a/runtime/modules.h b/runtime/modules.h index 39946b7e1..611a67217 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -200,6 +200,6 @@ PROTOTYPEObj(module); */ rsRetVal modulesProcessCnf(struct cnfobj *o); uchar *modGetName(modInfo_t *pThis); -rsRetVal addModToCnfList(cfgmodules_etry_t *pNew, cfgmodules_etry_t *pLast); +rsRetVal addModToCnfList(cfgmodules_etry_t **pNew, cfgmodules_etry_t *pLast); rsRetVal readyModForCnf(modInfo_t *pThis, cfgmodules_etry_t **ppNew, cfgmodules_etry_t **ppLast); #endif /* #ifndef MODULES_H_INCLUDED */ diff --git a/runtime/msg.c b/runtime/msg.c index 6f03cb5d7..3370e4f99 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -7,7 +7,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -71,7 +71,7 @@ /* TODO: move the global variable root to the config object - had no time to to it * right now before vacation -- rgerhards, 2013-07-22 */ -static pthread_rwlock_t glblVars_rwlock; +static pthread_mutex_t glblVars_lock; struct json_object *global_var_root = NULL; /* static data */ @@ -405,6 +405,7 @@ static rsRetVal jsonPathFindParent(struct json_object *jroot, uchar *name, uchar static uchar * jsonPathGetLeaf(uchar *name, int lenName); static struct json_object *jsonDeepCopy(struct json_object *src); static json_bool jsonVarExtract(struct json_object* root, const char *key, struct json_object **value); +void getRawMsgAfterPRI(msg_t * const pM, uchar **pBuf, int *piLen); /* the locking and unlocking implementations: */ @@ -549,6 +550,8 @@ propNameToID(uchar *pName, propid_t *pPropID) *pPropID = PROP_SYSLOGTAG; } else if(!strcmp((char*) pName, "rawmsg")) { *pPropID = PROP_RAWMSG; + } else if(!strcmp((char*) pName, "rawmsg-after-pri")) { + *pPropID = PROP_RAWMSG_AFTER_PRI; } else if(!strcmp((char*) pName, "inputname")) { *pPropID = PROP_INPUTNAME; } else if(!strcmp((char*) pName, "fromhost")) { @@ -608,6 +611,22 @@ propNameToID(uchar *pName, propid_t *pPropID) *pPropID = PROP_SYS_QHOUR; } else if(!strcmp((char*) pName, "$minute")) { *pPropID = PROP_SYS_MINUTE; + } else if(!strcmp((char*) pName, "$now-utc")) { + *pPropID = PROP_SYS_NOW; + } else if(!strcmp((char*) pName, "$year-utc")) { + *pPropID = PROP_SYS_YEAR_UTC; + } else if(!strcmp((char*) pName, "$month-utc")) { + *pPropID = PROP_SYS_MONTH_UTC; + } else if(!strcmp((char*) pName, "$day-utc")) { + *pPropID = PROP_SYS_DAY_UTC; + } else if(!strcmp((char*) pName, "$hour-utc")) { + *pPropID = PROP_SYS_HOUR_UTC; + } else if(!strcmp((char*) pName, "$hhour-utc")) { + *pPropID = PROP_SYS_HHOUR_UTC; + } else if(!strcmp((char*) pName, "$qhour-utc")) { + *pPropID = PROP_SYS_QHOUR_UTC; + } else if(!strcmp((char*) pName, "$minute-utc")) { + *pPropID = PROP_SYS_MINUTE_UTC; } else if(!strcmp((char*) pName, "$myhostname")) { *pPropID = PROP_SYS_MYHOSTNAME; } else if(!strcmp((char*) pName, "$!all-json")) { @@ -702,6 +721,22 @@ uchar *propIDToName(propid_t propID) return UCHAR_CONSTANT("$QHOUR"); case PROP_SYS_MINUTE: return UCHAR_CONSTANT("$MINUTE"); + case PROP_SYS_NOW_UTC: + return UCHAR_CONSTANT("$NOW_UTC"); + case PROP_SYS_YEAR_UTC: + return UCHAR_CONSTANT("$YEAR_UTC"); + case PROP_SYS_MONTH_UTC: + return UCHAR_CONSTANT("$MONTH_UTC"); + case PROP_SYS_DAY_UTC: + return UCHAR_CONSTANT("$DAY_UTC"); + case PROP_SYS_HOUR_UTC: + return UCHAR_CONSTANT("$HOUR_UTC"); + case PROP_SYS_HHOUR_UTC: + return UCHAR_CONSTANT("$HHOUR_UTC"); + case PROP_SYS_QHOUR_UTC: + return UCHAR_CONSTANT("$QHOUR_UTC"); + case PROP_SYS_MINUTE_UTC: + return UCHAR_CONSTANT("$MINUTE_UTC"); case PROP_SYS_MYHOSTNAME: return UCHAR_CONSTANT("$MYHOSTNAME"); case PROP_CEE: @@ -847,7 +882,7 @@ rsRetVal msgConstruct(msg_t **ppThis) * especially as I think there is no codepath currently where it would not be * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02 */ - datetime.getCurrTime(&((*ppThis)->tRcvdAt), &((*ppThis)->ttGenTime)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt), &((*ppThis)->ttGenTime), TIME_IN_LOCALTIME); memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime)); finalize_it: @@ -1566,6 +1601,38 @@ getRawMsg(msg_t * const pM, uchar **pBuf, int *piLen) } } +void +getRawMsgAfterPRI(msg_t * const pM, uchar **pBuf, int *piLen) +{ + if(pM == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + if(pM->pszRawMsg == NULL) { + *pBuf= UCHAR_CONSTANT(""); + *piLen = 0; + } else { + /* unfortunately, pM->offAfterPRI seems NOT to be + * correct/consistent in all cases. imuxsock and imudp + * seem to have other values than imptcp. Testbench + * covers some of that. As a work-around, we caluculate + * the value ourselfes here. -- rgerhards, 2015-10-09 + */ + size_t offAfterPRI = 0; + if(pM->pszRawMsg[0] == '<') { /* do we have a PRI? */ + if(pM->pszRawMsg[2] == '>') + offAfterPRI = 3; + else if(pM->pszRawMsg[3] == '>') + offAfterPRI = 4; + else if(pM->pszRawMsg[4] == '>') + offAfterPRI = 5; + } + *pBuf = pM->pszRawMsg + offAfterPRI; + *piLen = pM->iLenRawMsg - offAfterPRI; + } + } +} + /* note: setMSGLen() is only for friends who really know what they * do. Setting an invalid length can be desasterous! @@ -2685,15 +2752,19 @@ void MsgSetRawMsgWOSize(msg_t * const pMsg, char* pszRawMsg) * The variable pRes must point to a user-supplied buffer of * at least 20 characters. */ -static void -textpri(const msg_t *const __restrict__ pMsg, uchar *const __restrict__ pRes) +static uchar * +textpri(const msg_t *const __restrict__ pMsg) { - assert(pRes != NULL); - memcpy(pRes, syslog_fac_names[pMsg->iFacility], len_syslog_fac_names[pMsg->iFacility]); - pRes[len_syslog_fac_names[pMsg->iFacility]] = '.'; - memcpy(pRes+len_syslog_fac_names[pMsg->iFacility]+1, - syslog_severity_names[pMsg->iSeverity], - len_syslog_severity_names[pMsg->iSeverity]+1 /* for \0! */); + int lenfac = len_syslog_fac_names[pMsg->iFacility]; + int lensev = len_syslog_severity_names[pMsg->iSeverity]; + int totlen = lenfac + 1 + lensev + 1; + char *pRes = MALLOC(totlen); + if(pRes != NULL) { + memcpy(pRes, syslog_fac_names[pMsg->iFacility], lenfac); + pRes[lenfac] = '.'; + memcpy(pRes+lenfac+1, syslog_severity_names[pMsg->iSeverity], lensev+1 /* for \0! */); + } + return (uchar*)pRes; } @@ -2706,22 +2777,22 @@ textpri(const msg_t *const __restrict__ pMsg, uchar *const __restrict__ pRes) */ typedef enum ENOWType { NOW_NOW, NOW_YEAR, NOW_MONTH, NOW_DAY, NOW_HOUR, NOW_HHOUR, NOW_QHOUR, NOW_MINUTE } eNOWType; #define tmpBUFSIZE 16 /* size of formatting buffer */ -static uchar *getNOW(eNOWType eNow, struct syslogTime *t) +static uchar *getNOW(eNOWType eNow, struct syslogTime *t, const int inUTC) { uchar *pBuf; struct syslogTime tt; - if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { + if((pBuf = (uchar*) MALLOC(tmpBUFSIZE)) == NULL) { return NULL; } if(t == NULL) { /* can happen if called via script engine */ - datetime.getCurrTime(&tt, NULL); + datetime.getCurrTime(&tt, NULL, inUTC); t = &tt; } - if(t->year == 0) { /* not yet set! */ - datetime.getCurrTime(t, NULL); + if(t->year == 0 || t->inUTC != inUTC) { /* not yet set! */ + datetime.getCurrTime(t, NULL, inUTC); } switch(eNow) { @@ -2778,17 +2849,20 @@ getJSONPropVal(msg_t * const pMsg, msgPropDescr_t *pProp, uchar **pRes, rs_size_ if(pProp->id == PROP_CEE) { jroot = pMsg->json; + MsgLock(pMsg); } else if(pProp->id == PROP_LOCAL_VAR) { jroot = pMsg->localvars; + MsgLock(pMsg); } else if(pProp->id == PROP_GLOBAL_VAR) { - pthread_rwlock_rdlock(&glblVars_rwlock); jroot = global_var_root; + pthread_mutex_lock(&glblVars_lock); } else { DBGPRINTF("msgGetJSONPropVal; invalid property id %d\n", pProp->id); ABORT_FINALIZE(RS_RET_NOT_FOUND); } - if(jroot == NULL) goto finalize_it; + + if(jroot == NULL) FINALIZE; if(!strcmp((char*)pProp->name, "!")) { field = jroot; @@ -2806,7 +2880,9 @@ getJSONPropVal(msg_t * const pMsg, msgPropDescr_t *pProp, uchar **pRes, rs_size_ finalize_it: if(pProp->id == PROP_GLOBAL_VAR) - pthread_rwlock_unlock(&glblVars_rwlock); + pthread_mutex_unlock(&glblVars_lock); + else + MsgUnlock(pMsg); if(*pRes == NULL) { /* could not find any value, so set it to empty */ *pRes = (unsigned char*)""; @@ -2816,6 +2892,75 @@ finalize_it: } +/* Get a JSON-based-variable as native json object, except + * when it is string type, in which case a string is returned. + * This is an optimization to not use JSON when not strictly + * necessary. This in turn is helpful, as calling json-c is + * *very* expensive due to our need for locking and deep + * copies. + * The caller needs to check pjson and pcstr: one of them + * is non-NULL and contains the return value. Note that + * the caller is responsible for freeing the string pointer + * it if is being returned. + */ +rsRetVal +msgGetJSONPropJSONorString(msg_t * const pMsg, msgPropDescr_t *pProp, struct json_object **pjson, + uchar **pcstr) +{ + struct json_object *jroot; + uchar *leaf; + struct json_object *parent; + DEFiRet; + + *pjson = NULL, *pcstr = NULL; + + if(pProp->id == PROP_CEE) { + jroot = pMsg->json; + MsgLock(pMsg); + } else if(pProp->id == PROP_LOCAL_VAR) { + jroot = pMsg->localvars; + MsgLock(pMsg); + } else if(pProp->id == PROP_GLOBAL_VAR) { + jroot = global_var_root; + pthread_mutex_lock(&glblVars_lock); + } else { + DBGPRINTF("msgGetJSONPropJSON; invalid property id %d\n", + pProp->id); + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + + if(!strcmp((char*)pProp->name, "!")) { + *pjson = jroot; + FINALIZE; + } + leaf = jsonPathGetLeaf(pProp->name, pProp->nameLen); + CHKiRet(jsonPathFindParent(jroot, pProp->name, leaf, &parent, 1)); + if(jsonVarExtract(parent, (char*)leaf, pjson) == FALSE) { + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + if(*pjson == NULL) { + /* we had a NULL json object and represent this as empty string */ + *pcstr = (uchar*) strdup(""); + } else { + if(json_object_get_type(*pjson) == json_type_string) { + *pcstr = (uchar*) strdup(json_object_get_string(*pjson)); + *pjson = NULL; + } + } + +finalize_it: + /* we need a deep copy, as another thread may modify the object */ + if(*pjson != NULL) + *pjson = jsonDeepCopy(*pjson); + if(pProp->id == PROP_GLOBAL_VAR) + pthread_mutex_unlock(&glblVars_lock); + else + MsgUnlock(pMsg); + RETiRet; +} + + + /* Get a JSON-based-variable as native json object */ rsRetVal msgGetJSONPropJSON(msg_t * const pMsg, msgPropDescr_t *pProp, struct json_object **pjson) @@ -2829,21 +2974,18 @@ msgGetJSONPropJSON(msg_t * const pMsg, msgPropDescr_t *pProp, struct json_object if(pProp->id == PROP_CEE) { jroot = pMsg->json; + MsgLock(pMsg); } else if(pProp->id == PROP_LOCAL_VAR) { jroot = pMsg->localvars; + MsgLock(pMsg); } else if(pProp->id == PROP_GLOBAL_VAR) { - pthread_rwlock_rdlock(&glblVars_rwlock); jroot = global_var_root; + pthread_mutex_lock(&glblVars_lock); } else { DBGPRINTF("msgGetJSONPropJSON; invalid property id %d\n", pProp->id); ABORT_FINALIZE(RS_RET_NOT_FOUND); } - if(jroot == NULL) { - DBGPRINTF("msgGetJSONPropJSON; jroot empty for property %s\n", - pProp->name); - ABORT_FINALIZE(RS_RET_NOT_FOUND); - } if(!strcmp((char*)pProp->name, "!")) { *pjson = jroot; @@ -2856,14 +2998,13 @@ msgGetJSONPropJSON(msg_t * const pMsg, msgPropDescr_t *pProp, struct json_object } finalize_it: - if(pProp->id == PROP_GLOBAL_VAR) { - if (*pjson != NULL) - *pjson = jsonDeepCopy(*pjson); - pthread_rwlock_unlock(&glblVars_rwlock); - } else { - if (*pjson != NULL) - json_object_get(*pjson); - } + /* we need a deep copy, as another thread may modify the object */ + if(*pjson != NULL) + *pjson = jsonDeepCopy(*pjson); + if(pProp->id == PROP_GLOBAL_VAR) + pthread_mutex_unlock(&glblVars_lock); + else + MsgUnlock(pMsg); RETiRet; } @@ -3134,6 +3275,9 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri case PROP_RAWMSG: getRawMsg(pMsg, &pRes, &bufLen); break; + case PROP_RAWMSG_AFTER_PRI: + getRawMsgAfterPRI(pMsg, &pRes, &bufLen); + break; case PROP_INPUTNAME: getInputName(pMsg, &pRes, &bufLen); break; @@ -3147,10 +3291,10 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri pRes = (uchar*)getPRI(pMsg); break; case PROP_PRI_TEXT: - if((pRes = MALLOC(20 * sizeof(uchar))) == NULL) + pRes = textpri(pMsg); + if(pRes == NULL) RET_OUT_OF_MEMORY; *pbMustBeFreed = 1; - textpri(pMsg, pRes); break; case PROP_IUT: pRes = UCHAR_CONSTANT("1"); /* always 1 for syslog messages (a MonitorWare thing;)) */ @@ -3206,7 +3350,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri pRes = (uchar*)getParseSuccess(pMsg); break; case PROP_SYS_NOW: - if((pRes = getNOW(NOW_NOW, ttNow)) == NULL) { + if((pRes = getNOW(NOW_NOW, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3214,7 +3358,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_YEAR: - if((pRes = getNOW(NOW_YEAR, ttNow)) == NULL) { + if((pRes = getNOW(NOW_YEAR, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3222,7 +3366,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_MONTH: - if((pRes = getNOW(NOW_MONTH, ttNow)) == NULL) { + if((pRes = getNOW(NOW_MONTH, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3230,7 +3374,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_DAY: - if((pRes = getNOW(NOW_DAY, ttNow)) == NULL) { + if((pRes = getNOW(NOW_DAY, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3238,7 +3382,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_HOUR: - if((pRes = getNOW(NOW_HOUR, ttNow)) == NULL) { + if((pRes = getNOW(NOW_HOUR, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3246,7 +3390,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_HHOUR: - if((pRes = getNOW(NOW_HHOUR, ttNow)) == NULL) { + if((pRes = getNOW(NOW_HHOUR, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3254,7 +3398,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_QHOUR: - if((pRes = getNOW(NOW_QHOUR, ttNow)) == NULL) { + if((pRes = getNOW(NOW_QHOUR, ttNow, TIME_IN_LOCALTIME)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3262,7 +3406,71 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri } break; case PROP_SYS_MINUTE: - if((pRes = getNOW(NOW_MINUTE, ttNow)) == NULL) { + if((pRes = getNOW(NOW_MINUTE, ttNow, TIME_IN_LOCALTIME)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_NOW_UTC: + if((pRes = getNOW(NOW_NOW, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 10; + } + break; + case PROP_SYS_YEAR_UTC: + if((pRes = getNOW(NOW_YEAR, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 4; + } + break; + case PROP_SYS_MONTH_UTC: + if((pRes = getNOW(NOW_MONTH, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_DAY_UTC: + if((pRes = getNOW(NOW_DAY, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_HOUR_UTC: + if((pRes = getNOW(NOW_HOUR, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_HHOUR_UTC: + if((pRes = getNOW(NOW_HHOUR, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_QHOUR_UTC: + if((pRes = getNOW(NOW_QHOUR, ttNow, TIME_IN_UTC)) == NULL) { + RET_OUT_OF_MEMORY; + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } + break; + case PROP_SYS_MINUTE_UTC: + if((pRes = getNOW(NOW_MINUTE, ttNow, TIME_IN_UTC)) == NULL) { RET_OUT_OF_MEMORY; } else { *pbMustBeFreed = 1; @@ -3279,13 +3487,27 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri bufLen = 2; *pbMustBeFreed = 0; } else { + const char *jstr; MsgLock(pMsg); +#ifdef HAVE_JSON_OBJECT_TO_JSON_STRING_EXT + int jflag = 0; if(pProp->id == PROP_CEE_ALL_JSON) { - pRes = (uchar*)strdup(RS_json_object_to_json_string_ext(pMsg->json, JSON_C_TO_STRING_SPACED)); + jflag = JSON_C_TO_STRING_SPACED; } else if(pProp->id == PROP_CEE_ALL_JSON_PLAIN) { - pRes = (uchar*)strdup(RS_json_object_to_json_string_ext(pMsg->json, JSON_C_TO_STRING_PLAIN)); + jflag = JSON_C_TO_STRING_PLAIN; } + jstr = json_object_to_json_string_ext(pMsg->json, jflag); +#else + jstr = json_object_to_json_string(pMsg->json); +#endif MsgUnlock(pMsg); + if(jstr == NULL) { + RET_OUT_OF_MEMORY; + } + pRes = (uchar*)strdup(jstr); + if(pRes == NULL) { + RET_OUT_OF_MEMORY; + } *pbMustBeFreed = 1; } break; @@ -3311,7 +3533,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri { struct timespec tp; - if((pRes = (uchar*) MALLOC(sizeof(uchar) * 32)) == NULL) { + if((pRes = (uchar*) MALLOC(32)) == NULL) { RET_OUT_OF_MEMORY; } @@ -3323,7 +3545,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri *pbMustBeFreed = 1; - snprintf((char*) pRes, sizeof(uchar) * 32, "%ld", tp.tv_sec); + snprintf((char*) pRes, 32, "%ld", tp.tv_sec); } # else @@ -3331,7 +3553,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri { struct sysinfo s_info; - if((pRes = (uchar*) MALLOC(sizeof(uchar) * 32)) == NULL) { + if((pRes = (uchar*) MALLOC(32)) == NULL) { RET_OUT_OF_MEMORY; } @@ -3343,7 +3565,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri *pbMustBeFreed = 1; - snprintf((char*) pRes, sizeof(uchar) * 32, "%ld", s_info.uptime); + snprintf((char*) pRes, 32, "%ld", s_info.uptime); } # endif break; @@ -3410,7 +3632,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri /* we got our end pointer, now do the copy */ /* TODO: code copied from below, this is a candidate for a separate function */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(uchar)); + pBufStart = pBuf = MALLOC(iLen + 1); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3522,7 +3744,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo - pmatch[pTpe->data.field.iSubMatchToUse].rm_so; - pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); + pB = MALLOC(iLenBuf + 1); if (pB == NULL) { if (*pbMustBeFreed == 1) @@ -3583,12 +3805,12 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri */ ; /*DO NOTHING*/ } else { - if(iTo > bufLen) /* iTo is very large, if no to-position is set in the template! */ + if(iTo >= bufLen) /* iTo is very large, if no to-position is set in the template! */ if (pTpe->data.field.options.bFixedWidth == 0) - iTo = bufLen; + iTo = bufLen - 1; iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(uchar)); + pBufStart = pBuf = MALLOC(iLen + 1); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3646,7 +3868,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri uchar *pBStart; uchar *pB; uchar *pSrc; - pBStart = pB = MALLOC((bufLen + 1) * sizeof(uchar)); + pBStart = pB = MALLOC(bufLen + 1); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3767,7 +3989,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri int i; iLenBuf += iNumCC * 4; - pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); + pBStart = pB = MALLOC(iLenBuf + 1); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3902,7 +4124,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri /* check if we need to obtain a private copy */ if(*pbMustBeFreed == 0) { /* ok, original copy, need a private one */ - pB = MALLOC((iLn + 1) * sizeof(uchar)); + pB = MALLOC(iLn + 1); if(pB == NULL) { RET_OUT_OF_MEMORY; } @@ -3930,7 +4152,7 @@ uchar *MsgGetProp(msg_t *__restrict__ const pMsg, struct templateEntry *__restri bufLen = ustrlen(pRes); iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ - pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar)); + pBStart = pDst = MALLOC(2 * iBufLen + 3); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -3978,9 +4200,6 @@ msgSetPropViaJSON(msg_t *__restrict__ const pMsg, const char *name, struct json_ prop_t *propRcvFromIP = NULL; DEFiRet; - // TODO: think if we need to lock the message mutex. For some updates - // we probably need to! - /* note: json_object_get_string() manages the memory of the returned * string. So we MUST NOT free it! */ @@ -4201,6 +4420,9 @@ jsonPathFindParent(struct json_object *jroot, uchar *name, uchar *leaf, struct j while(name < leaf-1) { jsonPathFindNext(*parent, namestart, &name, leaf, parent, bCreate); } + if(*parent == NULL) + ABORT_FINALIZE(RS_RET_NOT_FOUND); +finalize_it: RETiRet; } @@ -4261,19 +4483,20 @@ msgAddJSON(msg_t * const pM, uchar *name, struct json_object *json, int force_re uchar *leaf; DEFiRet; - MsgLock(pM); if(name[0] == '!') { pjroot = &pM->json; + MsgLock(pM); } else if(name[0] == '.') { pjroot = &pM->localvars; + MsgLock(pM); } else if (name[0] == '/') { /* globl var */ - pthread_rwlock_wrlock(&glblVars_rwlock); pjroot = &global_var_root; if (sharedReference) { given = json; json = jsonDeepCopy(json); json_object_put(given); } + pthread_mutex_lock(&glblVars_lock); } else { DBGPRINTF("Passed name %s is unknown kind of variable (It is not CEE, Local or Global variable).", name); ABORT_FINALIZE(RS_RET_INVLD_SETOP); @@ -4329,8 +4552,9 @@ msgAddJSON(msg_t * const pM, uchar *name, struct json_object *json, int force_re finalize_it: if(name[0] == '/') - pthread_rwlock_unlock(&glblVars_rwlock); - MsgUnlock(pM); + pthread_mutex_unlock(&glblVars_lock); + else + MsgUnlock(pM); RETiRet; } @@ -4343,20 +4567,22 @@ msgDelJSON(msg_t * const pM, uchar *name) uchar *leaf; DEFiRet; - MsgLock(pM); - if(name[0] == '!') { jroot = &pM->json; + MsgLock(pM); } else if(name[0] == '.') { jroot = &pM->localvars; + MsgLock(pM); } else if (name[0] == '/') { /* globl var */ - pthread_rwlock_wrlock(&glblVars_rwlock); jroot = &global_var_root; + pthread_mutex_lock(&glblVars_lock); } else { - DBGPRINTF("Passed name %s is unknown kind of variable (It is not CEE, Local or Global variable).", name); + DBGPRINTF("Passed name %s is unknown kind of variable (It is not CEE, " + "Local or Global variable).", name); ABORT_FINALIZE(RS_RET_INVLD_SETOP); } - if(jroot == NULL) { + + if(*jroot == NULL) { DBGPRINTF("msgDelJSONVar; jroot empty in unset for property %s\n", name); FINALIZE; @@ -4370,10 +4596,6 @@ msgDelJSON(msg_t * const pM, uchar *name) json_object_put(*jroot); *jroot = NULL; } else { - if(*jroot == NULL) { - /* now we need a root obj */ - *jroot = json_object_new_object(); - } leaf = jsonPathGetLeaf(name, ustrlen(name)); CHKiRet(jsonPathFindParent(*jroot, name, leaf, &parent, 1)); if(jsonVarExtract(parent, (char*)leaf, &leafnode) == FALSE) @@ -4391,8 +4613,9 @@ msgDelJSON(msg_t * const pM, uchar *name) finalize_it: if(name[0] == '/') - pthread_rwlock_unlock(&glblVars_rwlock); - MsgUnlock(pM); + pthread_mutex_unlock(&glblVars_lock); + else + MsgUnlock(pM); RETiRet; } @@ -4577,7 +4800,7 @@ rsRetVal msgQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } * rgerhards, 2008-01-04 */ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) - pthread_rwlock_init(&glblVars_rwlock, NULL); + pthread_mutex_init(&glblVars_lock, NULL); /* request objects we use */ CHKiRet(objUse(datetime, CORE_COMPONENT)); diff --git a/runtime/msg.h b/runtime/msg.h index 3639a8fa2..89034598f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -214,6 +214,7 @@ uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(uchar *pName, propid_t *pPropID); uchar *propIDToName(propid_t propID); rsRetVal msgGetJSONPropJSON(msg_t *pMsg, msgPropDescr_t *pProp, struct json_object **pjson); +rsRetVal msgGetJSONPropJSONorString(msg_t * const pMsg, msgPropDescr_t *pProp, struct json_object **pjson, uchar **pcstr); rsRetVal getJSONPropVal(msg_t *pMsg, msgPropDescr_t *pProp, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed); rsRetVal msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *var, int force_reset); rsRetVal msgDelJSON(msg_t *pMsg, uchar *varname); diff --git a/runtime/net.c b/runtime/net.c index 55b12eee6..01129d62c 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1465,6 +1465,9 @@ finalize_it: * However, it caches entries in order to avoid too-frequent requery. * rgerhards, 2012-03-06 */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-align" /* TODO: how can we fix these warnings? */ +/* Problem with the warnings: they seem to stem back from the way the API is structured */ static rsRetVal getIFIPAddr(uchar *szif, int family, uchar *pszbuf, int lenBuf) { @@ -1503,6 +1506,7 @@ finalize_it: RETiRet; } +#pragma GCC diagnostic pop /* queryInterface function diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 481571adc..afc1e6063 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -643,6 +643,17 @@ gtlsInitSession(nsd_gtls_t *pThis) pThis->sess = session; +# if HAVE_GNUTLS_CERTIFICATE_SET_RETRIEVE_FUNCTION + /* store a pointer to ourselfs (needed by callback) */ + gnutls_session_set_ptr(pThis->sess, (void*)pThis); + iRet = gtlsLoadOurCertKey(pThis); /* first load .pem files */ + if(iRet == RS_RET_OK) { + gnutls_certificate_set_retrieve_function(xcred, gtlsClientCertCallback); + } else if(iRet != RS_RET_CERTLESS) { + FINALIZE; /* we have an error case! */ + } +# endif + finalize_it: RETiRet; } @@ -1565,14 +1576,6 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) } if(pThis->lenRcvBuf == 0) { /* EOS */ - *pLenBuf = 0; - /* in this case, we also need to free the receive buffer, if we - * allocated one. -- rgerhards, 2008-12-03 - */ - if(pThis->pszRcvBuf != NULL) { - free(pThis->pszRcvBuf); - pThis->pszRcvBuf = NULL; - } ABORT_FINALIZE(RS_RET_CLOSED); } @@ -1589,6 +1592,14 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) *pLenBuf = iBytesCopy; finalize_it: + if (iRet != RS_RET_OK) { + /* in this case, we also need to free the receive buffer, if we + * allocated one. -- rgerhards, 2008-12-03 -- moved here by alorbach, 2015-12-01 + */ + *pLenBuf = 0; + free(pThis->pszRcvBuf); + pThis->pszRcvBuf = NULL; + } dbgprintf("gtlsRcv return. nsd %p, iRet %d, lenRcvBuf %d, ptrRcvBuf %d\n", pThis, iRet, pThis->lenRcvBuf, pThis->ptrRcvBuf); RETiRet; } @@ -1651,6 +1662,8 @@ EnableKeepAlive(nsd_t *pNsd) * open a plain tcp socket and then, if in TLS mode, do a handshake on it. * rgerhards, 2008-03-19 */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" /* TODO: FIX Warnings! */ static rsRetVal Connect(nsd_t *pNsd, int family, uchar *port, uchar *host) { @@ -1743,6 +1756,7 @@ finalize_it: RETiRet; } +#pragma GCC diagnostic pop /* queryInterface function */ diff --git a/runtime/nsdpoll_ptcp.c b/runtime/nsdpoll_ptcp.c index 8d95811ad..2efa22c1b 100644 --- a/runtime/nsdpoll_ptcp.c +++ b/runtime/nsdpoll_ptcp.c @@ -2,7 +2,7 @@ * * An implementation of the nsd epoll() interface for plain tcp sockets. * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -252,12 +252,11 @@ Wait(nsdpoll_t *pNsdpoll, int timeout, int *numEntries, nsd_epworkset_t workset[ } /* we got valid events, so tell the caller... */ -dbgprintf("epoll returned %d entries\n", nfds); + DBGPRINTF("epoll returned %d entries\n", nfds); for(i = 0 ; i < nfds ; ++i) { pOurEvt = (nsdpoll_epollevt_lst_t*) event[i].data.ptr; workset[i].id = pOurEvt->id; workset[i].pUsr = pOurEvt->pUsr; -dbgprintf("epoll push ppusr[%d]: %p\n", i, pOurEvt->pUsr); } *numEntries = nfds; diff --git a/runtime/nsdsel_gtls.c b/runtime/nsdsel_gtls.c index 9d10ca1f9..76a62589f 100644 --- a/runtime/nsdsel_gtls.c +++ b/runtime/nsdsel_gtls.c @@ -2,7 +2,7 @@ * * An implementation of the nsd select() interface for GnuTLS. * - * Copyright (C) 2008-2012 Adiscon GmbH. + * Copyright (C) 2008-2016 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -175,7 +175,6 @@ doRetry(nsd_gtls_t *pNsd) finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_CLOSED && iRet != RS_RET_RETRY) pNsd->bAbortConn = 1; /* request abort */ -dbgprintf("XXXXXX: doRetry: iRet %d, pNsd->bAbortConn %d\n", iRet, pNsd->bAbortConn); RETiRet; } diff --git a/runtime/nssel.c b/runtime/nssel.c index 533d92ba4..241a84a63 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -192,7 +192,7 @@ Wait(nssel_t *pThis, int *piNumReady) * rgerhards, 2008-04-23 */ static rsRetVal -IsReady(nssel_t *pThis, netstrm_t *pStrm, nsdsel_waitOp_t waitOp, int *pbIsReady, int *piNumReady) +IsReady(nssel_t *pThis, netstrm_t *pStrm, nsdsel_waitOp_t waitOp, int *pbIsReady, int __attribute__((unused)) *piNumReady) { DEFiRet; ISOBJ_TYPE_assert(pThis, nssel); diff --git a/runtime/obj.c b/runtime/obj.c index 63f1f38cd..def88bc94 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -1106,7 +1106,7 @@ GetName(obj_t *pThis) ISOBJ_assert(pThis); if(pThis->pszName == NULL) { - snprintf((char*)szName, sizeof(szName)/sizeof(uchar), "%s %p", objGetClassName(pThis), pThis); + snprintf((char*)szName, sizeof(szName), "%s %p", objGetClassName(pThis), pThis); SetName(pThis, szName); /* looks strange, but we NEED to re-check because if there was an * error in objSetName(), the pointer may still be NULL @@ -1158,7 +1158,7 @@ finalize_it: /* DEV DEBUG ONLY dbgprintf("caller requested object '%s', found at index %d\n", (*ppInfo)->pszID, i);*/ /*EMPTY BY INTENSION*/; } else { - dbgprintf("caller requested object '%s', not found (iRet %d)\n", rsCStrGetSzStr(pstrOID), iRet); + dbgprintf("caller requested object '%s', not found (iRet %d)\n", rsCStrGetSzStrNoNULL(pstrOID), iRet); } RETiRet; diff --git a/runtime/parser.c b/runtime/parser.c index bac198df5..4cb11ec86 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -4,7 +4,7 @@ * * Module begun 2008-10-09 by Rainer Gerhards (based on previous code from syslogd.c) * - * Copyright 2008-2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2015 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -280,7 +280,7 @@ rsRetVal parserConstructViaModAndName(modInfo_t *__restrict__ pMod, uchar *const __restrict__ pName, void *pInst) { rsRetVal localRet; - parser_t *pParser; + parser_t *pParser = NULL; DEFiRet; if(pInst == NULL && pMod->mod.pm.newParserInst != NULL) { @@ -303,6 +303,8 @@ parserConstructViaModAndName(modInfo_t *__restrict__ pMod, uchar *const __restri pParser->pInst = pInst; CHKiRet(parserConstructFinalize(pParser)); finalize_it: + if(iRet != RS_RET_OK) + free(pParser); RETiRet; } BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ @@ -344,7 +346,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) */ int ret; iLenDefBuf = glbl.GetMaxLine(); - CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1))); + CHKmalloc(deflateBuf = MALLOC(iLenDefBuf + 1)); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, (int) (lenMsg-1)); @@ -474,7 +476,7 @@ SanitizeMsg(msg_t *pMsg) if(maxDest < sizeof(szSanBuf)) pDst = szSanBuf; else - CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pDst = MALLOC(iMaxLine + 1)); if(iSrc > 0) { iSrc--; /* go back to where everything is OK */ memcpy(pDst, pszMsg, iSrc); /* fast copy known good */ diff --git a/runtime/queue.c b/runtime/queue.c index 1f29f2aa5..765d65183 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -412,7 +412,7 @@ StartDA(qqueue_t *pThis) CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); /* give it a name */ - snprintf((char*) pszDAQName, sizeof(pszDAQName)/sizeof(uchar), "%s[DA]", obj.GetName((obj_t*) pThis)); + snprintf((char*) pszDAQName, sizeof(pszDAQName), "%s[DA]", obj.GetName((obj_t*) pThis)); obj.SetName((obj_t*) pThis->pqDA, pszDAQName); /* as the created queue is the same object class, we take the @@ -2119,7 +2119,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ /* pre-construct file name for .qi file */ - pThis->lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), + pThis->lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam), "%s/%s.qi", (char*) pThis->pszSpoolDir, (char*)pThis->pszFilePrefix); pThis->pszQIFNam = ustrdup(pszQIFNam); DBGOPRINT((obj_t*) pThis, ".qi file name is '%s', len %d\n", pThis->pszQIFNam, @@ -2627,7 +2627,7 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); - if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL) + if((pThis->pszFilePrefix = MALLOC(iLenPrefix + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1); pThis->lenFilePrefix = iLenPrefix; diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 14527b330..98bb7364e 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -167,6 +167,7 @@ void cnfSetDefaults(rsconf_t *pThis) BEGINobjConstruct(rsconf) /* be sure to specify the object type also in END macro! */ cnfSetDefaults(pThis); lookupInitCnf(&pThis->lu_tabs); + CHKiRet(dynstats_initCnf(&pThis->dynstats_buckets)); CHKiRet(llInit(&pThis->rulesets.llRulesets, rulesetDestructForLinkedList, rulesetKeyDestruct, strcasecmp)); finalize_it: @@ -208,6 +209,7 @@ BEGINobjDestruct(rsconf) /* be sure to specify the object type also in END and C CODESTARTobjDestruct(rsconf) freeCnf(pThis); tplDeleteAll(pThis); + dynstats_destroyAllBuckets(); free(pThis->globals.mainQ.pszMainMsgQFName); free(pThis->globals.pszConfDAGFile); lookupDestroyCnf(); @@ -427,6 +429,9 @@ void cnfDoObj(struct cnfobj *o) case CNFOBJ_LOOKUP_TABLE: lookupTableDefProcessCnf(o); break; + case CNFOBJ_DYN_STATS: + dynstats_processCnf(o); + break; case CNFOBJ_PARSER: parserProcessCnf(o); break; @@ -514,7 +519,7 @@ static void doDropPrivGid(int iGid) exit(1); } DBGPRINTF("setgid(%d): %d\n", iGid, res); - snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's groupid changed to %d", iGid); + snprintf((char*)szBuf, sizeof(szBuf), "rsyslogd's groupid changed to %d", iGid); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); } @@ -553,7 +558,7 @@ static void doDropPrivUid(int iUid) exit(1); } DBGPRINTF("setuid(%d): %d\n", iUid, res); - snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's userid changed to %d", iUid); + snprintf((char*)szBuf, sizeof(szBuf), "rsyslogd's userid changed to %d", iUid); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); } @@ -1004,7 +1009,7 @@ regBuildInModule(rsRetVal (*modInit)(), uchar *name, void *pModHdlr) DEFiRet; CHKiRet(module.doModInit(modInit, name, pModHdlr, &pMod)); readyModForCnf(pMod, &pNew, &pLast); - addModToCnfList(pNew, pLast); + addModToCnfList(&pNew, pLast); finalize_it: RETiRet; } @@ -1265,7 +1270,7 @@ validateConf(void) rsRetVal load(rsconf_t **cnf, uchar *confFile) { - int iNbrActions; + int iNbrActions = 0; int r; DEFiRet; diff --git a/runtime/rsconf.h b/runtime/rsconf.h index f64b70459..f43326ab2 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -26,6 +26,7 @@ #include "linkedlist.h" #include "queue.h" #include "lookup.h" +#include "dynstats.h" /* --- configuration objects (the plan is to have ALL upper layers in this file) --- */ @@ -145,6 +146,7 @@ struct rsconf_s { defaults_t defaults; templates_t templates; lookup_tables_t lu_tabs; + dynstats_buckets_t dynstats_buckets; outchannels_t och; actions_t actions; rulesets_t rulesets; diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index f72a1ed9d..68c31d454 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -195,6 +195,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(rsconfClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "lookup"; CHKiRet(lookupClassInit()); + if(ppErrObj != NULL) *ppErrObj = "dynstats"; + CHKiRet(dynstatsClassInit()); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 8bc259c0e..dd912eb32 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -171,6 +171,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_MORE_DATA = -3006, /**< insufficient data, e.g. end of string during parsing */ RS_RET_INVALID_IP = -3007, /**< invalid ip found where valid was expected */ RS_RET_OBJ_CREATION_FAILED = - 3008, /**< the creation of an object failed (no details available) */ + RS_RET_INOTIFY_INIT_FAILED = - 3009, /**< the initialization of an inotify instance failed (no details available) */ RS_RET_PARAM_ERROR = -1000, /**< invalid parameter in call to function */ RS_RET_MISSING_INTERFACE = -1001,/**< interface version mismatch, required missing */ RS_RET_INVALID_CORE_INTERFACE = -1002,/**< interface provided by host invalid, can not be used */ @@ -442,6 +443,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_CONF_PARAM_INVLD = -2425,/**< config parameter is invalid */ RS_RET_KSI_ERR = -2426,/**< error in KSI subsystem */ RS_RET_ERR_LIBLOGNORM = -2427,/**< cannot obtain liblognorm ctx */ + RS_RET_CONC_CTRL_ERR = -2428,/**< error in lock/unlock/condition/concurrent-modification operation */ + RS_RET_SENDER_GONE_AWAY = -2429,/**< warning: sender not seen for configured amount of time */ + RS_RET_SENDER_APPEARED = -2430,/**< info: new sender appeared */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ @@ -466,6 +470,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth # define CHKiRet(code) if((iRet = code) != RS_RET_OK) goto finalize_it #endif +# define CHKiConcCtrl(code) if (code != 0) { iRet = RS_RET_CONC_CTRL_ERR; goto finalize_it; } + /* macro below is to be used if we need our own handling, eg for cleanup */ #define CHKiRet_Hdlr(code) if((iRet = code) != RS_RET_OK) /* macro below is to handle failing malloc/calloc/strdup... which we almost always handle in the same way... */ @@ -619,14 +625,6 @@ void rsrtSetErrLogger(void (*errLogger)(const int, const int, const uchar*)); typedef int json_bool; #endif -#ifdef HAVE_JSON_OBJECT_TO_JSON_STRING_EXT -# define RS_json_object_to_json_string_ext(obj, flags) \ - json_object_to_json_string_ext((obj), (flags)) -#else -# define RS_json_object_to_json_string_ext(obj, flags) \ - json_object_to_json_string((obj)) -#endif - /* this define below is (later) intended to be used to implement empty * structs. TODO: check if compilers supports this and, if not, define * a dummy variable. This requires review of where in code empty structs diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 19611a40d..5f9476edf 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -516,6 +516,7 @@ processBatch(batch_t *pBatch, wti_t *pWti) int i; msg_t *pMsg; ruleset_t *pRuleset; + rsRetVal localRet; DEFiRet; DBGPRINTF("processBATCH: batch of %d elements must be processed\n", pBatch->nElem); @@ -527,15 +528,19 @@ processBatch(batch_t *pBatch, wti_t *pWti) pMsg = pBatch->pElem[i].pMsg; DBGPRINTF("processBATCH: next msg %d: %.128s\n", i, pMsg->pszRawMsg); pRuleset = (pMsg->pRuleset == NULL) ? ourConf->rulesets.pDflt : pMsg->pRuleset; - scriptExec(pRuleset->root, pMsg, pWti); - // TODO: think if we need a return state of scriptExec - most probably - // the answer is "no", as we need to process the batch in any case! - // TODO: we must refactor this! flag messages as committed - batchSetElemState(pBatch, i, BATCH_STATE_COMM); + localRet = scriptExec(pRuleset->root, pMsg, pWti); + /* the most important case here is that processing may be aborted + * due to pbShutdownImmediate, in which case we MUST NOT flag this + * message as committed. If we would do so, the message would + * potentially be lost. + */ + if(localRet == RS_RET_OK) + batchSetElemState(pBatch, i, BATCH_STATE_COMM); } /* commit phase */ - dbgprintf("END batch execution phase, entering to commit phase\n"); + DBGPRINTF("END batch execution phase, entering to commit phase " + "[processed %d of %d messages]\n", i, batchNumMsgs(pBatch)); actionCommitAllDirect(pWti); DBGPRINTF("processBATCH: batch of %d elements has been processed\n", pBatch->nElem); diff --git a/runtime/srutils.c b/runtime/srutils.c index de670721e..8b3e6cd58 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -206,7 +206,7 @@ int makeFileParentDirs(const uchar *const szFile, size_t lenFile, mode_t mode, assert(lenFile > 0); len = lenFile + 1; /* add one for '\0'-byte */ - if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL) + if((pszWork = MALLOC(len)) == NULL) return -1; memcpy(pszWork, szFile, len); for(p = pszWork+1 ; *p ; p++) @@ -356,7 +356,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * } lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ - if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL) + if((pName = MALLOC(lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); /* got memory, now construct string */ diff --git a/runtime/statsobj.c b/runtime/statsobj.c index fb767ad38..9bebadbfd 100644 --- a/runtime/statsobj.c +++ b/runtime/statsobj.c @@ -3,18 +3,18 @@ * This object provides a statistics-gathering facility inside rsyslog. This * functionality will be pragmatically implemented and extended. * - * Copyright 2010-2014 Adiscon GmbH. + * Copyright 2010-2016 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,8 +26,10 @@ #include #include #include +#include #include #include +#include #include #include "rsyslog.h" @@ -36,6 +38,9 @@ #include "statsobj.h" #include "srUtils.h" #include "stringbuf.h" +#include "errmsg.h" +#include "hashtable.h" +#include "hashtable_itr.h" /* externally-visiable data (see statsobj.h for explanation) */ @@ -43,6 +48,7 @@ int GatherStats = 0; /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(errmsg) /* doubly linked list of stats objects. Object is automatically linked to it * upon construction. Enqueue always happens at the front (simplifies logic). @@ -51,6 +57,9 @@ static statsobj_t *objRoot = NULL; static statsobj_t *objLast = NULL; static pthread_mutex_t mutStats; +static pthread_mutex_t mutSenders; + +static struct hashtable *stats_senders = NULL; /* ------------------------------ statsobj linked list maintenance ------------------------------ */ @@ -106,6 +115,7 @@ BEGINobjConstruct(statsobj) /* be sure to specify the object type also in END ma pthread_mutex_init(&pThis->mutCtr, NULL); pThis->ctrLast = NULL; pThis->ctrRoot = NULL; + pThis->read_notifier = NULL; ENDobjConstruct(statsobj) @@ -120,6 +130,17 @@ statsobjConstructFinalize(statsobj_t *pThis) RETiRet; } +/* set read_notifier (a function which is invoked after stats are read). + */ +static rsRetVal +setReadNotifier(statsobj_t *pThis, statsobj_read_notifier_t notifier, void* ctx) +{ + DEFiRet; + pThis->read_notifier = notifier; + pThis->read_notifier_ctx = ctx; + RETiRet; +} + /* set origin (module name, etc). * Note that we make our own copy of the memory, caller is @@ -156,15 +177,20 @@ finalize_it: * is called. */ static rsRetVal -addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr) +addManagedCounter(statsobj_t *pThis, const uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr, ctr_t **entryRef) { ctr_t *ctr; DEFiRet; - CHKmalloc(ctr = malloc(sizeof(ctr_t))); + *entryRef = NULL; + + CHKmalloc(ctr = calloc(1, sizeof(ctr_t))); ctr->next = NULL; ctr->prev = NULL; - CHKmalloc(ctr->name = ustrdup(ctrName)); + if((ctr->name = ustrdup(ctrName)) == NULL) { + DBGPRINTF("addCounter: OOM in strdup()\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } ctr->flags = flags; ctr->ctrType = ctrType; switch(ctrType) { @@ -176,11 +202,53 @@ addCounter(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, int8_t fla break; } addCtrToList(pThis, ctr); + *entryRef = ctr; finalize_it: + if (iRet != RS_RET_OK) { + if (ctr != NULL) { + free(ctr->name); + free(ctr); + } + } RETiRet; } +static rsRetVal +addCounter(statsobj_t *pThis, const uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr) +{ + ctr_t *ctr; + DEFiRet; + CHKiRet(addManagedCounter(pThis, ctrName, ctrType, flags, pCtr, &ctr)); +finalize_it: + RETiRet; +} + +static rsRetVal +destructCounter(statsobj_t *pThis, ctr_t *pCtr) +{ + DEFiRet; + + pthread_mutex_lock(&pThis->mutCtr); + if (pCtr->prev != NULL) { + pCtr->prev->next = pCtr->next; + } + if (pCtr->next != NULL) { + pCtr->next->prev = pCtr->prev; + } + if (pThis->ctrLast == pCtr) { + pThis->ctrLast = pCtr->prev; + } + if (pThis->ctrRoot == pCtr) { + pThis->ctrRoot = pCtr->next; + } + pthread_mutex_unlock(&pThis->mutCtr); + free(pCtr->name); + free(pCtr); + + RETiRet; +} + static inline void resetResettableCtr(ctr_t *pCtr, int8_t bResetCtrs) { @@ -198,7 +266,7 @@ resetResettableCtr(ctr_t *pCtr, int8_t bResetCtrs) /* get all the object's countes together as CEE. */ static rsRetVal -getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, int cee_cookie, int8_t bResetCtrs) +getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, const statsFmtType_t fmt, const int8_t bResetCtrs) { cstr_t *pcstr; ctr_t *pCtr; @@ -206,7 +274,7 @@ getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, int cee_cookie, int8_t bRese CHKiRet(cstrConstruct(&pcstr)); - if (cee_cookie == 1) + if (fmt == statsFmt_JSON) rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT("@cee: "), 6); rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT("{"), 1); @@ -234,7 +302,22 @@ getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, int cee_cookie, int8_t bRese pthread_mutex_lock(&pThis->mutCtr); for(pCtr = pThis->ctrRoot ; pCtr != NULL ; pCtr = pCtr->next) { rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT("\""), 1); - rsCStrAppendStr(pcstr, pCtr->name); + if (fmt == statsFmt_JSON_ES) { + /* work-around for broken Elasticsearch JSON implementation: + * we need to replace dots by a different char, we use bang. + * Note: ES 2.0 does not longer accept dot in name + */ + uchar esbuf[256]; + strncpy((char*)esbuf, (char*)pCtr->name, sizeof(esbuf)-1); + esbuf[sizeof(esbuf)-1] = '\0'; + for(uchar *c = esbuf ; *c ; ++c) { + if(*c == '.') + *c = '!'; + } + rsCStrAppendStr(pcstr, esbuf); + } else { + rsCStrAppendStr(pcstr, pCtr->name); + } rsCStrAppendStrWithLen(pcstr, UCHAR_CONSTANT("\""), 1); cstrAppendChar(pcstr, ':'); switch(pCtr->ctrType) { @@ -306,6 +389,56 @@ finalize_it: } + +/* this function obtains all sender stats. hlper to getAllStatsLines() + * We need to keep this looked to avoid resizing of the hash table + * (what could otherwise cause a segfault). + */ +static void +getSenderStats(rsRetVal(*cb)(void*, cstr_t*), + void *usrptr, + statsFmtType_t fmt, + const int8_t bResetCtrs) +{ + struct hashtable_itr *itr; + struct sender_stats *stat; + char fmtbuf[2048]; + + pthread_mutex_lock(&mutSenders); + + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty + */ + if(hashtable_count(stats_senders) > 0) { + itr = hashtable_iterator(stats_senders); + do { + stat = (struct sender_stats*)hashtable_iterator_value(itr); + if(fmt == statsFmt_Legacy) { + snprintf(fmtbuf, sizeof(fmtbuf), + "_sender_stat: sender=%s messages=%" + PRIu64, + stat->sender, stat->nMsgs); + } else { + snprintf(fmtbuf, sizeof(fmtbuf), + "{ \"name\":\"_sender_stat\", " + "\"sender\":\"%s\", \"messages\":\"%" + PRIu64 "\"}", + stat->sender, stat->nMsgs); + } + fmtbuf[sizeof(fmtbuf)-1] = '\0'; + cstr_t *cs; + rsCStrConstructFromszStr(&cs, (uchar*)fmtbuf); + cb(usrptr, cs); + rsCStrDestruct(&cs); + if(bResetCtrs) + stat->nMsgs = 0; + } while (hashtable_iterator_advance(itr)); + } + + pthread_mutex_unlock(&mutSenders); +} + + /* this function can be used to obtain all stats lines. In this case, * a callback must be provided. This module than iterates over all objects and * submits each stats line to the callback. The callback has two parameters: @@ -313,7 +446,7 @@ finalize_it: * line. If the callback reports an error, processing is stopped. */ static rsRetVal -getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr, statsFmtType_t fmt, int8_t bResetCtrs) +getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr, statsFmtType_t fmt, const int8_t bResetCtrs) { statsobj_t *o; cstr_t *cstr; @@ -325,16 +458,20 @@ getAllStatsLines(rsRetVal(*cb)(void*, cstr_t*), void *usrptr, statsFmtType_t fmt CHKiRet(getStatsLine(o, &cstr, bResetCtrs)); break; case statsFmt_CEE: - CHKiRet(getStatsLineCEE(o, &cstr, 1, bResetCtrs)); - break; case statsFmt_JSON: - CHKiRet(getStatsLineCEE(o, &cstr, 0, bResetCtrs)); + case statsFmt_JSON_ES: + CHKiRet(getStatsLineCEE(o, &cstr, fmt, bResetCtrs)); break; } CHKiRet(cb(usrptr, cstr)); rsCStrDestruct(&cstr); + if (o->read_notifier != NULL) { + o->read_notifier(o, o->read_notifier_ctx); + } } + getSenderStats(cb, usrptr, fmt, bResetCtrs); + finalize_it: RETiRet; } @@ -351,13 +488,54 @@ enableStats() } -/* destructor for the statsobj object */ -BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ - ctr_t *ctr, *ctrToDel; -CODESTARTobjDestruct(statsobj) - removeFromObjList(pThis); +rsRetVal +statsRecordSender(const uchar *sender, unsigned nMsgs, time_t lastSeen) +{ + struct sender_stats *stat; + int mustUnlock = 0; + DEFiRet; + + if(stats_senders == NULL) + FINALIZE; /* unlikely: we could not init our hash table */ + + pthread_mutex_lock(&mutSenders); + mustUnlock = 1; + stat = hashtable_search(stats_senders, (void*)sender); + if(stat == NULL) { + DBGPRINTF("statsRecordSender: sender '%s' not found, adding\n", + sender); + CHKmalloc(stat = calloc(1, sizeof(struct sender_stats))); + stat->sender = (const uchar*)strdup((const char*)sender); + stat->nMsgs = 0; + if(glblReportNewSenders) { + errmsg.LogMsg(0, RS_RET_SENDER_APPEARED, + LOG_INFO, "new sender '%s'", stat->sender); + } + if(hashtable_insert(stats_senders, (void*)stat->sender, + (void*)stat) == 0) { + errmsg.LogError(errno, RS_RET_INTERNAL_ERROR, + "error inserting sender '%s' into sender " + "hash table", sender); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } + } + + stat->nMsgs += nMsgs; + stat->lastSeen = lastSeen; + DBGPRINTF("DDDDD: statsRecordSender: '%s', nmsgs %u [%llu], lastSeen %llu\n", sender, nMsgs, (long long unsigned) stat->nMsgs, (long long unsigned) lastSeen); + +finalize_it: + if(mustUnlock) + pthread_mutex_unlock(&mutSenders); + RETiRet; +} + + +static rsRetVal +destructAllCounters(statsobj_t *pThis) { + DEFiRet; + ctr_t *ctr, *ctrToDel; - /* destruct counters */ ctr = pThis->ctrRoot; while(ctr != NULL) { ctrToDel = ctr; @@ -366,6 +544,60 @@ CODESTARTobjDestruct(statsobj) free(ctrToDel); } + pThis->ctrLast = NULL; + pThis->ctrRoot = NULL; + + RETiRet; +} + +/* check if a sender has not sent info to us for an extended period + * of time. + */ +void +checkGoneAwaySenders(const time_t tCurr) +{ + struct hashtable_itr *itr; + struct sender_stats *stat; + const time_t rqdLast = tCurr - glblSenderStatsTimeout; + struct tm tm; + + pthread_mutex_lock(&mutSenders); + + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty + */ + if(hashtable_count(stats_senders) > 0) { + itr = hashtable_iterator(stats_senders); + do { + stat = (struct sender_stats*)hashtable_iterator_value(itr); + if(stat->lastSeen < rqdLast) { + if(glblReportGoneAwaySenders) { + localtime_r(&stat->lastSeen, &tm); + errmsg.LogMsg(0, RS_RET_SENDER_GONE_AWAY, + LOG_WARNING, + "removing sender '%s' from connection " + "table, last seen at " + "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", + stat->sender, + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec); + } + hashtable_remove(stats_senders, (void*)stat->sender); + } + } while (hashtable_iterator_advance(itr)); + } + + pthread_mutex_unlock(&mutSenders); +} + +/* destructor for the statsobj object */ +BEGINobjDestruct(statsobj) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(statsobj) + removeFromObjList(pThis); + + /* destruct counters */ + CHKiRet(destructAllCounters(pThis)); + pthread_mutex_destroy(&pThis->mutCtr); free(pThis->name); free(pThis->origin); @@ -398,9 +630,12 @@ CODESTARTobjQueryInterface(statsobj) pIf->DebugPrint = statsobjDebugPrint; pIf->SetName = setName; pIf->SetOrigin = setOrigin; - //pIf->GetStatsLine = getStatsLine; + pIf->SetReadNotifier = setReadNotifier; pIf->GetAllStatsLines = getAllStatsLines; pIf->AddCounter = addCounter; + pIf->AddManagedCounter = addManagedCounter; + pIf->DestructCounter = destructCounter; + pIf->DestructAllCounters = destructAllCounters; pIf->EnableStats = enableStats; finalize_it: ENDobjQueryInterface(statsobj) @@ -415,10 +650,17 @@ BEGINAbstractObjClassInit(statsobj, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, statsobjDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, statsobjConstructFinalize); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* init other data items */ pthread_mutex_init(&mutStats, NULL); + pthread_mutex_init(&mutSenders, NULL); + if((stats_senders = create_hashtable(100, hash_from_string, key_equals_string, NULL)) == NULL) { + errmsg.LogError(0, RS_RET_INTERNAL_ERROR, "error trying to initialize hash-table " + "for sender table. Sender statistics and warnings are disabled."); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } ENDObjClassInit(statsobj) /* Exit the class. @@ -426,4 +668,5 @@ ENDObjClassInit(statsobj) BEGINObjClassExit(statsobj, OBJ_IS_CORE_MODULE) /* class, version */ /* release objects we no longer need */ pthread_mutex_destroy(&mutStats); + pthread_mutex_destroy(&mutSenders); ENDObjClassExit(statsobj) diff --git a/runtime/statsobj.h b/runtime/statsobj.h index 065c2949a..cac670465 100644 --- a/runtime/statsobj.h +++ b/runtime/statsobj.h @@ -1,17 +1,17 @@ /* The statsobj object. * - * Copyright 2010-2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010-2016 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,6 +47,7 @@ typedef enum statsCtrType_e { typedef enum statsFmtType_e { statsFmt_Legacy, statsFmt_JSON, + statsFmt_JSON_ES, statsFmt_CEE } statsFmtType_t; @@ -71,6 +72,8 @@ struct statsobj_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ uchar *name; uchar *origin; + statsobj_read_notifier_t read_notifier; + void *read_notifier_ctx; pthread_mutex_t mutCtr; /* to guard counter linked-list ops */ ctr_t *ctrRoot; /* doubly-linked list of statsobj counters */ ctr_t *ctrLast; @@ -79,6 +82,12 @@ struct statsobj_s { statsobj_t *next; }; +struct sender_stats { + const uchar *sender; + uint64_t nMsgs; + time_t lastSeen; +}; + /* interfaces */ BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */ @@ -88,9 +97,13 @@ BEGINinterface(statsobj) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Destruct)(statsobj_t **ppThis); rsRetVal (*SetName)(statsobj_t *pThis, uchar *name); rsRetVal (*SetOrigin)(statsobj_t *pThis, uchar *name); /* added v12, 2014-09-08 */ + rsRetVal (*SetReadNotifier)(statsobj_t *pThis, statsobj_read_notifier_t notifier, void* ctx); //rsRetVal (*GetStatsLine)(statsobj_t *pThis, cstr_t **ppcstr); rsRetVal (*GetAllStatsLines)(rsRetVal(*cb)(void*, cstr_t*), void *usrptr, statsFmtType_t fmt, int8_t bResetCtr); - rsRetVal (*AddCounter)(statsobj_t *pThis, uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr); + rsRetVal (*AddCounter)(statsobj_t *pThis, const uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr); + rsRetVal (*AddManagedCounter)(statsobj_t *pThis, const uchar *ctrName, statsCtrType_t ctrType, int8_t flags, void *pCtr, ctr_t **ref); + rsRetVal (*DestructCounter)(statsobj_t *pThis, ctr_t *ref); + rsRetVal (*DestructAllCounters)(statsobj_t *pThis); rsRetVal (*EnableStats)(void); ENDinterface(statsobj) #define statsobjCURR_IF_VERSION 12 /* increment whenever you change the interface structure! */ @@ -105,6 +118,12 @@ ENDinterface(statsobj) /* prototypes */ PROTOTYPEObj(statsobj); +rsRetVal statsRecordSender(const uchar *sender, unsigned nMsgs, time_t lastSeen); +/* checkGoneAwaySenders() is part of this module because all it needs is + * done by this module, so even though it's own processing is not directly + * related to stats, it makes sense to do it here... -- rgerhards, 2016-02-01 + */ +void checkGoneAwaySenders(time_t); /* macros to handle stats counters * These are to be used by "counter providers". Note that we MUST @@ -151,6 +170,10 @@ PROTOTYPEObj(statsobj); if(GatherStats) \ ATOMIC_INC_uint64(&ctr, &mut); +#define STATSCOUNTER_BUMP(ctr, mut, delta) \ + if(GatherStats) \ + ATOMIC_ADD_uint64(&ctr, &mut, delta); + #define STATSCOUNTER_DEC(ctr, mut) \ if(GatherStats) \ ATOMIC_DEC_uint64(&ctr, mut); diff --git a/runtime/stream.c b/runtime/stream.c index 7cca646f8..7fd9ddd3a 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -509,13 +509,17 @@ strmHandleEOFMonitor(strm_t *pThis) DBGPRINTF("stream checking for file change on '%s', inode %u/%u\n", pThis->pszCurrFName, (unsigned) pThis->inode, (unsigned) statName.st_ino); - if(pThis->inode == statName.st_ino) { - ABORT_FINALIZE(RS_RET_EOF); - } else { - /* we had a file change! */ + + /* Inode unchanged but file size on disk is less than current offset + * means file was truncated, we also reopen if 'reopenOnTruncate' is on + */ + if (pThis->inode != statName.st_ino + || (pThis->bReopenOnTruncate && statName.st_size < pThis->iCurrOffs)) { DBGPRINTF("we had a file change on '%s'\n", pThis->pszCurrFName); CHKiRet(strmCloseFile(pThis)); CHKiRet(strmOpenFile(pThis)); + } else { + ABORT_FINALIZE(RS_RET_EOF); } finalize_it: @@ -703,7 +707,8 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF) /* append previous message to current message if necessary */ if(pThis->prevLineSegment != NULL) { - dbgprintf("DDDDD: readLine: have previous line segment: '%s'\n", rsCStrGetSzStr(pThis->prevLineSegment)); + dbgprintf("readLine: have previous line segment: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->prevLineSegment)); CHKiRet(cstrAppendCStr(*ppCStr, pThis->prevLineSegment)); cstrDestruct(&pThis->prevLineSegment); } @@ -791,10 +796,9 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF) } finalize_it: -dbgprintf("DDDDD: readLine returns[%d]: '%s' [*ppCStr %p]\n", iRet, (char*)rsCStrGetSzStr(*ppCStr), *ppCStr); if(iRet != RS_RET_OK && *ppCStr != NULL) { - if(cstrLen(*ppCStr) > 0) { /* we may have an empty string in an unsuccsfull poll or after restart! */ - dbgprintf("DDDDD: readLine saves segment '%s'\n", rsCStrGetSzStr(*ppCStr)); + if(cstrLen(*ppCStr) > 0) { + /* we may have an empty string in an unsuccsfull poll or after restart! */ rsCStrConstructFromCStr(&pThis->prevLineSegment, *ppCStr); } cstrDestruct(ppCStr); @@ -915,7 +919,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) * to make sure we can write out everything with a SINGLE api call! * We add another 128 bytes to take care of the gzip header and "all eventualities". */ - CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * (pThis->sIOBufSize + 128))); + CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(pThis->sIOBufSize + 128)); } } @@ -947,7 +951,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) pthread_cond_init(&pThis->isEmpty, 0); pThis->iCnt = pThis->iEnq = pThis->iDeq = 0; for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { - CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(pThis->sIOBufSize)); } pThis->pIOBuf = pThis->asyncBuf[0].pBuf; pThis->bStopWriter = 0; @@ -961,7 +965,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); } else { /* we work synchronously, so we need to alloc a fixed pIOBuf */ - CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(pThis->sIOBufSize)); } finalize_it: @@ -1271,9 +1275,10 @@ asyncWriterThread(void *pPtr) } if(bTimedOut && pThis->iBufPtr > 0) { /* if we timed out, we need to flush pending data */ + d_pthread_mutex_unlock(&pThis->mut); strmFlushInternal(pThis, 0); bTimedOut = 0; - d_pthread_mutex_unlock(&pThis->mut); + d_pthread_mutex_lock(&pThis->mut); continue; } bTimedOut = 0; @@ -1310,7 +1315,7 @@ asyncWriterThread(void *pPtr) pthread_cond_broadcast(&pThis->isEmpty); } } - d_pthread_mutex_unlock(&pThis->mut); + /* Not reached */ finalize_it: ENDfunc @@ -1351,7 +1356,8 @@ syncFile(strm_t *pThis) } if(pThis->fdDir != -1) { - ret = fsync(pThis->fdDir); + if(fsync(pThis->fdDir) != 0) + DBGPRINTF("stream/syncFile: fsync returned error, ignoring\n"); } finalize_it: @@ -1789,6 +1795,7 @@ DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bVeryReliableZip, int) DEFpropSetMeth(strm, bSync, int) +DEFpropSetMeth(strm, bReopenOnTruncate, int) DEFpropSetMeth(strm, sIOBufSize, size_t) DEFpropSetMeth(strm, iSizeLimit, off_t) DEFpropSetMeth(strm, iFlushInterval, int) @@ -1832,7 +1839,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL) + if((pThis->pszFName = MALLOC(iLenName + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -1859,7 +1866,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - CHKmalloc(pThis->pszDir = MALLOC(sizeof(uchar) * (iLenDir + 1))); + CHKmalloc(pThis->pszDir = MALLOC(iLenDir + 1)); memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */ pThis->lenDir = iLenDir; @@ -2150,6 +2157,7 @@ CODESTARTobjQueryInterface(strm) pIf->SetiZipLevel = strmSetiZipLevel; pIf->SetbVeryReliableZip = strmSetbVeryReliableZip; pIf->SetbSync = strmSetbSync; + pIf->SetbReopenOnTruncate = strmSetbReopenOnTruncate; pIf->SetsIOBufSize = strmSetsIOBufSize; pIf->SetiSizeLimit = strmSetiSizeLimit; pIf->SetiFlushInterval = strmSetiFlushInterval; diff --git a/runtime/stream.h b/runtime/stream.h index b6d8bf34c..266b4f11d 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -111,6 +111,7 @@ typedef struct strm_s { /* dynamic properties, valid only during file open, not to be persistet */ sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ sbool bSync; /* sync this file after every write? */ + sbool bReopenOnTruncate; size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; @@ -187,6 +188,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); INTERFACEpropSetMeth(strm, bSync, int); + INTERFACEpropSetMeth(strm, bReopenOnTruncate, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); INTERFACEpropSetMeth(strm, iSizeLimit, off_t); INTERFACEpropSetMeth(strm, iFlushInterval, int); @@ -201,8 +203,9 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, cryprov, cryprov_if_t*); INTERFACEpropSetMeth(strm, cryprovData, void*); ENDinterface(strm) -#define strmCURR_IF_VERSION 10 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 11 /* increment whenever you change the interface structure! */ /* V10, 2013-09-10: added new parameter bEscapeLF, changed mode to uint8_t (rgerhards) */ +/* V11, 2015-12-03: added new parameter bReopenOnTruncate */ static inline int strmGetCurrFileNum(strm_t *pStrm) { diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 430e3b8e1..0e3135b7f 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -41,7 +41,6 @@ #include "regexp.h" #include "obj.h" -uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -90,7 +89,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); - if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -107,7 +106,7 @@ finalize_it: /* a helper function for rsCStr*Strf() */ -static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, char *fmt, va_list ap) __attribute__((format(gnu_printf,2, 0))); +static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, char *fmt, va_list ap) __attribute__((format(printf,2, 0))); static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, char *fmt, va_list ap) { DEFiRet; @@ -128,7 +127,7 @@ static rsRetVal rsCStrConstructFromszStrv(cstr_t **ppThis, char *fmt, va_list ap pThis->iBufSize = pThis->iStrLen = len; len++; /* account for the \0 written by vsnprintf */ - if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * len)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(len)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -168,7 +167,7 @@ rsRetVal cstrConstructFromESStr(cstr_t **ppThis, es_str_t *str) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = es_strlen(str); - if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -197,7 +196,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen; - if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -252,7 +251,7 @@ rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) iNewSize += pThis->iBufSize; /* add current size */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize)); pThis->iBufSize = iNewSize; pThis->pBuf = pNewBuf; @@ -366,7 +365,7 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->pszBuf = NULL; /* now save the new value */ - if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); return RS_RET_OUT_OF_MEMORY; } @@ -378,43 +377,23 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) return RS_RET_OK; } -/* Converts the CStr object to a classical sz string and returns that. - * Same restrictions as in rsCStrGetSzStr() applies (see there!). This - * function here guarantees that a valid string is returned, even if - * the CStr object currently holds a NULL pointer string buffer. If so, - * "" is returned. - * rgerhards 2005-10-19 - * WARNING: The returned pointer MUST NOT be freed, as it may be - * obtained from that constant memory pool (in case of NULL!) - */ -uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf == NULL) - return (uchar*) ""; - else - return rsCStrGetSzStr(pThis); -} - - /* Converts the CStr object to a classical zero-terminated C string * and returns that string. The caller must not free it and must not * destroy the CStr object as long as the ascii string is used. - * This function may return NULL, if the string is currently NULL. This - * is a feature, not a bug. If you need non-NULL in any case, use - * rsCStrGetSzStrNoNULL() instead. - * rgerhards, 2005-09-15 */ -uchar* rsCStrGetSzStr(cstr_t *pThis) +uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis) { size_t i; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + if(pThis->pBuf != NULL) if(pThis->pszBuf == NULL) { /* we do not yet have a usable sz version - so create it... */ - if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { + if((pThis->pszBuf = MALLOC(pThis->iStrLen + 1)) == NULL) { /* TODO: think about what to do - so far, I have no bright * idea... rgerhards 2005-09-07 */ @@ -471,7 +450,7 @@ rsRetVal cstrConvSzStrAndDestruct(cstr_t **ppThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - CHKmalloc(pRetBuf = MALLOC(sizeof(uchar))); + CHKmalloc(pRetBuf = MALLOC(1)); *pRetBuf = '\0'; } else { pRetBuf = NULL; @@ -715,7 +694,7 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *rc) if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { if (*cache == NULL) { *cache = calloc(sizeof(regex_t), 1); - regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); + regexp.regcomp(*cache, (char*) rsCStrGetSzStrNoNULL(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); } ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); if(ret != 0) @@ -888,9 +867,9 @@ rsCStrConvertToBool(cstr_t *pStr, number_t *pBool) } /* TODO: maybe we can do better than strcasecmp ;) -- overhead! */ - if(!strcasecmp((char*)rsCStrGetSzStr(pStr), "true")) { + if(!strcasecmp((char*)rsCStrGetSzStrNoNULL(pStr), "true")) { *pBool = 1; - } else if(!strcasecmp((char*)rsCStrGetSzStr(pStr), "yes")) { + } else if(!strcasecmp((char*)rsCStrGetSzStrNoNULL(pStr), "yes")) { *pBool = 1; } else { *pBool = 0; diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 51a1c3aff..ed0686fed 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -188,7 +188,6 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 3ee1c3a11..9339035eb 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -169,7 +169,11 @@ addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; - CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + if((pEntry->pszInputName = ustrdup(pThis->pszInputName)) == NULL) { + DBGPRINTF("strmsrv/addNewLstnPort: OOM in strdup()\n"); + free(pEntry); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } /* and add to list */ pEntry->pNext = pThis->pLstnPorts; diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index fe9dfa9f1..2cbe8039a 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -100,6 +100,7 @@ struct syslogTime { char OffsetMode; /* UTC offset + or - */ short year; int secfrac; /* fractional seconds (must be 32 bit!) */ + intTiny inUTC; /* forced UTC? */ }; typedef struct syslogTime syslogTime_t; diff --git a/runtime/tcpclt.c b/runtime/tcpclt.c index af3dcf238..cb3fc002a 100644 --- a/runtime/tcpclt.c +++ b/runtime/tcpclt.c @@ -169,7 +169,7 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) * I have added this comment so that the logic is not accidently * changed again. rgerhards, 2005-10-25 */ - if((buf = MALLOC((len + 2) * sizeof(char))) == NULL) { + if((buf = MALLOC(len + 2)) == NULL) { /* extreme mem shortage, try to solve * as good as we can. No point in calling * any alarms, they might as well run out @@ -213,11 +213,11 @@ TCPSendBldFrame(tcpclt_t *pThis, char **pmsg, size_t *plen, int *pbMustBeFreed) * comments with "IETF20061218". * rgerhards, 2006-12-19 */ - iLenBuf = snprintf(szLenBuf, sizeof(szLenBuf)/sizeof(char), "%d ", (int) len); + iLenBuf = snprintf(szLenBuf, sizeof(szLenBuf), "%d ", (int) len); /* IETF20061218 iLenBuf = - snprintf(szLenBuf, sizeof(szLenBuf)/sizeof(char), "%d ", len + iLenBuf);*/ + snprintf(szLenBuf, sizeof(szLenBuf), "%d ", len + iLenBuf);*/ - if((buf = MALLOC((len + iLenBuf) * sizeof(char))) == NULL) { + if((buf = MALLOC(len + iLenBuf)) == NULL) { /* we are out of memory. This is an extreme situation. We do not * call any alarm handlers because they most likely run out of mem, * too. We are brave enough to call debug output, though. Other than diff --git a/runtime/tcps_sess.c b/runtime/tcps_sess.c index 07ae1d5c5..73a89e8bb 100644 --- a/runtime/tcps_sess.c +++ b/runtime/tcps_sess.c @@ -70,7 +70,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ /* now allocate the message reception buffer */ - CHKmalloc(pThis->pMsg = (uchar*) MALLOC(sizeof(uchar) * glbl.GetMaxLine() + 1)); + CHKmalloc(pThis->pMsg = (uchar*) MALLOC(glbl.GetMaxLine() + 1)); finalize_it: ENDobjConstruct(tcps_sess) @@ -252,7 +252,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); - if(pThis->pLstnInfo->dfltTZ != NULL) + if(pThis->pLstnInfo->dfltTZ[0] != '\0') MsgSetDfltTZ(pMsg, (char*) pThis->pLstnInfo->dfltTZ); MsgSetFlowControlType(pMsg, pThis->pSrv->bUseFlowControl ? eFLOWCTL_LIGHT_DELAY : eFLOWCTL_NO_DELAY); @@ -315,7 +315,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ DBGPRINTF("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); defaultDoSubmitMessage(pThis, &stTime, ttGenTime, NULL); } @@ -353,7 +353,12 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) +processDataRcvd(tcps_sess_t *pThis, + char c, + struct syslogTime *stTime, + const time_t ttGenTime, + multi_submit_t *pMultiSub, + unsigned *const __restrict__ pnMsgs) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -408,6 +413,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt /* emergency, we now need to flush, no matter if we are at end of message or not... */ DBGPRINTF("error: message received is larger than max msg size, we split it\n"); defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -419,6 +425,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -436,6 +443,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); + ++(*pnMsgs); pThis->inputState = eAtStrtFram; } } @@ -463,20 +471,21 @@ finalize_it: */ #define NUM_MULTISUB 1024 static rsRetVal -DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) +DataRcvd(tcps_sess_t *pThis, char *pData, const size_t iLen) { multi_submit_t multiSub; msg_t *pMsgs[NUM_MULTISUB]; struct syslogTime stTime; time_t ttGenTime; char *pEnd; + unsigned nMsgs = 0; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); assert(pData != NULL); assert(iLen > 0); - datetime.getCurrTime(&stTime, &ttGenTime); + datetime.getCurrTime(&stTime, &ttGenTime, TIME_IN_LOCALTIME); multiSub.ppMsgs = pMsgs; multiSub.maxElem = NUM_MULTISUB; multiSub.nElem = 0; @@ -485,10 +494,13 @@ DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) pEnd = pData + iLen; /* this is one off, which is intensional */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime, &multiSub)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime, &multiSub, &nMsgs)); } iRet = multiSubmitFlush(&multiSub); + if(glblSenderKeepTrack) + statsRecordSender(propGetSzStr(pThis->fromHost), nMsgs, ttGenTime); + finalize_it: RETiRet; } diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c index 56d79fa3b..7e6ea8f71 100644 --- a/runtime/tcpsrv.c +++ b/runtime/tcpsrv.c @@ -133,11 +133,22 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort, int bSuppOctetFram, uchar *pszAd /* create entry */ CHKmalloc(pEntry = MALLOC(sizeof(tcpLstnPortList_t))); - CHKmalloc(pEntry->pszPort = ustrdup(pszPort)); + if((pEntry->pszPort = ustrdup(pszPort)) == NULL) { + DBGPRINTF("tcpsrv/addNewLstnPort: OOM in strdup()\n"); + free(pEntry); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } - pEntry->pszAddr = NULL; // Initalize address to null + pEntry->pszAddr = NULL; /* only if a bind adress is defined copy it in struct */ - if (pszAddr != NULL) CHKmalloc(pEntry->pszAddr = ustrdup(pszAddr)); + if (pszAddr != NULL) { + if((pEntry->pszAddr = ustrdup(pszAddr)) == NULL) { + DBGPRINTF("tcpsrv/addNewLstnPort: OOM in strdup() 2\n"); + free(pEntry->pszPort); + free(pEntry); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + } strcpy((char*)pEntry->dfltTZ, (char*)pThis->dfltTZ); pEntry->bSPFramingFix = pThis->bSPFramingFix; diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 30e3752a6..e53f9236a 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -84,6 +84,7 @@ typedef struct parserList_s parserList_t; typedef struct strgen_s strgen_t; typedef struct strgenList_s strgenList_t; typedef struct statsobj_s statsobj_t; +typedef void (*statsobj_read_notifier_t)(statsobj_t *, void *); typedef struct nsd_epworkset_s nsd_epworkset_t; typedef struct templates_s templates_t; typedef struct queuecnf_s queuecnf_t; @@ -117,6 +118,9 @@ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerou typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef struct actWrkrIParams actWrkrIParams_t; +typedef struct dynstats_bucket_s dynstats_bucket_t; +typedef struct dynstats_buckets_s dynstats_buckets_t; +typedef struct dynstats_ctr_s dynstats_ctr_t; /* under Solaris (actually only SPARC), we need to redefine some types * to be void, so that we get void* pointers. Otherwise, we will see @@ -201,6 +205,7 @@ typedef uintTiny propid_t; #define PROP_MSGID 22 #define PROP_PARSESUCCESS 23 #define PROP_JSONMESG 24 +#define PROP_RAWMSG_AFTER_PRI 25 #define PROP_SYS_NOW 150 #define PROP_SYS_YEAR 151 #define PROP_SYS_MONTH 152 @@ -213,6 +218,14 @@ typedef uintTiny propid_t; #define PROP_SYS_BOM 159 #define PROP_SYS_UPTIME 160 #define PROP_UUID 161 +#define PROP_SYS_NOW_UTC 162 +#define PROP_SYS_YEAR_UTC 163 +#define PROP_SYS_MONTH_UTC 164 +#define PROP_SYS_DAY_UTC 165 +#define PROP_SYS_HOUR_UTC 166 +#define PROP_SYS_HHOUR_UTC 167 +#define PROP_SYS_QHOUR_UTC 168 +#define PROP_SYS_MINUTE_UTC 169 #define PROP_CEE 200 #define PROP_CEE_ALL_JSON 201 #define PROP_LOCAL_VAR 202 diff --git a/runtime/var.c b/runtime/var.c index eecc5d6a2..2e77777a5 100644 --- a/runtime/var.c +++ b/runtime/var.c @@ -78,7 +78,7 @@ BEGINobjDebugPrint(var) /* be sure to specify the object type also in END and CO CODESTARTobjDebugPrint(var) switch(pThis->varType) { case VARTYPE_STR: - dbgoprint((obj_t*) pThis, "type: cstr, val '%s'\n", rsCStrGetSzStr(pThis->val.pStr)); + dbgoprint((obj_t*) pThis, "type: cstr, val '%s'\n", rsCStrGetSzStrNoNULL(pThis->val.pStr)); break; case VARTYPE_NUMBER: dbgoprint((obj_t*) pThis, "type: number, val %lld\n", pThis->val.num); diff --git a/runtime/wti.c b/runtime/wti.c index e95bdf1a4..ff919b1f7 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -408,7 +408,7 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg) free(pThis->pszDbgHdr); } - if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ @@ -437,7 +437,8 @@ wtiGetDummy(void) pWti = (wti_t*) pthread_getspecific(thrd_wti_key); if(pWti == NULL) { wtiConstruct(&pWti); - wtiConstructFinalize(pWti); + if(pWti != NULL) + wtiConstructFinalize(pWti); if(pthread_setspecific(thrd_wti_key, pWti) != 0) { DBGPRINTF("wtiGetDummy: error setspecific thrd_wti_key\n"); } @@ -469,7 +470,7 @@ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most r = pthread_key_create(&thrd_wti_key, NULL); if(r != 0) { dbgprintf("wti.c: pthread_key_create failed\n"); - iRet = RS_RET_ERR; + ABORT_FINALIZE(RS_RET_ERR); } ENDObjClassInit(wti) diff --git a/runtime/wtp.c b/runtime/wtp.c index 2ca1a1fb8..34c848154 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -525,7 +525,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg) pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ diff --git a/template.c b/template.c index eb7807497..893dfad47 100644 --- a/template.c +++ b/template.c @@ -308,6 +308,12 @@ tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr, struct syslogTime finalize_it: *ppArr = (iRet == RS_RET_OK) ? pArr : NULL; + if(iRet == RS_RET_OK) { + *ppArr = pArr; + } else { + *ppArr = NULL; + free(pArr); + } RETiRet; } @@ -330,7 +336,8 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson, struct DEFiRet; if(pTpl->bHaveSubtree){ - localRet = jsonFind(pMsg->json, &pTpl->subtree, pjson); + if(jsonFind(pMsg->json, &pTpl->subtree, pjson) != RS_RET_OK) + *pjson = NULL; if(*pjson == NULL) { /* we need to have a root object! */ *pjson = json_object_new_object(); @@ -681,7 +688,7 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) while(*p && *p != '%' && *p != ':') { /* outer loop - until end of options */ i = 0; - while((i < sizeof(Buf) / sizeof(char)) && + while((i < sizeof(Buf)-1) && *p && *p != '%' && *p != ':' && *p != ',') { /* inner loop - until end of ONE option */ Buf[i++] = tolower((int)*p); @@ -1225,7 +1232,6 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin struct template *pTpl; unsigned char *p; int bDone; - char optBuf[128]; /* buffer for options - should be more than enough... */ size_t i; rsRetVal localRet; @@ -1236,7 +1242,7 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin DBGPRINTF("tplAddLine processing template '%s'\n", pName); pTpl->iLenName = strlen(pName); - pTpl->pszName = (char*) MALLOC(sizeof(char) * (pTpl->iLenName + 1)); + pTpl->pszName = (char*) MALLOC(pTpl->iLenName + 1); if(pTpl->pszName == NULL) { dbgprintf("tplAddLine could not alloc memory for template name!"); pTpl->iLenName = 0; @@ -1325,8 +1331,9 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin ++p; /* read option word */ + char optBuf[128] = { '\0' }; /* buffer for options - should be more than enough... */ i = 0; - while(i < sizeof(optBuf) / sizeof(char) - 1 + while((i < (sizeof(optBuf) - 1)) && *p && *p != '=' && *p !=',' && *p != '\n') { optBuf[i++] = tolower((int)*p); ++p; @@ -1868,6 +1875,15 @@ tplProcessCnf(struct cnfobj *o) } } + /* the following check is just for clang static anaylzer: this condition + * cannot occur if all is setup well, because "name" is a required parameter + * inside the param block and so the code should err out above. + */ + if(name == NULL) { + DBGPRINTF("template/tplProcessConf: logic error name == NULL - pblk wrong?\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + /* do config sanity checks */ if(tplStr == NULL) { if(tplType == T_STRING) { @@ -1980,6 +1996,7 @@ tplProcessCnf(struct cnfobj *o) finalize_it: free(tplStr); + free(plugin); if(pvals != NULL) cnfparamvalsDestruct(pvals, &pblk); if(iRet != RS_RET_OK) { diff --git a/tests/Makefile.am b/tests/Makefile.am index 3e5d01403..091ec4496 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -12,6 +12,9 @@ TESTS = $(TESTRUNS) TESTS += \ stop-localvar.sh \ stop-msgvar.sh \ + msgvar-concurrency.sh \ + localvar-concurrency.sh \ + exec_tpl-concurrency.sh \ fac_authpriv.sh \ fac_local0.sh \ fac_local7.sh \ @@ -24,6 +27,8 @@ TESTS += \ fac_invld2.sh \ fac_invld3.sh \ fac_invld4_rfc5424.sh \ + now_family_utc.sh \ + rawmsg-after-pri.sh \ rfc5424parser.sh \ tcp_forwarding_tpl.sh \ tcp_forwarding_dflt_tpl.sh \ @@ -40,7 +45,6 @@ TESTS += \ rulesetmultiqueue-v6.sh \ manytcp.sh \ rsf_getenv.sh \ - imtcp_conndrop_tls.sh \ imtcp_conndrop.sh \ imtcp_addtlframedelim.sh \ imtcp_no_octet_counted.sh \ @@ -90,6 +94,7 @@ TESTS += \ dircreate_dflt.sh \ dircreate_off.sh \ imuxsock_logger.sh \ + imuxsock_logger_ruleset.sh \ imuxsock_logger_err.sh \ imuxsock_logger_parserchain.sh \ imuxsock_traillf.sh \ @@ -122,6 +127,18 @@ TESTS += \ rscript_wrap3.sh \ rscript_re_extract.sh \ rscript_re_match.sh \ + rscript_eq.sh \ + rscript_eq_var.sh \ + rscript_ge.sh \ + rscript_ge_var.sh \ + rscript_gt.sh \ + rscript_gt_var.sh \ + rscript_le.sh \ + rscript_le_var.sh \ + rscript_lt.sh \ + rscript_lt_var.sh \ + rscript_ne.sh \ + rscript_ne_var.sh \ rs_optimizer_pri.sh \ cee_simple.sh \ cee_diskqueue.sh \ @@ -187,6 +204,21 @@ endif endif endif +if ENABLE_IMPSTATS +TESTS += \ + dynstats.sh \ + dynstats_overflow.sh \ + dynstats_reset.sh \ + dynstats_ctr_reset.sh \ + dynstats_nometric.sh +if HAVE_VALGRIND +TESTS += \ + dynstats-vg.sh \ + dynstats_reset-vg.sh \ + dynstats_overflow-vg.sh +endif +endif + if ENABLE_IMPTCP # note that some tests simply USE imptcp, but they also # need to be disabled if we do not have this module @@ -197,6 +229,8 @@ TESTS += \ imptcp_conndrop.sh \ imptcp_no_octet_counted.sh \ imptcp_spframingfix.sh \ + imptcp_nonProcessingPoller.sh \ + imptcp_veryLargeOctateCountedMessages.sh \ rscript_random.sh \ rscript_replace.sh if HAVE_VALGRIND @@ -235,11 +269,10 @@ endif if ENABLE_MMNORMALIZE if ENABLE_IMPTCP TESTS += \ - mmnormalize_variable.sh \ - mmnormalize_tokenized.sh \ mmnormalize_regex_defaulted.sh \ - mmnormalize_regex_disabled.sh - + mmnormalize_regex_disabled.sh \ + mmnormalize_variable.sh \ + mmnormalize_tokenized.sh endif if LOGNORM_REGEX_SUPPORTED TESTS += \ @@ -256,8 +289,16 @@ TESTS += \ json_array_looping.sh \ json_nonarray_looping.sh endif +if HAVE_VALGRIND TESTS += \ - stop_when_array_has_element.sh + json_null_array-vg.sh \ + json_null-vg.sh +endif +TESTS += \ + stop_when_array_has_element.sh \ + json_null_array.sh \ + json_null.sh \ + json_var_cmpr.sh endif if ENABLE_GNUTLS @@ -461,16 +502,28 @@ EXTRA_DIST= \ testsuites/stop.conf \ rscript_le.sh \ testsuites/rscript_le.conf \ + rscript_le_var.sh \ + testsuites/rscript_le_var.conf \ rscript_ge.sh \ testsuites/rscript_ge.conf \ + rscript_ge_var.sh \ + testsuites/rscript_ge_var.conf \ rscript_lt.sh \ testsuites/rscript_lt.conf \ + rscript_lt_var.sh \ + testsuites/rscript_lt_var.conf \ rscript_gt.sh \ testsuites/rscript_gt.conf \ + rscript_gt_var.sh \ + testsuites/rscript_gt_var.conf \ rscript_ne.sh \ testsuites/rscript_ne.conf \ + rscript_ne_var.sh \ + testsuites/rscript_ne_var.conf \ rscript_eq.sh \ testsuites/rscript_eq.conf \ + rscript_eq_var.sh \ + testsuites/rscript_eq_var.conf \ rscript_set_modify.sh \ testsuites/rscript_set_modify.conf \ testsuites/rscript_unaffected_reset.conf \ @@ -478,6 +531,12 @@ EXTRA_DIST= \ testsuites/stop-localvar.conf \ stop-msgvar.sh \ testsuites/stop-msgvar.conf \ + msgvar-concurrency.sh \ + testsuites/msgvar-concurrency.conf \ + localvar-concurrency.sh \ + testsuites/localvar-concurrency.conf \ + exec_tpl-concurrency.sh \ + testsuites/exec_tpl-concurrency.conf \ global_vars.sh \ testsuites/global_vars.conf \ rfc5424parser.sh \ @@ -506,6 +565,10 @@ EXTRA_DIST= \ testsuites/fac_invld3.conf \ fac_invld4_rfc5424.sh \ testsuites/fac_invld4_rfc5424.conf \ + now_family_utc.sh \ + testsuites/now_family_utc.conf \ + rawmsg-after-pri.sh \ + testsuites/rawmsg-after-pri.conf \ rs_optimizer_pri.sh \ testsuites/rs_optimizer_pri.conf \ rscript_prifilt.sh \ @@ -594,6 +657,7 @@ EXTRA_DIST= \ imtcp_conndrop.sh \ testsuites/imtcp_conndrop.conf \ imtcp_conndrop_tls.sh \ + testsuites/imtcp_conndrop_tls.conf \ imtcp_conndrop_tls-vg.sh \ testsuites/imtcp_conndrop.conf \ imtcp_addtlframedelim.sh \ @@ -790,6 +854,8 @@ EXTRA_DIST= \ testsuites/imuxsock_logger_parserchain.conf \ imuxsock_logger.sh \ testsuites/imuxsock_logger.conf \ + imuxsock_logger_ruleset.sh \ + testsuites/imuxsock_logger_ruleset.conf \ imuxsock_logger_err.sh \ imuxsock_logger_root.sh \ testsuites/imuxsock_logger_root.conf \ @@ -826,11 +892,32 @@ EXTRA_DIST= \ mysql-actq-mt-withpause-vg.sh \ testsuites/mysql-actq-mt.conf \ mmpstrucdata.sh \ - mmpstrucdata-vg.sh \ + mmpstrucdata-vg.sh \ testsuites/mmpstrucdata.conf \ mmpstrucdata-invalid-vg.sh \ testsuites/mmpstrucdata-invalid.conf \ libdbi-basic-vg.sh \ + dynstats_ctr_reset.sh \ + dynstats_nometric.sh \ + dynstats_overflow.sh \ + dynstats_overflow-vg.sh \ + dynstats_reset.sh \ + dynstats_reset-vg.sh \ + dynstats.sh \ + dynstats-vg.sh \ + testsuites/dynstats.conf \ + testsuites/dynstats_ctr_reset.conf \ + testsuites/dynstats_empty_input \ + testsuites/dynstats_input \ + testsuites/dynstats_input_1 \ + testsuites/dynstats_input_2 \ + testsuites/dynstats_input_3 \ + testsuites/dynstats_input_more_0 \ + testsuites/dynstats_input_more_1 \ + testsuites/dynstats_input_more_2 \ + testsuites/dynstats_nometric.conf \ + testsuites/dynstats_overflow.conf \ + testsuites/dynstats_reset.conf \ mmnormalize_variable.sh \ mmnormalize_tokenized.sh \ testsuites/mmnormalize_variable.conf \ @@ -860,6 +947,12 @@ EXTRA_DIST= \ json_array_looping.sh \ json_nonarray_looping.sh \ testsuites/json_array_looping.conf \ + json_null.sh \ + json_null-vg.sh \ + testsuites/json_null.conf \ + json_null_array.sh \ + json_null_array-vg.sh \ + testsuites/json_null_array.conf \ mmnormalize_regex.sh \ testsuites/mmnormalize_regex.conf \ testsuites/mmnormalize_regex.rulebase \ @@ -927,6 +1020,12 @@ EXTRA_DIST= \ testsuites/xlate_more_with_duplicates_and_nomatch.lkp_tbl \ testsuites/xlate_sparse_array_more_with_duplicates_and_nomatch.lkp_tbl \ testsuites/multiple_lookup_tables.conf + json_var_cmpr.sh \ + testsuites/json_var_cmpr.conf \ + imptcp_nonProcessingPoller.sh \ + imptcp_veryLargeOctateCountedMessages.sh \ + testsuites/imptcp_nonProcessingPoller.conf \ + travis/trusty.supp \ cfg.sh # TODO: re-enable diff --git a/tests/chkseq.c b/tests/chkseq.c index 596d8dca2..a8976b0a3 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -11,6 +11,8 @@ * failure. This is necessary for some failover tests, where it is * impossible to totally guard against messagt loss. By default, NO * message is permitted to be lost. + * -T anticipate truncation (which means specified payload length may be + * more than actual payload (which may have been truncated) * * Part of the testbench for rsyslog. * @@ -48,6 +50,7 @@ int main(int argc, char *argv[]) int scanfOK; int verbose = 0; int bHaveExtraData = 0; + int bAnticipateTruncation = 0; int dupsPermitted = 0; int start = 0, end = 0; int opt; @@ -59,29 +62,32 @@ int main(int argc, char *argv[]) static char ioBuf[sizeof(edBuf)+1024]; char *file = NULL; - while((opt = getopt(argc, argv, "e:f:ds:vm:E")) != EOF) { + while((opt = getopt(argc, argv, "e:f:ds:vm:ET")) != EOF) { switch((char)opt) { case 'f': file = optarg; break; - case 'd': + case 'd': dupsPermitted = 1; break; - case 'e': + case 'e': end = atoi(optarg); break; - case 's': + case 's': start = atoi(optarg); break; - case 'v': + case 'v': ++verbose; break; - case 'm': + case 'm': lostok = atoi(optarg); break; - case 'E': + case 'E': bHaveExtraData = 1; break; + case 'T': + bAnticipateTruncation = 1; + break; default:printf("Invalid call of chkseq, optchar='%c'\n", opt); printf("Usage: chkseq file -sstart -eend -d -E\n"); exit(1); @@ -118,9 +124,18 @@ int main(int argc, char *argv[]) scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; } if(edLen != (int) strlen(edBuf)) { - printf("extra data length specified %d, but actually is %ld in record %d\n", - edLen, (long) strlen(edBuf), i); - exit(1); + if (bAnticipateTruncation == 1) { + if (edLen < strlen(edBuf)) { + printf("extra data length specified %d, but actually is %ld in record %d" + " (truncation was anticipated, but payload should have been smaller than data-length, not larger)\n", + edLen, (long) strlen(edBuf), i); + exit(1); + } + } else { + printf("extra data length specified %d, but actually is %ld in record %d\n", + edLen, (long) strlen(edBuf), i); + exit(1); + } } } else { if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { @@ -172,9 +187,18 @@ int main(int argc, char *argv[]) scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; } if(edLen != (int) strlen(edBuf)) { - printf("extra data length specified %d, but actually is %ld in record %d\n", - edLen, (long) strlen(edBuf), i); - exit(1); + if (bAnticipateTruncation == 1) { + if (edLen < strlen(edBuf)) { + printf("extra data length specified %d, but actually is %ld in record %d" + " (truncation was anticipated, but payload should have been smaller than data-length, not larger)\n", + edLen, (long) strlen(edBuf), i); + exit(1); + } + } else { + printf("extra data length specified %d, but actually is %ld in record %d\n", + edLen, (long) strlen(edBuf), i); + exit(1); + } } } else { if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { diff --git a/tests/diag.sh b/tests/diag.sh index 2bb233e2e..31355bab1 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -100,12 +100,29 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup, $3 is the instance (blank or 2!) + if [ ! -f $srcdir/testsuites/$2 ]; then + echo "ERROR: config file '$srcdir/testsuites/$2' not found!" + exit 1 + fi $valgrind ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & . $srcdir/diag.sh wait-startup $3 ;; + 'startup-silent') # start rsyslogd with default params. $2 is the config file name to use + # returns only after successful startup, $3 is the instance (blank or 2!) + if [ ! -f $srcdir/testsuites/$2 ]; then + echo "ERROR: config file '$srcdir/testsuites/$2' not found!" + exit 1 + fi + $valgrind ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 2>/dev/null & + . $srcdir/diag.sh wait-startup $3 + ;; 'startup-vg') # start rsyslogd with default params under valgrind control. $2 is the config file name to use # returns only after successful startup, $3 is the instance (blank or 2!) - valgrind --log-fd=1 --error-exitcode=10 --malloc-fill=ff --free-fill=fe --leak-check=full ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + if [ ! -f $srcdir/testsuites/$2 ]; then + echo "ERROR: config file '$srcdir/testsuites/$2' not found!" + exit 1 + fi + valgrind $RS_TESTBENCH_VALGRIND_EXTRA_OPTS --log-fd=1 --error-exitcode=10 --malloc-fill=ff --free-fill=fe --leak-check=full ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & . $srcdir/diag.sh wait-startup $3 echo startup-vg still running ;; @@ -114,10 +131,18 @@ case $1 in # that) we don't can influence and where we cannot provide suppressions as # they are platform-dependent. In that case, we can't test for leak checks # (obviously), but we can check for access violations, what still is useful. - valgrind --log-fd=1 --error-exitcode=10 --malloc-fill=ff --free-fill=fe --leak-check=no ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + if [ ! -f $srcdir/testsuites/$2 ]; then + echo "ERROR: config file '$srcdir/testsuites/$2' not found!" + exit 1 + fi + valgrind $RS_TESTBENCH_VALGRIND_EXTRA_OPTS --log-fd=1 --error-exitcode=10 --malloc-fill=ff --free-fill=fe --leak-check=no ../tools/rsyslogd -C -n -irsyslog$3.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & . $srcdir/diag.sh wait-startup $3 echo startup-vg still running ;; + 'msleep') + $srcdir/msleep $2 + ;; + 'wait-startup') # wait for rsyslogd startup ($2 is the instance) i=0 while test ! -f rsyslog$2.pid; do @@ -235,6 +260,11 @@ case $1 in echo injectmsg $2 $3 $4 $5 | ./diagtalker || . $srcdir/diag.sh error-exit $? # TODO: some return state checking? (does it really make sense here?) ;; + 'injectmsg-litteral') # inject litteral-payload via our inject interface (imdiag) + echo injecting msg payload from: $2 + cat $2 | sed -e 's/^/injectmsg litteral /g' | ./diagtalker || . $srcdir/diag.sh error-exit $? + # TODO: some return state checking? (does it really make sense here?) + ;; 'check-mainq-spool') # check if mainqueue spool files exist, if not abort (we just check .qi). echo There must exist some files now: ls -l test-spool @@ -273,7 +303,8 @@ case $1 in 'content-check') cat rsyslog.out.log | grep -qF "$2" if [ "$?" -ne "0" ]; then - echo content-check failed + echo content-check failed to find "'$2'", content is + cat rsyslog.out.log . $srcdir/diag.sh error-exit 1 fi ;; @@ -284,6 +315,19 @@ case $1 in . $srcdir/diag.sh error-exit 1 fi ;; + 'wait-for-stats-flush') + echo "will wait for stats push" + while [[ ! -f $2 ]]; do + echo waiting for stats file "'$2'" to be created + ./msleep 100 + done + prev_count=$(cat $2 | grep 'BEGIN$' | wc -l) + new_count=$prev_count + while [[ "x$prev_count" == "x$new_count" ]]; do + new_count=$(cat $2 | grep 'BEGIN$' | wc -l) # busy spin, because it allows as close timing-coordination in actual test run as possible + done + echo "stats push registered" + ;; 'custom-content-check') cat $3 | grep -qF "$2" if [ "$?" -ne "0" ]; then @@ -291,6 +335,20 @@ case $1 in . $srcdir/diag.sh error-exit 1 fi ;; + 'first-column-sum-check') + sum=$(cat $4 | grep $3 | sed -e $2 | awk '{s+=$1} END {print s}') + if [ "x${sum}" != "x$5" ]; then + echo sum of first column with edit-expr "'$2'" run over lines from file "'$4'" matched by "'$3'" equals "'$sum'" which is not equal to expected value of "'$5'" + . $srcdir/diag.sh error-exit 1 + fi + ;; + 'assert-first-column-sum-greater-than') + sum=$(cat $4 | grep $3 | sed -e $2 | awk '{s+=$1} END {print s}') + if [ ! $sum -gt $5 ]; then + echo sum of first column with edit-expr "'$2'" run over lines from file "'$4'" matched by "'$3'" equals "'$sum'" which is smaller than expected lower-limit of "'$5'" + . $srcdir/diag.sh error-exit 1 + fi + ;; 'content-pattern-check') cat rsyslog.out.log | grep -q "$2" if [ "$?" -ne "0" ]; then @@ -301,7 +359,14 @@ case $1 in 'assert-content-missing') cat rsyslog.out.log | grep -qF "$2" if [ "$?" -eq "0" ]; then - echo content-missing assertion failed + echo content-missing assertion failed, some line matched pattern "'$2'" + . $srcdir/diag.sh error-exit 1 + fi + ;; + 'custom-assert-content-missing') + cat $3 | grep -qF "$2" + if [ "$?" -eq "0" ]; then + echo content-missing assertion failed, some line in "'$3'" matched pattern "'$2'" . $srcdir/diag.sh error-exit 1 fi ;; diff --git a/tests/dynstats-vg.sh b/tests/dynstats-vg.sh new file mode 100755 index 000000000..06684ac18 --- /dev/null +++ b/tests/dynstats-vg.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# added 2015-11-13 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats-vg.sh\]: test for gathering stats over dynamic metric names with valgrind +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-vg dynstats.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +. $srcdir/diag.sh msleep 1100 # wait for stats flush +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh check-exit-vg +. $srcdir/diag.sh custom-content-check 'bar=1' 'rsyslog.out.stats.log' +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh exit diff --git a/tests/dynstats.sh b/tests/dynstats.sh new file mode 100755 index 000000000..2564a4bfd --- /dev/null +++ b/tests/dynstats.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# added 2015-11-10 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats.sh\]: test for gathering stats over dynamic metric names +. $srcdir/diag.sh init +. $srcdir/diag.sh startup dynstats.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +. $srcdir/diag.sh msleep 1100 # wait for stats flush +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh custom-content-check 'bar=1' 'rsyslog.out.stats.log' +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh exit diff --git a/tests/dynstats_ctr_reset.sh b/tests/dynstats_ctr_reset.sh new file mode 100755 index 000000000..43e5959f4 --- /dev/null +++ b/tests/dynstats_ctr_reset.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# added 2015-11-16 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_ctr_reset.sh\]: test to ensure correctness of stats-ctr reset +. $srcdir/diag.sh init +. $srcdir/diag.sh startup dynstats_ctr_reset.conf +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_1 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_2 +. $srcdir/diag.sh wait-queueempty +sleep 1 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_3 +. $srcdir/diag.sh wait-queueempty +sleep 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh content-check "foo 006" +. $srcdir/diag.sh custom-content-check 'bar=1' 'rsyslog.out.stats.log' +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'msg_stats_resettable_on.\+foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'msg_stats_resettable_on.\+bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'msg_stats_resettable_on.\+baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh assert-first-column-sum-greater-than 's/.*foo=\([0-9]\+\)/\1/g' 'msg_stats_resettable_off.\+foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh assert-first-column-sum-greater-than 's/.*bar=\([0-9]\+\)/\1/g' 'msg_stats_resettable_off.\+bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh assert-first-column-sum-greater-than 's/.*baz=\([0-9]\+\)/\1/g' 'msg_stats_resettable_off.\+baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'msg_stats_resettable_default.\+foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'msg_stats_resettable_default.\+bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'msg_stats_resettable_default.\+baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh exit diff --git a/tests/dynstats_nometric.sh b/tests/dynstats_nometric.sh new file mode 100755 index 000000000..6c17f912c --- /dev/null +++ b/tests/dynstats_nometric.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# added 2015-11-17 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_nometric.sh\]: test for dyn-stats meta-metric behavior with zero-length metric name +. $srcdir/diag.sh init +. $srcdir/diag.sh startup dynstats_nometric.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh wait-queueempty +rm $srcdir/rsyslog.out.stats.log +. $srcdir/diag.sh issue-HUP #reopen stats file +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_empty_input +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh msleep 1100 # wait for stats flush +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh first-column-sum-check 's/.*no_metric=\([0-9]\+\)/\1/g' 'no_metric=' 'rsyslog.out.stats.log' 5 +. $srcdir/diag.sh custom-assert-content-missing 'foo' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'bar' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'baz' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'corge' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-content-check 'quux=1' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-content-check 'grault=1' 'rsyslog.out.stats.log' +. $srcdir/diag.sh exit diff --git a/tests/dynstats_overflow-vg.sh b/tests/dynstats_overflow-vg.sh new file mode 100755 index 000000000..8fb3ce376 --- /dev/null +++ b/tests/dynstats_overflow-vg.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# added 2015-11-13 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_overflow-vg.sh\]: test for gathering stats when metrics exceed provisioned capacity +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-vg dynstats_overflow.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_0 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh msleep 800 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_1 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh msleep 1300 #sleep above + this = 2 seconds, so metric-names reset should have happened +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 5 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh custom-assert-content-missing 'quux' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'corge' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'grault' 'rsyslog.out.stats.log' +rm $srcdir/rsyslog.out.stats.log +. $srcdir/diag.sh issue-HUP #reopen stats file +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_2 +. $srcdir/diag.sh msleep 2100 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh check-exit-vg +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +. $srcdir/diag.sh content-check "quux 007 -6" +. $srcdir/diag.sh content-check "corge 008 -6" +. $srcdir/diag.sh content-check "quux 009 -6" +. $srcdir/diag.sh content-check "foo 010 0" +. $srcdir/diag.sh content-check "corge 011 -6" +. $srcdir/diag.sh content-check "grault 012 -6" +. $srcdir/diag.sh content-check "foo 013 0" +. $srcdir/diag.sh content-check "corge 014 0" +. $srcdir/diag.sh content-check "grault 015 0" +. $srcdir/diag.sh content-check "quux 016 0" +. $srcdir/diag.sh content-check "foo 017 -6" +. $srcdir/diag.sh content-check "corge 018 0" + +. $srcdir/diag.sh first-column-sum-check 's/.*corge=\([0-9]\+\)/\1/g' 'corge=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh first-column-sum-check 's/.*grault=\([0-9]\+\)/\1/g' 'grault=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*quux=\([0-9]\+\)/\1/g' 'quux=' 'rsyslog.out.stats.log' 1 + +. $srcdir/diag.sh custom-assert-content-missing 'foo' 'rsyslog.out.stats.log' +. $srcdir/diag.sh exit diff --git a/tests/dynstats_overflow.sh b/tests/dynstats_overflow.sh new file mode 100755 index 000000000..30a7e95c4 --- /dev/null +++ b/tests/dynstats_overflow.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# added 2015-11-13 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_overflow.sh\]: test for gathering stats when metrics exceed provisioned capacity +. $srcdir/diag.sh init +. $srcdir/diag.sh startup dynstats_overflow.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_0 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh msleep 800 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_1 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh msleep 1300 #sleep above + this = 2 seconds, so metric-names reset should have happened +. $srcdir/diag.sh wait-queueempty + +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 5 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 + +. $srcdir/diag.sh custom-assert-content-missing 'quux' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'corge' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'grault' 'rsyslog.out.stats.log' + +. $srcdir/diag.sh first-column-sum-check 's/.*new_metric_add=\([0-9]\+\)/\1/g' 'new_metric_add=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*ops_overflow=\([0-9]\+\)/\1/g' 'ops_overflow=' 'rsyslog.out.stats.log' 5 +. $srcdir/diag.sh first-column-sum-check 's/.*no_metric=\([0-9]\+\)/\1/g' 'no_metric=' 'rsyslog.out.stats.log' 0 +. $srcdir/diag.sh first-column-sum-check 's/.*metrics_purged=\([0-9]\+\)/\1/g' 'metrics_purged=' 'rsyslog.out.stats.log' 3 + +rm $srcdir/rsyslog.out.stats.log +. $srcdir/diag.sh issue-HUP #reopen stats file +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_more_2 +. $srcdir/diag.sh msleep 2100 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +. $srcdir/diag.sh content-check "quux 007 -6" +. $srcdir/diag.sh content-check "corge 008 -6" +. $srcdir/diag.sh content-check "quux 009 -6" +. $srcdir/diag.sh content-check "foo 010 0" +. $srcdir/diag.sh content-check "corge 011 -6" +. $srcdir/diag.sh content-check "grault 012 -6" +. $srcdir/diag.sh content-check "foo 013 0" +. $srcdir/diag.sh content-check "corge 014 0" +. $srcdir/diag.sh content-check "grault 015 0" +. $srcdir/diag.sh content-check "quux 016 0" +. $srcdir/diag.sh content-check "foo 017 -6" +. $srcdir/diag.sh content-check "corge 018 0" + +. $srcdir/diag.sh first-column-sum-check 's/.*corge=\([0-9]\+\)/\1/g' 'corge=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh first-column-sum-check 's/.*grault=\([0-9]\+\)/\1/g' 'grault=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*quux=\([0-9]\+\)/\1/g' 'quux=' 'rsyslog.out.stats.log' 1 + +. $srcdir/diag.sh first-column-sum-check 's/.*new_metric_add=\([0-9]\+\)/\1/g' 'new_metric_add=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*ops_overflow=\([0-9]\+\)/\1/g' 'ops_overflow=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*no_metric=\([0-9]\+\)/\1/g' 'no_metric=' 'rsyslog.out.stats.log' 0 +. $srcdir/diag.sh first-column-sum-check 's/.*metrics_purged=\([0-9]\+\)/\1/g' 'metrics_purged=' 'rsyslog.out.stats.log' 3 + +. $srcdir/diag.sh custom-assert-content-missing 'foo' 'rsyslog.out.stats.log' +. $srcdir/diag.sh exit diff --git a/tests/dynstats_reset-vg.sh b/tests/dynstats_reset-vg.sh new file mode 100755 index 000000000..9303afbee --- /dev/null +++ b/tests/dynstats_reset-vg.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# added 2015-11-13 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_reset-vg.sh\]: test for gathering stats with a known-dyn-metrics reset in-between +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-vg dynstats_reset.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_1 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_2 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_3 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh check-exit-vg + # because dyn-metrics would be reset before it can accumulate and report high counts, sleep between msg-injection ensures that +. $srcdir/diag.sh custom-assert-content-missing 'baz=2' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'foo=2' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'foo=3' 'rsyslog.out.stats.log' +# but actual reported stats (aggregate) should match +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh exit diff --git a/tests/dynstats_reset.sh b/tests/dynstats_reset.sh new file mode 100755 index 000000000..708b4d7c5 --- /dev/null +++ b/tests/dynstats_reset.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# added 2015-11-13 by singh.janmejay +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[dynstats_reset.sh\]: test for gathering stats with a known-dyn-metrics reset inbetween +. $srcdir/diag.sh init +. $srcdir/diag.sh startup dynstats_reset.conf +. $srcdir/diag.sh wait-for-stats-flush 'rsyslog.out.stats.log' +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_1 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_2 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh injectmsg-litteral $srcdir/testsuites/dynstats_input_3 +. $srcdir/diag.sh msleep 1100 +. $srcdir/diag.sh wait-queueempty +. $srcdir/diag.sh content-check "foo 001 0" +. $srcdir/diag.sh content-check "bar 002 0" +. $srcdir/diag.sh content-check "baz 003 0" +. $srcdir/diag.sh content-check "foo 004 0" +. $srcdir/diag.sh content-check "baz 005 0" +. $srcdir/diag.sh content-check "foo 006 0" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown + # because dyn-metrics would be reset before it can accumulate and report high counts, sleep between msg-injection ensures that +. $srcdir/diag.sh custom-assert-content-missing 'baz=2' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'foo=2' 'rsyslog.out.stats.log' +. $srcdir/diag.sh custom-assert-content-missing 'foo=3' 'rsyslog.out.stats.log' +# but actual reported stats (aggregate) should match +. $srcdir/diag.sh first-column-sum-check 's/.*foo=\([0-9]\+\)/\1/g' 'foo=' 'rsyslog.out.stats.log' 3 +. $srcdir/diag.sh first-column-sum-check 's/.*bar=\([0-9]\+\)/\1/g' 'bar=' 'rsyslog.out.stats.log' 1 +. $srcdir/diag.sh first-column-sum-check 's/.*baz=\([0-9]\+\)/\1/g' 'baz=' 'rsyslog.out.stats.log' 2 +. $srcdir/diag.sh first-column-sum-check 's/.*new_metric_add=\([0-9]\+\)/\1/g' 'new_metric_add=' 'rsyslog.out.stats.log' 6 +. $srcdir/diag.sh first-column-sum-check 's/.*ops_overflow=\([0-9]\+\)/\1/g' 'ops_overflow=' 'rsyslog.out.stats.log' 0 +. $srcdir/diag.sh first-column-sum-check 's/.*no_metric=\([0-9]\+\)/\1/g' 'no_metric=' 'rsyslog.out.stats.log' 0 +. $srcdir/diag.sh first-column-sum-check 's/.*metrics_purged=\([0-9]\+\)/\1/g' 'metrics_purged=' 'rsyslog.out.stats.log' 6 +. $srcdir/diag.sh exit diff --git a/tests/exec_tpl-concurrency.sh b/tests/exec_tpl-concurrency.sh new file mode 100755 index 000000000..40b1810a2 --- /dev/null +++ b/tests/exec_tpl-concurrency.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Test concurrency of exec_template function with msg variables +# Added 2015-12-11 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[exec_tpl-concurrency.sh\]: testing concurrency of exec_template w variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup exec_tpl-concurrency.conf +sleep 1 +. $srcdir/diag.sh tcpflood -m500000 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 499999 +. $srcdir/diag.sh exit diff --git a/tests/imfile-endregex.sh b/tests/imfile-endregex.sh index 8985a3e96..c9cb3f5d3 100755 --- a/tests/imfile-endregex.sh +++ b/tests/imfile-endregex.sh @@ -3,7 +3,12 @@ # This test mimics the test imfile-readmode2.sh, but works via # endmsg.regex. It's kind of a base test for the regex functionality. echo ====================================================================== -echo [imfile-endregex.sh] +# Check if inotify header exist +if [ -n "$(find /usr/include -name 'inotify.h' -print -quit)" ]; then + echo [imfile-endregex.sh] +else + exit 77 # no inotify available, skip this test +fi . $srcdir/diag.sh init . $srcdir/diag.sh startup imfile-endregex.conf diff --git a/tests/imfile-readmode2-with-persists-data-during-stop.sh b/tests/imfile-readmode2-with-persists-data-during-stop.sh index 7f75bebad..99cdf91f8 100755 --- a/tests/imfile-readmode2-with-persists-data-during-stop.sh +++ b/tests/imfile-readmode2-with-persists-data-during-stop.sh @@ -1,7 +1,12 @@ #!/bin/bash # This is part of the rsyslog testbench, licensed under ASL 2.0 echo ====================================================================== -echo [imfile-readmode2-with-persists-data-during-stop.sh] +# Check if inotify header exist +if [ -n "$(find /usr/include -name 'inotify.h' -print -quit)" ]; then + echo [imfile-readmode2-with-persists-data-during-stop.sh] +else + exit 77 # no inotify available, skip this test +fi . $srcdir/diag.sh init . $srcdir/diag.sh startup imfile-readmode2-with-persists-data-during-stop.conf diff --git a/tests/imfile-readmode2-with-persists.sh b/tests/imfile-readmode2-with-persists.sh index 6cad060fd..88fd92e98 100755 --- a/tests/imfile-readmode2-with-persists.sh +++ b/tests/imfile-readmode2-with-persists.sh @@ -1,7 +1,12 @@ #!/bin/bash # This is part of the rsyslog testbench, licensed under ASL 2.0 echo ====================================================================== -echo [imfile-readmode2-with-persists.sh] +# Check if inotify header exist +if [ -n "$(find /usr/include -name 'inotify.h' -print -quit)" ]; then + echo [imfile-readmode2-with-persists.sh] +else + exit 77 # no inotify available, skip this test +fi . $srcdir/diag.sh init . $srcdir/diag.sh startup imfile-readmode2-with-persists.conf diff --git a/tests/imfile-readmode2.sh b/tests/imfile-readmode2.sh index 30ece5f60..2f403f910 100755 --- a/tests/imfile-readmode2.sh +++ b/tests/imfile-readmode2.sh @@ -1,7 +1,12 @@ #!/bin/bash # This is part of the rsyslog testbench, licensed under ASL 2.0 echo ====================================================================== -echo [imfile-readmode2.sh] +# Check if inotify header exist +if [ -n "$(find /usr/include -name 'inotify.h' -print -quit)" ]; then + echo [imfile-readmode2.sh] +else + exit 77 # no inotify available, skip this test +fi . $srcdir/diag.sh init . $srcdir/diag.sh startup imfile-readmode2.conf diff --git a/tests/imptcp_conndrop-vg.sh b/tests/imptcp_conndrop-vg.sh index b8c1ea918..02649fc43 100755 --- a/tests/imptcp_conndrop-vg.sh +++ b/tests/imptcp_conndrop-vg.sh @@ -7,7 +7,7 @@ echo TEST: \[imptcp_conndrop-vg.sh\]: test imptcp with random connection drops . $srcdir/diag.sh startup-vg imptcp_conndrop.conf # 100 byte messages to gain more practical data use . $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 4 # due to large messages, we need this time for the tcp receiver to settle... +sleep 10 # due to large messages, we need this time for the tcp receiver to settle... . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages . $srcdir/diag.sh wait-shutdown-vg # and wait for it to terminate . $srcdir/diag.sh check-exit-vg diff --git a/tests/imptcp_conndrop.sh b/tests/imptcp_conndrop.sh index 0c918bfae..953c4049a 100755 --- a/tests/imptcp_conndrop.sh +++ b/tests/imptcp_conndrop.sh @@ -9,7 +9,7 @@ echo TEST: \[imptcp_conndrop.sh\]: test imptcp with random connection drops . $srcdir/diag.sh startup imptcp_conndrop.conf # 100 byte messages to gain more practical data use . $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 4 # due to large messages, we need this time for the tcp receiver to settle... +sleep 10 # due to large messages, we need this time for the tcp receiver to settle... . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages . $srcdir/diag.sh wait-shutdown # and wait for it to terminate . $srcdir/diag.sh seq-check 0 49999 -E diff --git a/tests/imptcp_nonProcessingPoller.sh b/tests/imptcp_nonProcessingPoller.sh new file mode 100755 index 000000000..540da3765 --- /dev/null +++ b/tests/imptcp_nonProcessingPoller.sh @@ -0,0 +1,13 @@ +# Test imptcp with poller not processing any messages +# added 2015-10-16 by singh.janmejay +# This file is part of the rsyslog project, released under GPLv3 +echo ==================================================================================== +echo TEST: \[imptcp_nonProcessingPoller.sh\]: test imptcp with poller driven processing disabled +. $srcdir/diag.sh init +. $srcdir/diag.sh startup imptcp_nonProcessingPoller.conf +. $srcdir/diag.sh tcpflood -c1 -m20000 -r -d10000 -P129 -O +sleep 2 # due to large messages, we need this time for the tcp receiver to settle... +. $srcdir/diag.sh shutdown-when-empty +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 19999 -E +. $srcdir/diag.sh exit diff --git a/tests/imptcp_veryLargeOctateCountedMessages.sh b/tests/imptcp_veryLargeOctateCountedMessages.sh new file mode 100755 index 000000000..d0c5b0179 --- /dev/null +++ b/tests/imptcp_veryLargeOctateCountedMessages.sh @@ -0,0 +1,13 @@ +# Test imptcp with poller not processing any messages +# added 2015-10-17 by singh.janmejay +# This file is part of the rsyslog project, released under GPLv3 +echo ==================================================================================== +echo TEST: \[imptcp_veryLargeOctateCountedMessages.sh\]: test imptcp with very large messages while poller driven processing is disabled +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-silent imptcp_nonProcessingPoller.conf +. $srcdir/diag.sh tcpflood -c1 -m20000 -r -d100000 -P129 -O +sleep 2 # due to large messages, we need this time for the tcp receiver to settle... +. $srcdir/diag.sh shutdown-when-empty +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 19999 -E -T +. $srcdir/diag.sh exit diff --git a/tests/imtcp-tls-basic-vg.sh b/tests/imtcp-tls-basic-vg.sh index fcf894b52..998ae6e99 100755 --- a/tests/imtcp-tls-basic-vg.sh +++ b/tests/imtcp-tls-basic-vg.sh @@ -7,7 +7,7 @@ echo \[imtcp-tls-basic-vg.sh\]: testing imtcp in TLS mode - basic test echo \$DefaultNetstreamDriverCAFile $srcdir/tls-certs/ca.pem >rsyslog.conf.tlscert echo \$DefaultNetstreamDriverCertFile $srcdir/tls-certs/cert.pem >>rsyslog.conf.tlscert echo \$DefaultNetstreamDriverKeyFile $srcdir/tls-certs/key.pem >>rsyslog.conf.tlscert -. $srcdir/diag.sh startup-vg imtcp-tls-basic.conf +. $srcdir/diag.sh startup-vg-noleak imtcp-tls-basic.conf . $srcdir/diag.sh tcpflood -p13514 -m50000 -Ttls -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages . $srcdir/diag.sh wait-shutdown-vg diff --git a/tests/imtcp_conndrop_tls-vg.sh b/tests/imtcp_conndrop_tls-vg.sh index b3d622cbd..9f6973100 100755 --- a/tests/imtcp_conndrop_tls-vg.sh +++ b/tests/imtcp_conndrop_tls-vg.sh @@ -5,14 +5,15 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imtcp_conndrop_tls-vg.sh\]: test imtcp/tls with random connection drops -cat rsyslog.action.1.include . $srcdir/diag.sh init -. $srcdir/diag.sh startup-vg imtcp_conndrop.conf +echo \$DefaultNetstreamDriverCAFile $srcdir/tls-certs/ca.pem >rsyslog.conf.tlscert +echo \$DefaultNetstreamDriverCertFile $srcdir/tls-certs/cert.pem >>rsyslog.conf.tlscert +echo \$DefaultNetstreamDriverKeyFile $srcdir/tls-certs/key.pem >>rsyslog.conf.tlscert +. $srcdir/diag.sh startup-vg imtcp_conndrop_tls.conf # 100 byte messages to gain more practical data use -. $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 10 # due to large messages, we need this time for the tcp receiver to settle... +. $srcdir/diag.sh tcpflood -c20 -p13514 -m10000 -r -d100 -P129 -D -l0.995 -Ttls -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem +sleep 5 # due to large messages, we need this time for the tcp receiver to settle... . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages -. $srcdir/diag.sh wait-shutdown-vg # and wait for it to terminate -. $srcdir/diag.sh check-exit-vg -. $srcdir/diag.sh seq-check 0 49999 -E +. $srcdir/diag.sh wait-shutdown # and wait for it to terminate +. $srcdir/diag.sh seq-check 0 9999 -E . $srcdir/diag.sh exit diff --git a/tests/imtcp_conndrop_tls.sh b/tests/imtcp_conndrop_tls.sh index c19f2c13f..505fc6bce 100755 --- a/tests/imtcp_conndrop_tls.sh +++ b/tests/imtcp_conndrop_tls.sh @@ -5,13 +5,16 @@ # This file is part of the rsyslog project, released under GPLv3 echo ==================================================================================== echo TEST: \[imtcp_conndrop_tls.sh\]: test imtcp/tls with random connection drops -cat rsyslog.action.1.include . $srcdir/diag.sh init -. $srcdir/diag.sh startup imtcp_conndrop.conf +echo \$DefaultNetstreamDriverCAFile $srcdir/tls-certs/ca.pem >rsyslog.conf.tlscert +echo \$DefaultNetstreamDriverCertFile $srcdir/tls-certs/cert.pem >>rsyslog.conf.tlscert +echo \$DefaultNetstreamDriverKeyFile $srcdir/tls-certs/key.pem >>rsyslog.conf.tlscert +. $srcdir/diag.sh startup imtcp_conndrop_tls.conf # 100 byte messages to gain more practical data use -. $srcdir/diag.sh tcpflood -c20 -m50000 -r -d100 -P129 -D -sleep 10 # due to large messages, we need this time for the tcp receiver to settle... +. $srcdir/diag.sh tcpflood -c20 -p13514 -m50000 -r -d100 -P129 -D -l0.995 -Ttls -Z$srcdir/tls-certs/cert.pem -z$srcdir/tls-certs/key.pem +sleep 5 # due to large messages, we need this time for the tcp receiver to settle... . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages . $srcdir/diag.sh wait-shutdown # and wait for it to terminate . $srcdir/diag.sh seq-check 0 49999 -E +# . $srcdir/diag.sh content-check 'XXXXX' # Not really a check if it worked, but in TLS stuff in unfished TLS Packets gets lost, so we can't use seq-check. . $srcdir/diag.sh exit diff --git a/tests/imuxsock_logger_ruleset.sh b/tests/imuxsock_logger_ruleset.sh new file mode 100755 index 000000000..666eb34eb --- /dev/null +++ b/tests/imuxsock_logger_ruleset.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# rgerhards, 2016-02-02 released under ASL 2.0 +echo \[imuxsock_logger_ruleset.sh\]: test imuxsock with ruleset definition +. $srcdir/diag.sh init +. $srcdir/diag.sh startup imuxsock_logger_ruleset.conf +# send a message with trailing LF +logger -d -u testbench_socket test +# the sleep below is needed to prevent too-early termination of rsyslogd +./msleep 100 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished! +cmp rsyslog.out.log $srcdir/resultdata/imuxsock_logger.log + echo \"`cat rsyslog.out.log`\" +if [ ! $? -eq 0 ]; then + echo "imuxsock_logger.sh failed" + echo contents of rsyslog.out.log: + echo \"`cat rsyslog.out.log`\" + exit 1 +fi; +. $srcdir/diag.sh exit diff --git a/tests/json_null-vg.sh b/tests/json_null-vg.sh new file mode 100755 index 000000000..b25d189cf --- /dev/null +++ b/tests/json_null-vg.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# added 2015-11-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +# Note: the aim of this test is to test against misadressing, so we do +# not actually check the output +echo =============================================================================== +echo \[json_null.sh\]: test for json containung \"null\" value +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-vg json_null.conf +. $srcdir/diag.sh tcpflood -m 1 -M "\"<167>Mar 6 16:57:54 172.20.245.8 test: @cee: { \\\"nope\\\": null }\"" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh exit diff --git a/tests/json_null.sh b/tests/json_null.sh new file mode 100755 index 000000000..b51f6e407 --- /dev/null +++ b/tests/json_null.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# added 2015-11-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +# Note: the aim of this test is to test against misadressing, so we do +# not actually check the output +echo =============================================================================== +echo \[json_null.sh\]: test for json containung \"null\" value +. $srcdir/diag.sh init +. $srcdir/diag.sh startup json_null.conf +. $srcdir/diag.sh tcpflood -m 1 -M "\"<167>Mar 6 16:57:54 172.20.245.8 test: @cee: { \\\"nope\\\": null }\"" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh exit diff --git a/tests/json_null_array-vg.sh b/tests/json_null_array-vg.sh new file mode 100755 index 000000000..8cd50c5bb --- /dev/null +++ b/tests/json_null_array-vg.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2015-11-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[json_null_array.sh\]: test for json containung \"null\" value +. $srcdir/diag.sh init +. $srcdir/diag.sh startup-vg json_null_array.conf +. $srcdir/diag.sh tcpflood -m 1 -M "\"<167>Mar 6 16:57:54 172.20.245.8 test: @cee: { \\\"array\\\": [0, 1, null, 2, 3, null, 4] }\"" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh seq-check 0 4 +. $srcdir/diag.sh exit diff --git a/tests/json_null_array.sh b/tests/json_null_array.sh new file mode 100755 index 000000000..4fe4ef707 --- /dev/null +++ b/tests/json_null_array.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2015-11-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[json_null_array.sh\]: test for json containung \"null\" value +. $srcdir/diag.sh init +. $srcdir/diag.sh startup json_null_array.conf +. $srcdir/diag.sh tcpflood -m 1 -M "\"<167>Mar 6 16:57:54 172.20.245.8 test: @cee: { \\\"array\\\": [0, 1, null, 2, 3, null, 4] }\"" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 4 +. $srcdir/diag.sh exit diff --git a/tests/json_var_cmpr.sh b/tests/json_var_cmpr.sh new file mode 100755 index 000000000..fe139fe89 --- /dev/null +++ b/tests/json_var_cmpr.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2015-11-24 by portant +# This file is part of the rsyslog project, released under ASL 2.0 +echo ============================================================================================= +echo \[json_var_case.sh\]: test for referencing local and global variables properly in comparisons +. $srcdir/diag.sh init +. $srcdir/diag.sh startup json_var_cmpr.conf +. $srcdir/diag.sh tcpflood -m 1 -M "\"<167>Nov 6 12:34:56 172.0.0.1 test: @cee: { \\\"val\\\": \\\"abc\\\" }\"" +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh content-check "json prop:abc local prop:def global prop:ghi" +. $srcdir/diag.sh exit diff --git a/tests/localvar-concurrency.sh b/tests/localvar-concurrency.sh new file mode 100755 index 000000000..b05989c35 --- /dev/null +++ b/tests/localvar-concurrency.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Test concurrency of message variables +# Added 2015-11-03 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[localvar-concurrency.sh\]: testing concurrency of local variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup localvar-concurrency.conf +sleep 1 +. $srcdir/diag.sh tcpflood -m500000 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 499999 +. $srcdir/diag.sh exit diff --git a/tests/msgvar-concurrency-array.sh b/tests/msgvar-concurrency-array.sh new file mode 100755 index 000000000..851be1701 --- /dev/null +++ b/tests/msgvar-concurrency-array.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Test concurrency of message variables +# Added 2015-11-03 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +export TCPFLOOD_EXTRA_OPTS="-M'msg:msg: 1:2, 3:4, 5:6, 7:8 b test'" +echo =============================================================================== +echo \[msgvar-concurrency-array.sh\]: testing concurrency of local variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup msgvar-concurrency-array.conf +sleep 1 +. $srcdir/diag.sh tcpflood -m500000 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown +#. $srcdir/diag.sh seq-check 0 499999 +. $srcdir/diag.sh exit diff --git a/tests/msgvar-concurrency.sh b/tests/msgvar-concurrency.sh new file mode 100755 index 000000000..a21f65ebc --- /dev/null +++ b/tests/msgvar-concurrency.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Test concurrency of message variables +# Added 2015-11-03 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[msgvar-concurrency.sh\]: testing concurrency of local variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup msgvar-concurrency.conf +sleep 1 +. $srcdir/diag.sh tcpflood -m500000 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 499999 +. $srcdir/diag.sh exit diff --git a/tests/now_family_utc.sh b/tests/now_family_utc.sh new file mode 100755 index 000000000..e9b86760d --- /dev/null +++ b/tests/now_family_utc.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# test many concurrent tcp connections +# addd 2016-01-12 by RGerhards, released under ASL 2.0 +# requires faketime +echo \[now_family_utc\]: test \$NOW family of system properties +export TZ=TEST+06:30 +faketime '2016-01-01 01:00:00' date +if [ $? -ne 0 ]; then + echo "faketime command missing, skipping test" + exit 77 +fi +. $srcdir/diag.sh init +faketime '2016-01-01 01:00:00' $srcdir/diag.sh startup now_family_utc.conf +# what we send actually is irrelevant, as we just use system properties. +# but we need to send one message in order to gain output! +. $srcdir/diag.sh tcpflood -m1 +. $srcdir/diag.sh shutdown-when-empty +. $srcdir/diag.sh wait-shutdown +echo "01:00,07:30" | cmp rsyslog.out.log +if [ ! $? -eq 0 ]; then + echo "invalid timestamps generated, rsyslog.out.log is:" + cat rsyslog.out.log + exit 1 +fi; +#. $srcdir/diag.sh exit diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh index 1cb7ed432..7dc30b3d8 100755 --- a/tests/queue-persist-drvr.sh +++ b/tests/queue-persist-drvr.sh @@ -16,8 +16,8 @@ echo "*.* :omtesting:sleep 0 1000" > work-delay.conf # inject 5000 msgs, so that we do not hit the high watermark . $srcdir/diag.sh startup queue-persist.conf . $srcdir/diag.sh injectmsg 0 5000 -$srcdir/diag.sh shutdown-immediate -$srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh shutdown-immediate +. $srcdir/diag.sh wait-shutdown . $srcdir/diag.sh check-mainq-spool # restart engine and have rest processed @@ -25,7 +25,7 @@ $srcdir/diag.sh wait-shutdown echo "#" > work-delay.conf . $srcdir/diag.sh startup queue-persist.conf . $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages -./msleep 500 +./msleep 1000 $srcdir/diag.sh wait-shutdown # note: we need to permit duplicate messages, as due to the forced # shutdown some messages may be flagged as "unprocessed" while they diff --git a/tests/rawmsg-after-pri.sh b/tests/rawmsg-after-pri.sh new file mode 100755 index 000000000..2e1776651 --- /dev/null +++ b/tests/rawmsg-after-pri.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# This file is part of the rsyslog project, released under ASL 2.0 +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rawmsg-after-pri.conf +. $srcdir/diag.sh tcpflood -m1 -P 129 +. $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +. $srcdir/diag.sh wait-shutdown # and wait for it to terminate +echo "Mar 1 01:00:00 172.20.245.8 tag msgnum:00000000:" > rsyslog.out.compare +NUMLINES=$(grep -c "^Mar 1 01:00:00 172.20.245.8 tag msgnum:00000000:$" rsyslog.out.log 2>/dev/null) + +if [ -z $NUMLINES ]; then + echo "ERROR: output file seems not to exist" + ls -l rsyslog.out.log + cat rsyslog.out.log + . $srcdir/diag.sh error-exit 1 +else + if [ ! $NUMLINES -eq 1 ]; then + echo "ERROR: output format does not match expectation" + cat rsyslog.out.log + . $srcdir/diag.sh error-exit 1 + fi +fi +. $srcdir/diag.sh exit diff --git a/tests/rscript_eq_var.sh b/tests/rscript_eq_var.sh new file mode 100755 index 000000000..be5dafe36 --- /dev/null +++ b/tests/rscript_eq_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_eq.sh\]: testing rainerscript EQ statement comparing two variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_eq_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/rscript_ge_var.sh b/tests/rscript_ge_var.sh new file mode 100755 index 000000000..5bb753892 --- /dev/null +++ b/tests/rscript_ge_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_ge.sh\]: testing rainerscript GE statement for two JSON variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_ge_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/rscript_gt_var.sh b/tests/rscript_gt_var.sh new file mode 100755 index 000000000..cca33781f --- /dev/null +++ b/tests/rscript_gt_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_gt.sh\]: testing rainerscript GT statement for two JSON variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_gt_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/rscript_le_var.sh b/tests/rscript_le_var.sh new file mode 100755 index 000000000..2e04d3b39 --- /dev/null +++ b/tests/rscript_le_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_le.sh\]: testing rainerscript LE statement for two JSON variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_le_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/rscript_lt_var.sh b/tests/rscript_lt_var.sh new file mode 100755 index 000000000..168cf74d2 --- /dev/null +++ b/tests/rscript_lt_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_lt.sh\]: testing rainerscript LT statement for two JSON variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_lt_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/rscript_ne_var.sh b/tests/rscript_ne_var.sh new file mode 100755 index 000000000..e861b6f68 --- /dev/null +++ b/tests/rscript_ne_var.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# added 2014-01-17 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_ne.sh\]: testing rainerscript NE statement for two JSON variables +. $srcdir/diag.sh init +. $srcdir/diag.sh startup rscript_ne_var.conf +. $srcdir/diag.sh injectmsg 0 1 +echo doing shutdown +. $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +. $srcdir/diag.sh wait-shutdown +. $srcdir/diag.sh seq-check 0 0 +. $srcdir/diag.sh exit diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 2b06a28dd..40e105d0f 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -58,6 +58,7 @@ * -Z cert (public key) file for TLS mode * -L loglevel to use for GnuTLS troubleshooting (0-off to 10-all, 0 default) * -j format message in json, parameter is JSON cookie + * -O Use octate-count framing * * Part of the testbench for rsyslog. * @@ -104,6 +105,25 @@ # endif #endif +char *test_rs_strerror_r(int errnum, char *buf, size_t buflen) { +#ifndef HAVE_STRERROR_R + char *pszErr; + pszErr = strerror(errnum); + snprintf(buf, buflen, "%s", pszErr); +#else +# ifdef STRERROR_R_CHAR_P + char *p = strerror_r(errnum, buf, buflen); + if (p != buf) { + strncpy(buf, p, buflen); + buf[buflen - 1] = '\0'; + } +# else + strerror_r(errnum, buf, buflen); +# endif +#endif /* #ifdef __hpux */ + return buf; +} + #define EXIT_FAILURE 1 #define INVALID_SOCKET -1 /* Name of input file, must match $IncludeConfig in test suite .conf files */ @@ -128,6 +148,7 @@ static int msgNum = 0; /* initial message number to start with */ static int bShowProgress = 1; /* show progress messages */ static int bSilent = 0; /* completely silent operation */ static int bRandConnDrop = 0; /* randomly drop connections? */ +static double dbRandConnDrop = 0.95; /* random drop probability */ static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */ static int bBinaryFile = 0; /* is -I file binary */ static char *dataFile = NULL; /* name of data file, if NULL, generate own data */ @@ -147,6 +168,7 @@ static char *tlsCertFile = NULL; static char *tlsKeyFile = NULL; static int tlsLogLevel = 0; static char *jsonCookie = NULL; /* if non-NULL, use JSON format with this cookie */ +static int octateCountFramed = 0; #ifdef ENABLE_GNUTLS static gnutls_session_t *sessArray; /* array of TLS sessions to use */ @@ -358,6 +380,8 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) char extraData[MAX_EXTRADATA_LEN + 1]; char dynFileIDBuf[128] = ""; int done; + char payloadLen[32]; + int payloadStringLen; if(dataFP != NULL) { /* get message from file */ @@ -416,6 +440,13 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) /* use fixed message format from command line */ *pLenBuf = snprintf(buf, maxBuf, "%s\n", MsgToSend); } + if (octateCountFramed == 1) { + snprintf(payloadLen, sizeof(payloadLen), "%d ", *pLenBuf); + payloadStringLen = strlen(payloadLen); + memmove(buf + payloadStringLen, buf, *pLenBuf); + memcpy(buf, payloadLen, payloadStringLen); + *pLenBuf += payloadStringLen; + } ++inst->numSent; finalize_it: /*EMPTY to keep the compiler happy */; @@ -439,11 +470,17 @@ int sendMessages(struct instdata *inst) char buf[MAX_EXTRADATA_LEN + 1024]; char sendBuf[MAX_SENDBUF]; int offsSendBuf = 0; + char errStr[1024]; + int error_number = 0; + unsigned show_progress_interval = 100; if(!bSilent) { if(dataFile == NULL) { printf("Sending %llu messages.\n", inst->numMsgs); statusText = "messages"; + if ((inst->numMsgs / 100) > show_progress_interval) { + show_progress_interval = inst->numMsgs / 100; + } } else { printf("Sending file '%s' %d times.\n", dataFile, numFileIterations); @@ -477,9 +514,19 @@ int sendMessages(struct instdata *inst) } } lenSend = send(sockArray[socknum], buf, lenBuf, 0); + error_number = errno; } else if(transport == TP_UDP) { lenSend = sendto(udpsock, buf, lenBuf, 0, &udpRcvr, sizeof(udpRcvr)); + error_number = errno; } else if(transport == TP_TLS) { + if(sockArray[socknum] == -1) { + /* connection was dropped, need to re-establish */ + if(openConn(&(sockArray[socknum])) != 0) { + printf("error in trying to re-open connection %d\n", socknum); + exit(1); + } + initTLSSess(socknum); + } if(offsSendBuf + lenBuf < MAX_SENDBUF) { memcpy(sendBuf+offsSendBuf, buf, lenBuf); offsSendBuf += lenBuf; @@ -494,13 +541,14 @@ int sendMessages(struct instdata *inst) if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); - perror("send test data"); - printf("send() failed at socket %d, index %d, msgNum %lld\n", - sockArray[socknum], i, inst->numSent); + test_rs_strerror_r(error_number, errStr, sizeof(errStr)); + printf("send() failed \"%s\" at socket %d, index %d, msgNum %lld\n", + errStr, sockArray[socknum], i, inst->numSent); fflush(stderr); + return(1); } - if(i % 100 == 0) { + if(i % show_progress_interval == 0) { if(bShowProgress) printf("\r%8.8d", i); } @@ -508,7 +556,7 @@ int sendMessages(struct instdata *inst) /* if we need to randomly drop connections, see if we * are a victim */ - if(rand() > (int) (RAND_MAX * 0.95)) { + if(rand() > (int) (RAND_MAX * dbRandConnDrop)) { ++nConnDrops; close(sockArray[socknum]); sockArray[socknum] = -1; @@ -783,6 +831,8 @@ initTLS(void) } +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" static void initTLSSess(int i) { @@ -813,6 +863,8 @@ initTLSSess(int i) exit(1); } } +#pragma GCC diagnostic pop + static int sendTLS(int i, char *buf, int lenBuf) @@ -867,7 +919,7 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:yYz:Z:j:")) != -1) { + while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:l:L:M:rsBR:S:T:XW:yYz:Z:j:O")) != -1) { switch (opt) { case 'b': batchsize = atoll(optarg); break; @@ -902,6 +954,10 @@ int main(int argc, char *argv[]) break; case 'D': bRandConnDrop = 1; break; + case 'l': + dbRandConnDrop = atof(optarg); + printf("RandConnDrop Level: '%lf' \n", dbRandConnDrop); + break; case 'r': bRandomizeExtraData = 1; break; case 'f': dynFileIDs = atoi(optarg); @@ -957,6 +1013,8 @@ int main(int argc, char *argv[]) break; case 'Z': tlsCertFile = optarg; break; + case 'O': octateCountFramed = 1; + break; default: printf("invalid option '%c' or value missing - terminating...\n", opt); exit (1); break; diff --git a/tests/testsuites/dynstats.conf b/tests/testsuites/dynstats.conf new file mode 100644 index 000000000..56ae9da7f --- /dev/null +++ b/tests/testsuites/dynstats.conf @@ -0,0 +1,17 @@ +$IncludeConfig diag-common.conf + +ruleset(name="stats") { + action(type="omfile" file="./rsyslog.out.stats.log") +} + +module(load="../plugins/impstats/.libs/impstats" interval="1" severity="7" resetCounters="on" Ruleset="stats" bracketing="on") + +template(name="outfmt" type="string" string="%msg% %$.increment_successful%\n") + +dyn_stats(name="msg_stats") + +set $.msg_prefix = field($msg, 32, 1); + +set $.increment_successful = dyn_inc("msg_stats", $.msg_prefix); + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") \ No newline at end of file diff --git a/tests/testsuites/dynstats_ctr_reset.conf b/tests/testsuites/dynstats_ctr_reset.conf new file mode 100644 index 000000000..848dc9902 --- /dev/null +++ b/tests/testsuites/dynstats_ctr_reset.conf @@ -0,0 +1,21 @@ +$IncludeConfig diag-common.conf + +ruleset(name="stats") { + action(type="omfile" file="./rsyslog.out.stats.log") +} + +module(load="../plugins/impstats/.libs/impstats" interval="1" severity="7" resetCounters="on" Ruleset="stats" bracketing="on") + +template(name="outfmt" type="string" string="%msg%\n") + +dyn_stats(name="msg_stats_resettable_on" resettable="on") +dyn_stats(name="msg_stats_resettable_off" resettable="off") +dyn_stats(name="msg_stats_resettable_default") + +set $.msg_prefix = field($msg, 32, 1); + +set $.x = dyn_inc("msg_stats_resettable_on", $.msg_prefix); +set $.y = dyn_inc("msg_stats_resettable_off", $.msg_prefix); +set $.z = dyn_inc("msg_stats_resettable_default", $.msg_prefix); + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/dynstats_empty_input b/tests/testsuites/dynstats_empty_input new file mode 100644 index 000000000..d47777b8d --- /dev/null +++ b/tests/testsuites/dynstats_empty_input @@ -0,0 +1,7 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix foo 001 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix bar 002 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix baz 003 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix quux 004 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix corge 005 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix foo 006 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:prefix grault 007 diff --git a/tests/testsuites/dynstats_input b/tests/testsuites/dynstats_input new file mode 100644 index 000000000..0a8cd1047 --- /dev/null +++ b/tests/testsuites/dynstats_input @@ -0,0 +1,6 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 001 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:bar 002 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 003 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 004 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 005 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 006 diff --git a/tests/testsuites/dynstats_input_1 b/tests/testsuites/dynstats_input_1 new file mode 100644 index 000000000..345f5e405 --- /dev/null +++ b/tests/testsuites/dynstats_input_1 @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 001 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:bar 002 diff --git a/tests/testsuites/dynstats_input_2 b/tests/testsuites/dynstats_input_2 new file mode 100644 index 000000000..bee20ae9e --- /dev/null +++ b/tests/testsuites/dynstats_input_2 @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 003 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 004 diff --git a/tests/testsuites/dynstats_input_3 b/tests/testsuites/dynstats_input_3 new file mode 100644 index 000000000..4f88f2411 --- /dev/null +++ b/tests/testsuites/dynstats_input_3 @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 005 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 006 diff --git a/tests/testsuites/dynstats_input_more_0 b/tests/testsuites/dynstats_input_more_0 new file mode 100644 index 000000000..8d9ec2393 --- /dev/null +++ b/tests/testsuites/dynstats_input_more_0 @@ -0,0 +1,10 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 001 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:bar 002 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 003 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 004 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:baz 005 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 006 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:quux 007 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:corge 008 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:quux 009 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 010 diff --git a/tests/testsuites/dynstats_input_more_1 b/tests/testsuites/dynstats_input_more_1 new file mode 100644 index 000000000..64da0ef18 --- /dev/null +++ b/tests/testsuites/dynstats_input_more_1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:corge 011 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:grault 012 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 013 diff --git a/tests/testsuites/dynstats_input_more_2 b/tests/testsuites/dynstats_input_more_2 new file mode 100644 index 000000000..d5b575c39 --- /dev/null +++ b/tests/testsuites/dynstats_input_more_2 @@ -0,0 +1,5 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:corge 014 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:grault 015 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:quux 016 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:foo 017 +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005:corge 018 diff --git a/tests/testsuites/dynstats_nometric.conf b/tests/testsuites/dynstats_nometric.conf new file mode 100644 index 000000000..bb7a74d05 --- /dev/null +++ b/tests/testsuites/dynstats_nometric.conf @@ -0,0 +1,17 @@ +$IncludeConfig diag-common.conf + +ruleset(name="stats") { + action(type="omfile" file="./rsyslog.out.stats.log") +} + +module(load="../plugins/impstats/.libs/impstats" interval="1" severity="7" resetCounters="on" Ruleset="stats" bracketing="on") + +template(name="outfmt" type="string" string="%msg% %$.increment_successful%\n") + +dyn_stats(name="msg_stats") + +set $.msg_prefix = field($msg, 32, 2); + +set $.increment_successful = dyn_inc("msg_stats", $.msg_prefix); + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") \ No newline at end of file diff --git a/tests/testsuites/dynstats_overflow.conf b/tests/testsuites/dynstats_overflow.conf new file mode 100644 index 000000000..54f7fdcef --- /dev/null +++ b/tests/testsuites/dynstats_overflow.conf @@ -0,0 +1,22 @@ +$IncludeConfig diag-common.conf + +ruleset(name="stats") { + action(type="omfile" file="./rsyslog.out.stats.log") +} + +module(load="../plugins/impstats/.libs/impstats" interval="1" severity="7" resetCounters="on" Ruleset="stats" bracketing="on") + +template(name="outfmt" type="string" string="%msg% %$.increment_successful%\n") + +dyn_stats(name="msg_stats" unusedMetricLife="2" maxCardinality="3") + +set $.msg_prefix = field($msg, 32, 1); + +if (re_match($.msg_prefix, "foo|bar|baz|quux|corge|grault")) then { + set $.increment_successful = dyn_inc("msg_stats", $.msg_prefix); +} else { + set $.increment_successful = -1; +} + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") + diff --git a/tests/testsuites/dynstats_reset.conf b/tests/testsuites/dynstats_reset.conf new file mode 100644 index 000000000..9ae36e67b --- /dev/null +++ b/tests/testsuites/dynstats_reset.conf @@ -0,0 +1,21 @@ +$IncludeConfig diag-common.conf + +ruleset(name="stats") { + action(type="omfile" file="./rsyslog.out.stats.log") +} + +module(load="../plugins/impstats/.libs/impstats" interval="1" severity="7" resetCounters="on" Ruleset="stats" bracketing="on") + +template(name="outfmt" type="string" string="%msg% %$.increment_successful%\n") + +dyn_stats(name="msg_stats" unusedMetricLife="1") + +set $.msg_prefix = field($msg, 32, 1); + +if (re_match($.msg_prefix, "foo|bar|baz|quux|corge|grault")) then { + set $.increment_successful = dyn_inc("msg_stats", $.msg_prefix); +} else { + set $.increment_successful = -1; +} + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/exec_tpl-concurrency.conf b/tests/testsuites/exec_tpl-concurrency.conf new file mode 100644 index 000000000..b3b68eedd --- /dev/null +++ b/tests/testsuites/exec_tpl-concurrency.conf @@ -0,0 +1,18 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="interim" type="string" string="%$!tree!here!nbr%") +template(name="outfmt" type="string" string="%$!interim%\n") + +if $msg contains "msgnum:" then { + set $!tree!here!nbr = field($msg, 58, 2); + action(type="omfile" file="rsyslog2.out.log" template="outfmt" + queue.type="linkedList") + + set $!interim = exec_template("interim"); + set $!tree!here!nbr = ""; + action(type="omfile" file="rsyslog.out.log" template="outfmt" + queue.type="fixedArray") +} diff --git a/tests/testsuites/imptcp_nonProcessingPoller.conf b/tests/testsuites/imptcp_nonProcessingPoller.conf new file mode 100644 index 000000000..b82dad005 --- /dev/null +++ b/tests/testsuites/imptcp_nonProcessingPoller.conf @@ -0,0 +1,12 @@ +# test with poller-thread-processing disabled +# singh.janmejay, 2015-10-16 +$MaxMessageSize 10k +$IncludeConfig diag-common.conf +template(name="outfmt" type="string" string="%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n") + +module(load="../plugins/imptcp/.libs/imptcp" threads="32" processOnPoller="off") +input(type="imptcp" port="13514") + +if (prifilt("local0.*")) then { + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/imtcp_conndrop_tls.conf b/tests/testsuites/imtcp_conndrop_tls.conf new file mode 100644 index 000000000..a9584183e --- /dev/null +++ b/tests/testsuites/imtcp_conndrop_tls.conf @@ -0,0 +1,21 @@ +# simple async writing test +# rgerhards, 2010-03-09 +$MaxMessageSize 10k +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 + +# TLS Stuff - certificate files - just CA for a client +$DefaultNetstreamDriver gtls +$IncludeConfig rsyslog.conf.tlscert +$InputTCPServerStreamDriverMode 1 +$InputTCPServerStreamDriverAuthMode anon +$InputTCPServerRun 13514 + +$template outfmt,"%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +$OMFileFlushOnTXEnd off +$OMFileFlushInterval 2 +$OMFileIOBufferSize 256k +local0.* ?dynfile;outfmt diff --git a/tests/testsuites/imuxsock_logger_ruleset.conf b/tests/testsuites/imuxsock_logger_ruleset.conf new file mode 100644 index 000000000..8a1760d77 --- /dev/null +++ b/tests/testsuites/imuxsock_logger_ruleset.conf @@ -0,0 +1,13 @@ +# rgerhards, 2016-02-02 +$IncludeConfig diag-common.conf + +module(load="../plugins/imuxsock/.libs/imuxsock" sysSock.use="off") +input( type="imuxsock" socket="testbench_socket" + useSpecialParser="off" + ruleset="testruleset" + parseHostname="on") +template(name="outfmt" type="string" string="%msg:%\n") + +ruleset(name="testruleset") { + ./rsyslog.out.log;outfmt +} diff --git a/tests/testsuites/json_null.conf b/tests/testsuites/json_null.conf new file mode 100644 index 000000000..5b4ea56e2 --- /dev/null +++ b/tests/testsuites/json_null.conf @@ -0,0 +1,14 @@ +$IncludeConfig diag-common.conf +module(load="../plugins/mmjsonparse/.libs/mmjsonparse") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +# we must make sure the template contains a reference to the +# data item with null value +template(name="outfmt" type="string" string="%$!nope%\n") +template(name="outfmt-all-json" type="string" string="%$!all-json%\n") + +action(type="mmjsonparse") +action(type="omfile" file="./rsyslog.out.log" template="outfmt") +if $!nope == "" then + action(type="omfile" file="./rsyslog2.out.log" template="outfmt-all-json") diff --git a/tests/testsuites/json_null_array.conf b/tests/testsuites/json_null_array.conf new file mode 100644 index 000000000..e8032c7fb --- /dev/null +++ b/tests/testsuites/json_null_array.conf @@ -0,0 +1,12 @@ +$IncludeConfig diag-common.conf +module(load="../plugins/mmjsonparse/.libs/mmjsonparse") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="outfmt" type="string" string="%$.data%\n") + +action(type="mmjsonparse") +foreach ($.data in $!array) do { + if not ($.data == "") then + action(type="omfile" file="rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/json_var_cmpr.conf b/tests/testsuites/json_var_cmpr.conf new file mode 100644 index 000000000..b19108232 --- /dev/null +++ b/tests/testsuites/json_var_cmpr.conf @@ -0,0 +1,22 @@ +$IncludeConfig diag-common.conf +module(load="../plugins/mmjsonparse/.libs/mmjsonparse") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +# we must make sure the template contains references to the variables +template(name="outfmt" type="string" string="json prop:%$!val% local prop:%$.val% global prop:%$/val%\n") + +action(type="mmjsonparse") + +set $.val = "123"; +set $.rval = "123"; +if ($.val == $.rval) then { + set $.val = "def"; +} +set $/val = "123"; +set $/rval = "123"; +if ($/val == $/rval) then { + set $/val = "ghi"; +} + +action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/localvar-concurrency.conf b/tests/testsuites/localvar-concurrency.conf new file mode 100644 index 000000000..087c26b3d --- /dev/null +++ b/tests/testsuites/localvar-concurrency.conf @@ -0,0 +1,17 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="outfmt" type="string" string="%$.tree!here!nbr%\n") + +if $msg contains "msgnum:" then { + set $.tree!here!nbr = field($msg, 58, 2); + action(type="omfile" file="rsyslog2.out.log" template="outfmt" + queue.type="linkedList") + + set $.tree!here!save = $.tree!here!nbr; + set $.tree!here!nbr = ""; + set $.tree!here!nbr = $.tree!here!save; + action(type="omfile" file="rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/mmnormalize_regex.rulebase b/tests/testsuites/mmnormalize_regex.rulebase index f58e8f295..d9dbb474e 100644 --- a/tests/testsuites/mmnormalize_regex.rulebase +++ b/tests/testsuites/mmnormalize_regex.rulebase @@ -1 +1 @@ -rule=:http host ports are %hps:regex:([0-9.\x3a]+(, )?)+% etc \ No newline at end of file +rule=:http host ports are %hps:regex:([0-9.\x3a]+(, )?)+% etc diff --git a/tests/testsuites/mmnormalize_tokenized.rulebase b/tests/testsuites/mmnormalize_tokenized.rulebase index da5242d79..0b9bd863e 100644 --- a/tests/testsuites/mmnormalize_tokenized.rulebase +++ b/tests/testsuites/mmnormalize_tokenized.rulebase @@ -2,4 +2,4 @@ rule=only_ips:%only_ips:tokenized:, :ipv4% rule=local_ips:local ips are %local_ips:tokenized:, :ipv4% rule=external_ips:%external_ips:tokenized:, :ipv4% are external ips rule=paths:for %user:char-to:@%@localhost path was %fragments:tokenized:\x3a:char-sep:\x3a% -rule=recur_comma_colon_nos:comma separated list of colon separated numbers: %some_nos:tokenized:, :tokenized: \x3a :tokenized:#:number% \ No newline at end of file +rule=recur_comma_colon_nos:comma separated list of colon separated numbers: %some_nos:tokenized:, :tokenized: \x3a :tokenized:#:number% diff --git a/tests/testsuites/mmnormalize_variable.rulebase b/tests/testsuites/mmnormalize_variable.rulebase index 4d40d4cc2..b3867879c 100644 --- a/tests/testsuites/mmnormalize_variable.rulebase +++ b/tests/testsuites/mmnormalize_variable.rulebase @@ -1 +1 @@ -rule=hms:%hr:number%:%min:number%:%sec:number% %zone:word% \ No newline at end of file +rule=hms:%hr:number%:%min:number%:%sec:number% %zone:word% diff --git a/tests/testsuites/msgvar-concurrency-array.conf b/tests/testsuites/msgvar-concurrency-array.conf new file mode 100644 index 000000000..d0ea152c2 --- /dev/null +++ b/tests/testsuites/msgvar-concurrency-array.conf @@ -0,0 +1,18 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/mmnormalize/.libs/mmnormalize") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="outfmt" type="string" string="%$!%\n") + +#action(type="omfile" file="rsyslog2.out.log" template="outfmt" queue.type="linkedList") +action(type="mmnormalize" ruleBase="testsuites/msgvar-concurrency-array.rulebase") +if $msg contains "msg:" then { +# set $!tree!here!nbr = field($msg, 58, 2); # Delimiter = : + action(type="omfile" file="rsyslog2.out.log" template="outfmt" queue.type="linkedList") + set $!tree!here!save = $!tree!here!nbr; + set $!tree!here!nbr = ""; + set $!tree!here!nbr = $!tree!here!save; + action(type="omfile" file="rsyslog.out.log" template="outfmt" queue.type="linkedList") +} diff --git a/tests/testsuites/msgvar-concurrency-array.rulebase b/tests/testsuites/msgvar-concurrency-array.rulebase new file mode 100644 index 000000000..8f1951401 --- /dev/null +++ b/tests/testsuites/msgvar-concurrency-array.rulebase @@ -0,0 +1,11 @@ +version=2 +rule=:msg: %{"name":"numbers", "type":"repeat", + "parser":[ + {"name":"n1", "type":"number"}, + {"type":"literal", "text":":"}, + {"name":"n2", "type":"number"} + ], + "while":[ + {"type":"literal", "text":", "} + ] + }% b %w:word% diff --git a/tests/testsuites/msgvar-concurrency.conf b/tests/testsuites/msgvar-concurrency.conf new file mode 100644 index 000000000..31da7fc14 --- /dev/null +++ b/tests/testsuites/msgvar-concurrency.conf @@ -0,0 +1,17 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="outfmt" type="string" string="%$!tree!here!nbr%\n") + +if $msg contains "msgnum:" then { + set $!tree!here!nbr = field($msg, 58, 2); + action(type="omfile" file="rsyslog2.out.log" template="outfmt" + queue.type="linkedList") + + set $!tree!here!save = $!tree!here!nbr; + set $!tree!here!nbr = ""; + set $!tree!here!nbr = $!tree!here!save; + action(type="omfile" file="rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/now_family_utc.conf b/tests/testsuites/now_family_utc.conf new file mode 100644 index 000000000..9f7e9160f --- /dev/null +++ b/tests/testsuites/now_family_utc.conf @@ -0,0 +1,9 @@ +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +template(name="outfmt" type="string" + string="%$hour%:%$minute%,%$hour-utc%:%$minute-utc%\n") +:msg, contains, "msgnum:" action(type="omfile" template="outfmt" + file="rsyslog.out.log") diff --git a/tests/testsuites/rawmsg-after-pri.conf b/tests/testsuites/rawmsg-after-pri.conf new file mode 100644 index 000000000..0485898a7 --- /dev/null +++ b/tests/testsuites/rawmsg-after-pri.conf @@ -0,0 +1,8 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(type="string" name="outfmt" string="%rawmsg-after-pri%\n") +if $syslogfacility-text == "local0" then + action(type="omfile" file="rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/rscript_eq_var.conf b/tests/testsuites/rscript_eq_var.conf new file mode 100644 index 000000000..022afd4c9 --- /dev/null +++ b/tests/testsuites/rscript_eq_var.conf @@ -0,0 +1,57 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "value"; +set $!var2 = "value"; +if $!var1 == $!var2 then { + set $!var2 = "bad"; + if $!var1 == $!var2 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + } +} else { + # Failure + stop +} +set $.var1 = "value"; +set $.var2 = "value"; +if $.var1 == $.var2 then { + set $.var2 = "bad"; + if $.var1 == $.var2 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + } +} else { + # Failure + stop +} +set $/var1 = "value"; +set $/var2 = "value"; +if $/var1 == $/var2 then { + set $/var2 = "bad"; + if $/var1 == $/var2 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_ge_var.conf b/tests/testsuites/rscript_ge_var.conf new file mode 100644 index 000000000..bb627369e --- /dev/null +++ b/tests/testsuites/rscript_ge_var.conf @@ -0,0 +1,60 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "42"; +set $!var2 = "42"; +set $!var3 = "41"; +if $!var1 >= $!var2 and $!var1 >= $!var3 then { + if $!var3 >= $!var1 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + unset $!var3; + } +} else { + # Failure + stop +} +set $.var1 = "42"; +set $.var2 = "42"; +set $.var3 = "41"; +if $.var1 >= $.var2 and $.var1 >= $.var3 then { + if $.var3 >= $.var1 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + unset $.var3; + } +} else { + # Failure + stop +} +set $/var1 = "42"; +set $/var2 = "42"; +set $/var3 = "41"; +if $/var1 >= $/var2 and $/var1 >= $/var3 then { + if $/var3 >= $/var1 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + unset $/var3; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_gt_var.conf b/tests/testsuites/rscript_gt_var.conf new file mode 100644 index 000000000..7751667dd --- /dev/null +++ b/tests/testsuites/rscript_gt_var.conf @@ -0,0 +1,54 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "43"; +set $!var2 = "42"; +if $!var1 > $!var2 then { + if $!var2 > $!var1 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + } +} else { + # Failure + stop +} +set $.var1 = "43"; +set $.var2 = "42"; +if $.var1 > $.var2 then { + if $.var2 > $.var1 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + } +} else { + # Failure + stop +} +set $/var1 = "43"; +set $/var2 = "42"; +if $/var1 > $/var2 then { + if $/var2 > $/var1 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_le_var.conf b/tests/testsuites/rscript_le_var.conf new file mode 100644 index 000000000..1c8293f5d --- /dev/null +++ b/tests/testsuites/rscript_le_var.conf @@ -0,0 +1,60 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "42"; +set $!var2 = "42"; +set $!var3 = "43"; +if $!var1 <= $!var2 and $!var1 <= $!var3 then { + if $!var3 <= $!var1 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + unset $!var3; + } +} else { + # Failure + stop +} +set $.var1 = "42"; +set $.var2 = "42"; +set $.var3 = "43"; +if $.var1 <= $.var2 and $.var1 <= $.var3 then { + if $.var3 <= $.var1 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + unset $.var3; + } +} else { + # Failure + stop +} +set $/var1 = "42"; +set $/var2 = "42"; +set $/var3 = "43"; +if $/var1 <= $/var2 and $/var1 <= $/var3 then { + if $/var3 <= $/var1 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + unset $/var3; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_lt_var.conf b/tests/testsuites/rscript_lt_var.conf new file mode 100644 index 000000000..52328e1d0 --- /dev/null +++ b/tests/testsuites/rscript_lt_var.conf @@ -0,0 +1,54 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "41"; +set $!var2 = "42"; +if $!var1 < $!var2 then { + if $!var2 < $!var1 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + } +} else { + # Failure + stop +} +set $.var1 = "41"; +set $.var2 = "42"; +if $.var1 < $.var2 then { + if $.var2 < $.var1 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + } +} else { + # Failure + stop +} +set $/var1 = "41"; +set $/var2 = "42"; +if $/var1 < $/var2 then { + if $/var2 < $/var1 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_ne_var.conf b/tests/testsuites/rscript_ne_var.conf new file mode 100644 index 000000000..26e71eabf --- /dev/null +++ b/tests/testsuites/rscript_ne_var.conf @@ -0,0 +1,60 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +set $!var1 = "value1"; +set $!var2 = "value2"; +if $!var1 != $!var2 then { + set $!var1 = "value"; + set $!var2 = "value"; + if $!var1 != $!var2 then { + # Failure + stop + } else { + unset $!var1; + unset $!var2; + } +} else { + # Failure + stop +} +set $.var1 = "value1"; +set $.var2 = "value2"; +if $.var1 != $.var2 then { + set $.var1 = "value"; + set $.var2 = "value"; + if $.var1 != $.var2 then { + # Failure + stop + } else { + unset $.var1; + unset $.var2; + } +} else { + # Failure + stop +} +set $/var1 = "value1"; +set $/var2 = "value2"; +if $/var1 != $/var2 then { + set $/var1 = "value"; + set $/var2 = "value"; + if $/var1 != $/var2 then { + # Failure + stop + } else { + unset $/var1; + unset $/var2; + } +} else { + # Failure + stop +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/travis/trusty.supp b/tests/travis/trusty.supp new file mode 100644 index 000000000..b490807b4 --- /dev/null +++ b/tests/travis/trusty.supp @@ -0,0 +1,10 @@ +{ + gnu_libc_memerr + Memcheck:Free + fun:free + fun:__libc_freeres + fun:_vgnU_freeres + fun:__run_exit_handlers + fun:exit + fun:(below main) +} diff --git a/tools/coreanalysis.sh b/tools/coreanalysis.sh new file mode 100755 index 000000000..bebd447ee --- /dev/null +++ b/tools/coreanalysis.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# this shell script provides automatic coredump analysis with gdb. +# begun 2015-10-14 by alorbach +# This file is part of the rsyslog project, released under ASL 2.0 + +echo "----------------------------------" +echo "--- Coredump Analysis with GDB ---" +echo "----------------------------------" +read -p "Where are the core files localed (default /)?" DIRECTORY +if [ "$DIRECTORY" == "" ]; then + DIRECTORY="/" +fi + +COREFILES=`ls $DIRECTORY/core* 2> /dev/null` +# &> /dev/null` + +select COREFILE in $COREFILES +do + echo "Which coredump do you want to analyse? " + break; +done + +# Check for found core files +if [ "$COREFILE" == "" ]; then + echo "Failed to find any coredump files in $DIRECTORY or invalid selection." + exit; +fi + +# generating tmp commandfile +echo "Trying to analyze core for main rsyslogd binary" +echo "set pagination off" >gdb.in +echo "core $COREFILE" >>gdb.in +echo "info thread" >> gdb.in +echo "thread apply all bt full" >> gdb.in +echo "q" >> gdb.in + +# Run GDB with commandfile +gdb /usr/sbin/rsyslogd < gdb.in + +# rm tmp commandfile +rm gdb.in diff --git a/tools/omdiscard.c b/tools/omdiscard.c index 4cef1ec60..567800b38 100644 --- a/tools/omdiscard.c +++ b/tools/omdiscard.c @@ -81,8 +81,9 @@ BEGINtryResume CODESTARTtryResume ENDtryResume -BEGINdoAction +BEGINdoAction_NoStrings CODESTARTdoAction + (void)pMsgData; /* Suppress compiler warning on unused var */ dbgprintf("\n"); iRet = RS_RET_DISCARDMSG; ENDdoAction diff --git a/tools/omfile.c b/tools/omfile.c index 8fa84b234..8388b5d36 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -409,7 +409,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR i = 0; /* get outchannel name */ while(*p && *p != ';' && *p != ' ' && - i < sizeof(szBuf) / sizeof(char)) { + i < (sizeof(szBuf) - 1) ) { szBuf[i++] = *p++; } szBuf[i] = '\0'; @@ -1037,14 +1037,16 @@ CODESTARTcommitTransaction writeFile(pData, pParams, i); } /* Note: pStrm may be NULL if there was an error opening the stream */ - if(pData->bFlushOnTXEnd && pData->pStrm != NULL) { - /* if we have an async writer, it controls the flush via - * a timeout. However, without it, we actually need to flush, - * else incomplete records are written. - */ - if(!pData->bUseAsyncWriter) + if(pData->bUseAsyncWriter) { + if(pData->bFlushOnTXEnd && pData->pStrm != NULL) { CHKiRet(strm.Flush(pData->pStrm)); + } + } else { + if(pData->pStrm != NULL) { + CHKiRet(strm.Flush(pData->pStrm)); + } } + finalize_it: pthread_mutex_unlock(&pData->mutWrite); ENDcommitTransaction diff --git a/tools/omfwd.c b/tools/omfwd.c index 22536bcbb..f2a721cee 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -597,8 +597,7 @@ doZipFinish(wrkrInstanceData_t *pWrkrData) if(!pWrkrData->bzInitDone) goto done; -// TODO: can we get this into a single common function? -dbgprintf("DDDD: in doZipFinish()\n"); + // TODO: can we get this into a single common function? pWrkrData->zstrm.avail_in = 0; /* run deflate() on buffer until everything has been compressed */ do { @@ -634,10 +633,15 @@ static rsRetVal TCPSendFrame(void *pvData, char *msg, size_t len) DBGPRINTF("omfwd: add %u bytes to send buffer (curr offs %u)\n", (unsigned) len, pWrkrData->offsSndBuf); if(pWrkrData->offsSndBuf != 0 && pWrkrData->offsSndBuf + len >= sizeof(pWrkrData->sndBuf)) { - /* no buffer space left, need to commit previous records */ + /* no buffer space left, need to commit previous records. With the + * current API, there unfortunately is no way to signal this + * state transition to the upper layer. + */ + DBGPRINTF("omfwd: we need to do a tcp send due to buffer " + "out of space. If the transaction fails, this will " + "lead to duplication of messages"); CHKiRet(TCPSendBuf(pWrkrData, pWrkrData->sndBuf, pWrkrData->offsSndBuf, NO_FLUSH)); pWrkrData->offsSndBuf = 0; - iRet = RS_RET_PREVIOUS_COMMITTED; } /* check if the message is too large to fit into buffer */ @@ -777,14 +781,14 @@ finalize_it: BEGINtryResume CODESTARTtryResume - dbgprintf("DDDD: tryResume: pWrkrData %p\n", pWrkrData); + dbgprintf("omfwd: tryResume: pWrkrData %p\n", pWrkrData); iRet = doTryResume(pWrkrData); ENDtryResume BEGINbeginTransaction CODESTARTbeginTransaction -dbgprintf("omfwd: beginTransaction\n"); + dbgprintf("omfwd: beginTransaction\n"); iRet = doTryResume(pWrkrData); ENDbeginTransaction @@ -876,7 +880,6 @@ CODESTARTcommitTransaction FINALIZE; } -dbgprintf("omfwd: endTransaction, offsSndBuf %u, iRet %d\n", pWrkrData->offsSndBuf, iRet); if(pWrkrData->offsSndBuf != 0) { iRet = TCPSendBuf(pWrkrData, pWrkrData->sndBuf, pWrkrData->offsSndBuf, IS_FLUSH); pWrkrData->offsSndBuf = 0; diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c index 5e93f7c93..633d74139 100644 --- a/tools/pmrfc3164.c +++ b/tools/pmrfc3164.c @@ -109,6 +109,7 @@ BEGINnewParserInst CODESTARTnewParserInst DBGPRINTF("newParserInst (pmrfc3164)\n"); + inst = NULL; CHKiRet(createInstance(&inst)); if(lst == NULL) @@ -139,6 +140,8 @@ finalize_it: CODE_STD_FINALIZERnewParserInst if(lst != NULL) cnfparamvalsDestruct(pvals, &parserpblk); + if(iRet != RS_RET_OK) + free(inst); ENDnewParserInst diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c index 18dc8464c..4c9d49b0c 100644 --- a/tools/pmrfc5424.c +++ b/tools/pmrfc5424.c @@ -6,7 +6,7 @@ * * File begun on 2009-11-03 by RGerhards * - * Copyright 2007-2014 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2015 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -235,7 +235,7 @@ CODESTARTparse * message, so we can not run into any troubles. I think this is * wiser than to use individual buffers. */ - CHKmalloc(pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1))); + CHKmalloc(pBuf = MALLOC(lenMsg + 1)); /* IMPORTANT NOTE: * Validation is not actually done below nor are any errors handled. I have @@ -245,7 +245,11 @@ CODESTARTparse */ /* TIMESTAMP */ - if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { + if(lenMsg >= 2 && p2parse[0] == '-' && p2parse[1] == ' ') { + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + p2parse += 2; + lenMsg -= 2; + } else if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) { if(pMsg->msgFlags & IGNDATE) { /* we need to ignore the msg data, so simply copy over reception date */ memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); diff --git a/tools/rscryutil.c b/tools/rscryutil.c index 17e206817..28158cf2a 100644 --- a/tools/rscryutil.c +++ b/tools/rscryutil.c @@ -276,7 +276,7 @@ doDecrypt(FILE *logfp, FILE *eifp, FILE *outfp) { off64_t blkEnd; off64_t currOffs = 0; - int r; + int r = 1; int fd; struct stat buf; diff --git a/tools/rsgtutil.c b/tools/rsgtutil.c index a96c6b68b..bb5906396 100644 --- a/tools/rsgtutil.c +++ b/tools/rsgtutil.c @@ -46,11 +46,16 @@ typedef unsigned char uchar; static enum { MD_DUMP, MD_DETECT_FILE_TYPE, MD_SHOW_SIGBLK_PARAMS, - MD_VERIFY, MD_EXTEND, MD_CONVERT + MD_VERIFY, MD_EXTEND, MD_CONVERT, MD_EXTRACT } mode = MD_DUMP; +static enum { FILEMODE_LOGSIG, FILEMODE_RECSIG }; static enum { API_GT, API_KSI } apimode = API_GT; static int verbose = 0; static int debug = 0; +/* Helper variables for EXTRACT Mode */ +static int append = 0; +char *outputfile = NULL; +char *linenumbers = ""; #ifdef ENABLEGT static void @@ -171,7 +176,7 @@ convertFile(char *name) if ( fwrite(LOGSIGHDR, sizeof(LOGSIGHDR)-1, 1, newsigfp) != 1) goto err; } - if ((r = rsgt_ConvertSigFile(name, oldsigfp, newsigfp, verbose)) != 0) + if ((r = rsgt_ConvertSigFile(oldsigfp, newsigfp, verbose)) != 0) goto err; else { /* Close FILES */ @@ -292,10 +297,10 @@ showSigblkParamsKSI(char *name) goto err; } } - if((r = rsksi_chkFileHdr(fp, "LOGSIG11")) != 0) goto err; + if((r = rsksi_chkFileHdr(fp, "LOGSIG11", verbose)) != 0) goto err; while(1) { /* we will err out on EOF */ - if((r = rsksi_getBlockParams(fp, 0, &bs, &bh, &bHasRecHashes, + if((r = rsksi_getBlockParams(NULL, fp, 0, &bs, &bh, &bHasRecHashes, &bHasIntermedHashes)) != 0) goto err; ++blkCnt; @@ -445,7 +450,7 @@ err: fprintf(stderr, "error %d (%s) processing file %s\n", r, RSGTE2String(r), n static inline int doVerifyRec(FILE *logfp, FILE *sigfp, FILE *nsigfp, - block_sig_t *bs, gtfile gf, gterrctx_t *ectx, uint8_t bInBlock) + gtfile gf, gterrctx_t *ectx, uint8_t bInBlock) { int r; size_t lenRec; @@ -473,7 +478,7 @@ doVerifyRec(FILE *logfp, FILE *sigfp, FILE *nsigfp, if(bInBlock == 0) rsgt_errctxFrstRecInBlk(ectx, line); - r = rsgt_vrfy_nextRec(bs, gf, sigfp, nsigfp, (unsigned char*)line, lenRec, ectx); + r = rsgt_vrfy_nextRec(gf, sigfp, nsigfp, (unsigned char*)line, lenRec, ectx); done: return r; } @@ -552,7 +557,7 @@ verifyGT(char *name, char *errbuf, char *sigfname, char *oldsigfname, char *nsig ++ectx.blkNum; } ++ectx.recNum, ++ectx.recNumInFile; - if((r = doVerifyRec(logfp, sigfp, nsigfp, bs, gf, &ectx, bInBlock)) != 0) + if((r = doVerifyRec(logfp, sigfp, nsigfp, gf, &ectx, bInBlock)) != 0) goto done; if(ectx.recNum == bs->recCount) { if((r = verifyBLOCK_SIG(bs, gf, sigfp, nsigfp, @@ -685,7 +690,7 @@ err: fprintf(stderr, "error %d (%s) processing file %s\n", r, RSKSIE2String(r), static inline int doVerifyRecKSI(FILE *logfp, FILE *sigfp, FILE *nsigfp, - /*block_sig_t *bs, */ ksifile ksi, ksierrctx_t *ectx, uint8_t bInBlock) + ksifile ksi, ksierrctx_t *ectx, uint8_t bInBlock) { int r; size_t lenRec; @@ -727,6 +732,7 @@ done: static int verifyKSI(char *name, char *errbuf, char *sigfname, char *oldsigfname, char *nsigfname, FILE *logfp, FILE *sigfp, FILE *nsigfp) { + int FILEMODE = FILEMODE_LOGSIG; /* Default FileMode */ block_sig_t *bs = NULL; block_hdr_t *bh = NULL; ksifile ksi; @@ -734,73 +740,165 @@ verifyKSI(char *name, char *errbuf, char *sigfname, char *oldsigfname, char *nsi uint8_t bInBlock; int r = 0; int bInitDone = 0; + tlvrecord_t tlvrec; ksierrctx_t ectx; - rsksi_errctxInit(&ectx); + /* Helpers for extract verify */ + int iCurrentLine = 0; + int iMaxLine = 0; + char* lineRec = NULL; + size_t stlen = 0; + ssize_t ssread; + + /* Init KSI related variables */ + rsksi_errctxInit(&ectx); rsksiInit("rsyslog rsksiutil " VERSION); bInitDone = 1; ectx.verbose = verbose; ectx.fp = stderr; ectx.filename = strdup(sigfname); - - if((r = rsksi_chkFileHdr(sigfp, "LOGSIG11")) != 0) { - if (debug) - fprintf(stderr, "error %d in rsksi_chkFileHdr\n", r); - goto done; - } - if(mode == MD_EXTEND) { - if(fwrite("LOGSIG11", 8, 1, nsigfp) != 1) { - perror(nsigfname); - r = RSGTE_IO; - goto done; - } - } ksi = rsksi_vrfyConstruct_gf(); if(ksi == NULL) { - fprintf(stderr, "error initializing signature file structure\n"); + fprintf(stderr, "verifyKSI:\t\t\t Error initializing signature file structure\n"); goto done; } - bInBlock = 0; - ectx.blkNum = 0; - ectx.recNumInFile = 0; + /* Check if we have a logsignature file */ + if((r = rsksi_chkFileHdr(sigfp, "LOGSIG11", verbose)) == 0) { + /* Verify Log signature */ + if(debug) printf("debug: verifyKSI:\t\t\t Found log signature file ... \n"); + if(mode == MD_EXTEND) { + if(fwrite("LOGSIG11", 8, 1, nsigfp) != 1) { + perror(nsigfname); + r = RSGTE_IO; + goto done; + } + } - while(!feof(logfp)) { - if(bInBlock == 0) { - if(bs != NULL) - rsksi_objfree(0x0904, bs); - if (bh != NULL) - rsksi_objfree(0x0901, bh); - if((r = rsksi_getBlockParams(sigfp, 1, &bs, &bh, &bHasRecHashes, - &bHasIntermedHashes)) != 0) { + bInBlock = 0; + ectx.blkNum = 0; + ectx.recNumInFile = 0; + + while(!feof(logfp)) { + if(bInBlock == 0) { + if (bs != NULL) rsksi_objfree(0x0904, bs); + if (bh != NULL) rsksi_objfree(0x0901, bh); + if((r = rsksi_getBlockParams(ksi, sigfp, 1, &bs, &bh, &bHasRecHashes, + &bHasIntermedHashes)) != 0) { + if(ectx.blkNum == 0) { + fprintf(stderr, "verifyKSI:\t\t\t Error %d before finding any signature block - is the file still open and being written to?\n", r); + } else { + if(verbose) fprintf(stderr, "verifyKSI:\t\t\t EOF after signature block %lld\n", (long long unsigned) ectx.blkNum); + } + goto done; + } + /* Copy block header */ + if ((r = verifyBLOCK_HDRKSI(ksi, sigfp, nsigfp, &tlvrec)) != 0) goto done; + + rsksi_vrfyBlkInit(ksi, bh, bHasRecHashes, bHasIntermedHashes); + ectx.recNum = 0; + ++ectx.blkNum; + } + ++ectx.recNum, ++ectx.recNumInFile; + if((r = doVerifyRecKSI(logfp, sigfp, nsigfp, ksi, &ectx, bInBlock)) != 0) + goto done; + if(ectx.recNum == bs->recCount) { + /* And Verify Block signature */ + if((r = verifyBLOCK_SIGKSI(bs, ksi, sigfp, nsigfp, (mode == MD_EXTEND) ? 1 : 0, NULL, &ectx)) != 0) + goto done; + bInBlock = 0; + } else bInBlock = 1; + } + } else if((r = rsksi_chkFileHdr(sigfp, "RECSIG11", verbose)) == 0) { + /* Verify Log Excerpts */ + if(debug) printf("verifyKSI:\t\t\t Found record integrity proof file ... \n"); + FILEMODE = FILEMODE_RECSIG; + + bInBlock = 0; + ectx.blkNum = 0; + ectx.recNumInFile = 0; + + while( !feof(logfp) && + !feof(sigfp)) { + /* Free memory */ + if (bs != NULL) rsksi_objfree(0x0905, bs); + if (bh != NULL) rsksi_objfree(0x0901, bh); + + /* Get/Verify Block Paramaters */ + if((r = rsksi_getExcerptBlockParams(ksi, sigfp, 1, &bs, &bh)) != 0) { if(ectx.blkNum == 0) { - fprintf(stderr, "Error %d before finding any signature block - " - "is the file still open and being written to?\n", r); + fprintf(stderr, "verifyKSI:\t\t\t Error %d before finding any signature block\n", r); } else { - if(verbose) - fprintf(stderr, "EOF after signature block %lld\n", - (long long unsigned) ectx.blkNum); + if(verbose) fprintf(stderr, "verifyKSI:\t\t\t EOF after signature block %lld\n", (long long unsigned) ectx.blkNum); } goto done; } - /* Copy block header */ - if ((r = verifyBLOCK_HDRKSI(sigfp, nsigfp)) != 0) goto done; - - rsksi_vrfyBlkInit(ksi, bh, bHasRecHashes, bHasIntermedHashes); ectx.recNum = 0; ++ectx.blkNum; - } - ++ectx.recNum, ++ectx.recNumInFile; - if((r = doVerifyRecKSI(logfp, sigfp, nsigfp, /*bs,*/ ksi, &ectx, bInBlock)) != 0) - goto done; - if(ectx.recNum == bs->recCount) { - if((r = verifyBLOCK_SIGKSI(bs, ksi, sigfp, nsigfp, - (mode == MD_EXTEND) ? 1 : 0, &ectx)) != 0) +// NEEDED ??? ++ectx.recNum, ++ectx.recNumInFile; + + /* Verify if records were found */ + if (bs->recCount <= 0) { + r = RSGTE_INVLD_RECCNT; + if(verbose) fprintf(stderr, "verifyKSI:\t\t\t Either signature block or hash chains are missing.\n"); goto done; - bInBlock = 0; - } else bInBlock = 1; + } + + /* Init Minimal KSI Helper */ + rsksi_vrfyBlkInit(ksi, bh, 1, 0); + + if(debug) printf("debug: verifyKSI:\t\t\t Verifying %lld hashchains ... \n", (long long unsigned) bs->recCount); + + /* Set new MAXLINE */ + iMaxLine = iCurrentLine + bs->recCount; + + do + { +if (debug) printf("debug: verifyKSI:\t\t\t NEXT IN LOOP .\n"); + /* Free memory from last lineRec first*/ + if (lineRec != NULL) { + free(lineRec); + lineRec = NULL; + } + + /* Get next line from file! */ + ssread = getline(&lineRec, &stlen, logfp); + if (ssread != -1) { + iCurrentLine++; /* Increment Current Line */ + + /* Remove Linefeed for verification */ + if( *(lineRec+ssread-1) == '\n') { + *(lineRec+ssread-1) = '\0'; + --ssread; + rsksi_errctxSetErrRec(&ectx, lineRec); + } + } else { + /* END of file reached */ + if (debug) printf("debug: verifyKSI:\t\t\t End of file reached.\n"); + r = RSGTE_EOF; + break; + } + + /* we need to preserve the first line (record) of each block for error-reporting purposes */ + rsksi_errctxFrstRecInBlk(&ectx, lineRec); + if (debug) printf("debug: verifyKSI:\t\t\t Processing line '%d': %.64s... \n", iCurrentLine, lineRec); + + /* Verify logline record against hash chain */ + if ((r = rsksi_vrfy_nextHashChain(ksi, bs, sigfp, (unsigned char*)lineRec, ssread, &ectx)) != RSGTE_SUCCESS) { + fprintf(stderr, "verifyKSI:\t\t\t error %d while verifiying hash chain record for logline (%d): '%.64s...'\n", r, iCurrentLine, lineRec); + goto done; + } + } while (iCurrentLine > iMaxLine && r != RSGTE_EOF); +if (debug) printf("debug: verifyKSI:\t\t\t DONE with LOOP iCurrentLine=%d > iMaxLine=%d r=%d\n", iCurrentLine, iMaxLine, r); + } + } else { + fprintf(stderr, "verifyKSI:\t\t\t Error %d invalid file header found \n", r); } done: + /* Free mem first */ + if (lineRec != NULL) + free(lineRec); + if(r != RSGTE_EOF) goto err; @@ -817,15 +915,13 @@ done: goto err; } - + /* Close File handles */ fclose(logfp); logfp = NULL; fclose(sigfp); sigfp = NULL; - if(nsigfp != NULL) { - fclose(nsigfp); nsigfp = NULL; - } + if(nsigfp != NULL) { fclose(nsigfp); nsigfp = NULL; } - /* everything went fine, so we rename files if we updated them */ - if(mode == MD_EXTEND) { + /* Check for Extend in LogSig Mode: Everything went fine, so we rename files if we updated them */ + if(FILEMODE == FILEMODE_LOGSIG && mode == MD_EXTEND) { if(unlink(oldsigfname) != 0) { if(errno != ENOENT) { perror("unlink oldsig"); @@ -860,30 +956,444 @@ done: goto err; } } - /* OLDCODE rsksiExit();*/ + rsksi_errctxExit(&ectx); return 1; - err: if(r != 0) - sprintf(errbuf, "error %d (%s) processing file %s\n", - r, RSKSIE2String(r), name); + sprintf(errbuf, "error %d (%s) processing file %s\n", r, RSKSIE2String(r), name); else errbuf[0] = '\0'; - if(logfp != NULL) - fclose(logfp); - if(sigfp != NULL) - fclose(sigfp); - if(nsigfp != NULL) { - fclose(nsigfp); - unlink(nsigfname); - } - if(bInitDone) { - /* OLDCODE rsksiExit();*/ - rsksi_errctxExit(&ectx); - } + /* Close File handles */ + if(logfp != NULL) fclose(logfp); + if(sigfp != NULL) fclose(sigfp); + if(nsigfp != NULL) { fclose(nsigfp); unlink(nsigfname); } + + if(bInitDone) { rsksi_errctxExit(&ectx); } return 0; } + +/* EXTRACT Loglines Function using KSI API + * + * Input: logfilename and open file handles + */ +static int +extractKSI(char *name, char *errbuf, char *sigfname, FILE *logfp, FILE *sigfp) +{ + char newsigfname[4096]; + FILE *newsigfp = NULL; + FILE *newlogfp = NULL; + char* lineRec = NULL; + size_t stlen = 0; + ssize_t ssread; + char writeMode[2] = "w"; /* Default = Create new file! */ + int iLineNumbers = 0; + int* paiLineNumbers = NULL; + int r = 0; + unsigned int j = 0; + int iReturn = 1; + + /* KSI Signature related variables */ + block_sig_t *bs = NULL; + block_hdr_t *bh = NULL; + tlvrecord_t tlvbhrec; + ksifile ksi; + uint8_t bHasRecHashes, bHasIntermedHashes; + uint8_t bInBlock; + int bInitDone = 0; + block_hashchain_t *hashchain = NULL; + KSI_DataHash *ksiRootHash = NULL; + off_t sigfilerewindPos = 0; /* Helper needed to rewind sigfile */ + + /* Create default outputfilename if needed*/ + if (outputfile == NULL) { + int iNameLength = strlen(name); + outputfile = malloc( iNameLength + 5 ); + memcpy(outputfile, name, iNameLength); + memcpy(outputfile+iNameLength, ".out", 4); + *(outputfile+iNameLength+4) = '\0'; + if (debug) printf("debug: extractKSI:\t\t\t default Outputfile: '%s' \n", outputfile); + } else { + if (debug) printf("debug: extractKSI:\t\t\t Outputfile: '%s' \n", outputfile); + } + + /* Check/set User output writemode */ + if (append > 0 ) { + writeMode[0] = 'a'; + } + + /* Count number of linenumbers */ + if (strlen(linenumbers) > 0 ) { + /* Get count of line numbers */ + char* pszTmp = linenumbers; + if (*(pszTmp) != ',') + iLineNumbers++; + + while(pszTmp != NULL) { + pszTmp = strchr(pszTmp, ','); + + if( pszTmp != NULL) { + pszTmp++; + if ( *(pszTmp) == ',' ) { + /* Invalid number, double ,, - skip char */ + pszTmp++; + } else { + iLineNumbers++; + } + } + } + if (debug) printf("debug: extractKSI:\t\t\t found '%d' linenumbers\n", iLineNumbers); + + /* Convert line numbers into int Array */ + paiLineNumbers = malloc(iLineNumbers*sizeof(int)); + int iNumPos = 0; + int iNumLength = 0; + char szTmpNum[11]; + char* pszBegin = linenumbers; + char* pszEnd = linenumbers; + while(pszBegin != NULL) { + /* Cut number from string */ + pszEnd = strchr(pszBegin, ','); + if (pszEnd != NULL ) + iNumLength = (pszEnd-pszBegin); + else /* Rest of string is last number */ + iNumLength = strlen(pszBegin); + + /* Check for valid linenumber */ + if ( iNumLength <= 0 || iNumLength > 10 ) { + fprintf(stderr, "extractKSI:\t\t\t error invalid linenumbers\n"); + r = RSGTE_IO; + goto done; + } + /* Copy into tmp string and terminate buffer */ + strncpy(szTmpNum, pszBegin, iNumLength); + szTmpNum[iNumLength] = '\0'; + + /* Process next linenumber */ + if (pszEnd != NULL ) { pszBegin = pszEnd+1; } else { pszBegin = NULL; } + /* if (debug) fprintf(stderr, "Adding linenumber: '%s' \n", szTmpNum); */ + + /* try to convert into INT now! */ + paiLineNumbers[iNumPos] = atoi(szTmpNum); /* invalid numbers will become 0 and ignored */ + if (debug) printf("debug: extractKSI:\t\t\t Adding Linenumber: '%d' \n", paiLineNumbers[iNumPos]); + iNumPos++; + } + } else { + fprintf(stderr, "extractKSI:\t\t\t error missing linenumbers to extract\n"); + r = RSGTE_IO; + goto done; + } + + /* Init KSI library */ + ksierrctx_t ectx; + rsksi_errctxInit(&ectx); + rsksiInit("rsyslog rsksiutil " VERSION); + bInitDone = 1; + /* Set defaults for KSI Signature extraction */ + ectx.verbose = verbose; + ectx.fp = stderr; + ectx.filename = strdup(sigfname); + + /* Check for valid file header in sigfile */ + if((r = rsksi_chkFileHdr(sigfp, "LOGSIG11", verbose)) != 0) { + if (debug) printf("debug: extractKSI:\t\t\t error %d in rsksi_chkFileHdr\n", r); + goto done; + } + sigfilerewindPos = ftello(sigfp); /* Store rewind position */ + + + /* Init KSIFiles for IN and OUT */ + ksi = rsksi_vrfyConstruct_gf(); + if(ksi == NULL) { + if (debug) printf("debug: extractKSI:\t\t\t error initializing signature file structures\n"); + r = RSGTE_IO; + goto done; + } + + /* Start extracting process */ + if (debug) printf("debug: extractKSI:\t\t\t extracting lines(%d) %s from %s now ...\n", iLineNumbers, linenumbers, name); + + /* Open output logfile for extracted loglines */ + if((newlogfp = fopen(outputfile, writeMode)) == NULL) { + perror(outputfile); + r = RSGTE_IO; + goto done; + } else { + if (debug) printf("debug: extractKSI:\t\t\t Output logfile %s opened with mode: '%s'\n", outputfile, writeMode); + } + + /* Open output signaturefile for extracted signatures */ + snprintf(newsigfname, sizeof(newsigfname), "%s.ksisig", outputfile); + newsigfname[sizeof(newsigfname)-1] = '\0'; + if((newsigfp = fopen(newsigfname, writeMode)) == NULL) { + perror(newsigfname); + r = RSGTE_IO; + goto done; + } else { + if (debug) printf("debug: extractKSI:\t\t\t Output sigfile %s opened with mode: '%s'\n", newsigfname, writeMode); + /* write KSI fileheader */ + if (writeMode[0] == 'w') { + if(fwrite("RECSIG11", 8, 1, newsigfp) != 1) { + perror(newsigfname); + r = RSGTE_IO; + goto done; + } + } + } + + /* Variables needed for the logfile extraction process */ + int iIndex = 0; + int iLineSearch = 0; + int iLineMax = 0; + int iLineCurrent = 0; + int bLogLineFound = 0; /* Helper variable to detect when right HASH is found! */ + int bBlockSigWritten = 0; /* Helper variable to verify if block signature is written */ + + /* Find highest possible linenumber */ + for(iIndex = 0; iIndex < iLineNumbers; iIndex++) { + if (paiLineNumbers[iIndex] > iLineMax) + iLineMax = paiLineNumbers[iIndex]; + } + + /* Main Extraction Loop Starts here */ + do + { + /* Free previous hashchain */ + if (hashchain != NULL) { + if (hashchain->rec_hash.data == NULL) free(hashchain->rec_hash.data); + for(j = 0 ; j < hashchain->stepCount ; ++j) { + if (hashchain->hashsteps[j]->sib_hash.data == NULL) free(hashchain->hashsteps[j]->sib_hash.data); + } + free(hashchain); + } + + /* Init new HashChain */ + if((hashchain = calloc(1, sizeof(block_hashchain_t))) == NULL) { + r = RSGTE_OOM; + goto done; + } + hashchain->rec_hash.data = NULL; + hashchain->stepCount = 0; + hashchain->level = 0; + + /* Get Next linenumber for extraction */ + for(iIndex = 0; iIndex < iLineNumbers; iIndex++) { + if (paiLineNumbers[iIndex] > iLineSearch) { + iLineSearch = paiLineNumbers[iIndex]; + break; + } + } + if (debug) printf("debug: extractKSI:\t\t\t Extracting line number '%d'\n", iLineSearch); + + /* Rewind LOG and SIG File for now */ + rewind(logfp); + iLineCurrent = 0; + fseeko(sigfp, sigfilerewindPos, SEEK_SET); + + /* Reset Helper variables */ + ectx.blkNum = 0; + ectx.recNumInFile = 0; + bInBlock = 0; + bLogLineFound = 0; + bBlockSigWritten = 0; + + do + { + /* Free memory from last lineRec first*/ + if (lineRec != NULL) { + free(lineRec); + lineRec = NULL; + } + + /* Get next line from file! */ + ssread = getline(&lineRec, &stlen, logfp); + if (ssread != -1) { + iLineCurrent++; /* Increment Current Line */ + } else { + /* END of file reached */ + if (debug) printf("debug: extractKSI:\t\t\t End of file reached.\n"); + r = RSGTE_EOF; + break; + } + +if (debug) printf("debug: extractKSI:\t\t\t line '%d': %.64s...\n", iLineSearch, lineRec); + + /* Extract line if correct one */ + if (iLineCurrent == iLineSearch) { + if (debug) printf("debug: extractKSI:\t\t\t Extracted line '%d': %.64s...\n", iLineSearch, lineRec); + + /* Write logline into output */ + if( (fwrite(lineRec, ssread, 1, newlogfp) != 1) /*|| + (fwrite("\n", sizeof(char), 1, newlogfp) != 1)*/ ) { + free(lineRec); + fprintf(stderr, "extractKSI:\t\t\t error '%d' while writing into output logfile %s\n", ferror(newlogfp), outputfile); + r = RSGTE_IO; + goto done; + } + + /* Found correct record, need to extract hash in next step! */ + bLogLineFound = 1; + } + + /* Remove Linefeed for verification */ + if( *(lineRec+ssread-1) == '\n') { + *(lineRec+ssread-1) = '\0'; + --ssread; + rsksi_errctxSetErrRec(&ectx, lineRec); + } + + /* Check if this is a new signature block */ + if(bInBlock == 0) { + /* Free memory */ + if(bs != NULL) rsksi_objfree(0x0904, bs); + if(bh != NULL) rsksi_objfree(0x0901, bh); + + /* Get/Verify Block Paramaters */ + if((r = rsksi_getBlockParams(ksi, sigfp, 1, &bs, &bh, &bHasRecHashes, &bHasIntermedHashes)) != 0) { + if(ectx.blkNum == 0) { + fprintf(stderr, "extractKSI:\t\t\t Error %d before finding any signature block - is the file still open and being written to?\n", r); + r = RSGTE_IO; + } else { + if(verbose) + fprintf(stderr, "extractKSI:\t\t\t EOF after signature block %lld\n", (long long unsigned) ectx.blkNum); + r = RSGTE_EOF; + } + perror(sigfname); + goto done; + } else { + if (debug) printf("debug: extractKSI:\t\t\t %ju records in Block \n", bs->recCount); + } + + /* Verify block header */ + if ((r = verifyBLOCK_HDRKSI(ksi, sigfp, NULL, &tlvbhrec)) != 0) { + perror(sigfname); + r = RSGTE_IO; + goto done; + } + + /* Init Signature Block */ + rsksi_vrfyBlkInit(ksi, bh, bHasRecHashes, bHasIntermedHashes); + ectx.recNum = 0; + ++ectx.blkNum; + } + ++ectx.recNum, ++ectx.recNumInFile; + + /* we need to preserve the first line (record) of each block for + * error-reporting purposes (bInBlock==0 meanst start of block) + */ + if(bInBlock == 0) rsksi_errctxFrstRecInBlk(&ectx, lineRec); + + /* Verify next record in signature file */ + if ((r = rsksi_vrfy_nextRecExtract(ksi, sigfp, NULL, (unsigned char*)lineRec, ssread, &ectx, hashchain, bLogLineFound)) != RSGTE_SUCCESS) { + fprintf(stderr, "extractKSI:\t\t\t error %d while verifiying next signature record for logline (%d): '%.64s...'\n", r, iLineCurrent, lineRec); + goto done; + } + + if (bLogLineFound == 1) { + if (bBlockSigWritten == 0) { + /* WRITE BLOCK Signature */ + if (debug) printf("debug: extractKSI:\t\t\t rsksi_ExtractBlockSignature #1: \n"); + if ((r = rsksi_ExtractBlockSignature(newsigfp, ksi, bs, &ectx, verbose)) != RSGTE_SUCCESS) { + fprintf(stderr, "extractKSI:\t\t\t error %d while writing block signature for (%d): '%.64s...'\n", r, iLineCurrent, lineRec); + goto done; + } + bBlockSigWritten = 1; + } + } + + if(ectx.recNum == bs->recCount) { + if (bBlockSigWritten == 1) { + /* We need additional Block Finish handling! */ + if((r = verifySigblkFinishChain(ksi, hashchain, &ksiRootHash, &ectx)) != 0) { + fprintf(stderr, "extractKSI:\t\t\t error %d while finishing BLOCK signature\n", r); + goto done; + } + } else { + /* Finish Block ! */ + if((r = verifySigblkFinish(ksi, &ksiRootHash)) != 0) { + fprintf(stderr, "extractKSI:\t\t\t error %d while finish signature BLOCK\n", r); + goto done; + } + } + + /* Verify Block signature */ + if((r = verifyBLOCK_SIGKSI(bs, ksi, sigfp, NULL, 0, ksiRootHash, &ectx)) != RSGTE_SUCCESS) { + fprintf(stderr, "extractKSI:\t\t\t error %d while verifiying BLOCK signature for logline (%d): '%.64s...'\n", r, iLineCurrent, lineRec); + goto done; + } + + /* Reset Block state variables */ + bInBlock = 0; + + if (bLogLineFound == 1 ) { + /* Write HashChain now */ + if ((r = rsksi_WriteHashChain(newsigfp, hashchain, bs, verbose)) != RSGTE_SUCCESS) { + fprintf(stderr, "extractKSI:\t\t\t error %d while starting new hash chain for (%d): '%.64s...'\n", r, iLineCurrent, lineRec); + goto done; + } + + /* Abort Loop from here and start over */ + break; + } + } else { + bInBlock = 1; + } +// } while (iLineCurrent < iLineSearch && r != RSGTE_EOF); + } while (r != RSGTE_EOF); + + /* Variables will be resetted at Loop begin */ + } while ( (iLineMax > iLineSearch || bInBlock == 1) && r != RSGTE_EOF); + +done: + /* Free mem */ + if (lineRec != NULL) + free(lineRec); + if (paiLineNumbers != NULL) + free(paiLineNumbers); + + if(r != RSGTE_EOF) { + goto done2; + } + + /* Make sure we've reached the end of file in both log and signature file */ + if (fgetc(logfp) != EOF) { + fprintf(stderr, "There are unsigned records in the end of log.\n"); + fprintf(stderr, "Last signed record: %s\n", ectx.errRec); + r = RSGTE_END_OF_SIG; + goto done2; + } + if (fgetc(sigfp) != EOF) { + fprintf(stderr, "There are records missing from the end of the log file.\n"); + r = RSGTE_END_OF_LOG; + goto done2; + } + +done2: + if(r != 0 && r != RSGTE_EOF) { + sprintf(errbuf, "extractKSI:\t\t\t error %d (%s) processing file %s\n", r, RSKSIE2String(r), name); + iReturn = 0; + } else + errbuf[0] = '\0'; + + /* Close file handles */ + if(logfp != NULL) { + fclose(logfp); + } + if(sigfp != NULL) { + fclose(sigfp); + } + if(newlogfp != NULL) { + fclose(newlogfp); + } + if(newsigfp != NULL) { + fclose(newsigfp); + } + + /* Deinit KSI stuff */ + if(bInitDone) + rsksi_errctxExit(&ectx); + return iReturn; +} #endif /* VERIFY if logfile has a Guardtime Signfile @@ -965,6 +1475,12 @@ verifyGT: verifyKSI: #ifdef ENABLEKSI + /* puburl is mandatory for KSI now! */ + if (strlen(rsksi_read_puburl) <= 0) { + iSuccess = 0; + sprintf(errbuf, "ERROR, missing --publications-server parameter is mandatory when verifying KSI signatures.\n"); + goto done; /* abort */ + } iSuccess = verifyKSI(name, errbuf, sigfname, oldsigfname, nsigfname, logfp, sigfp, nsigfp); #else iSuccess = 0; @@ -980,6 +1496,77 @@ done: return; } +/* EXTRACT loglines including their signatures from a logfile +*/ +static void +extract(char *name, char *errbuf) +{ + int iSuccess = 1; + char sigfname[4096]; + FILE *logfp = NULL, *sigfp = NULL; + + if(!strcmp(name, "-")) { + fprintf(stderr, "extract mode cannot work on stdin\n"); + goto err; + } else { + /* First check for the logfile itself */ + if((logfp = fopen(name, "r")) == NULL) { + perror(name); + goto err; + } + + /* Check for .gtsig file */ + snprintf(sigfname, sizeof(sigfname), "%s.gtsig", name); + sigfname[sizeof(sigfname)-1] = '\0'; + if((sigfp = fopen(sigfname, "r")) == NULL) { + /* Use errbuf to output error later */ + sprintf(errbuf, "%s: No such file or directory\n", sigfname); + + /* Check for .ksisig file */ + snprintf(sigfname, sizeof(sigfname), "%s.ksisig", name); + sigfname[sizeof(sigfname)-1] = '\0'; + if((sigfp = fopen(sigfname, "r")) == NULL) { + /* Output old error first*/ + fputs(errbuf, stderr); + sprintf(errbuf, "%s: No such file or directory\n", sigfname); + fputs(errbuf, stderr); + goto err; + } + goto extractKSI; + } + goto extractGT; + } + +extractGT: +#ifdef ENABLEGT + iSuccess = 0; + sprintf(errbuf, "ERROR, extract loglines from old signature files is NOT supported.\n"); +#endif + goto done; + +extractKSI: +#ifdef ENABLEKSI + /* puburl is mandatory for KSI now! */ + if (strlen(rsksi_read_puburl) <= 0) { + iSuccess = 0; + sprintf(errbuf, "ERROR, missing --publications-server parameter is mandatory when extracting KSI signatures.\n"); + goto done; /* abort */ + } + iSuccess = extractKSI(name, errbuf, sigfname, logfp, sigfp); +#else + iSuccess = 0; + sprintf(errbuf, "ERROR, unable to extract loglines from %s using GuardTime KSI library, rsyslog need to be configured with --enable-gt-ksi.\n", name); +#endif + goto done; + +err: +done: + /* Output error if return was 0*/ + if (iSuccess == 0) + fputs(errbuf, stderr); + return; +} + static void processFile(char *name) { @@ -1043,6 +1630,11 @@ processFile(char *name) fprintf(stdout, "ProcessMode: Verify/Extend\n"); verify(name, errbuf); break; + case MD_EXTRACT: + if(verbose) + fprintf(stdout, "ProcessMode: Extract"); + extract(name, errbuf); + break; } } @@ -1057,11 +1649,17 @@ static struct option long_options[] = {"version", no_argument, NULL, 'V'}, {"detect-file-type", no_argument, NULL, 'T'}, {"show-sigblock-params", no_argument, NULL, 'B'}, - {"verify", no_argument, NULL, 't'}, /* 't' as in "test signatures" */ - {"extend", no_argument, NULL, 'e'}, {"show-verified", no_argument, NULL, 's'}, {"publications-server", required_argument, NULL, 'P'}, + {"extend-server", required_argument, NULL, 'E'}, + {"userid", required_argument, NULL, 'u'}, + {"userkey", required_argument, NULL, 'k'}, {"api", required_argument, NULL, 'a'}, + {"verify", no_argument, NULL, 't'}, /* 't' as in "test signatures" */ + {"extend", no_argument, NULL, 'e'}, + {"extract", required_argument, NULL, 'x'}, + {"output", required_argument, NULL, 'o'}, + {"append", no_argument, NULL, 'A'}, {NULL, 0, NULL, 0} }; @@ -1075,6 +1673,7 @@ rsgtutil_usage(void) "\t-D, --dump \t\t\t dump operations mode.\n" "\t-t, --verify \t\t\t Verify operations mode.\n" "\t-e, --extend \t\t\t Extends the RFC3161 signatures.\n" + "\t-x, --extract \t\t\t Extract these linenumbers including signatures.\n" "\t-B, --show-sigblock-params \t Show signature block parameters.\n" "\t-T, --detect-file-type \t Show Type of signature file.\n" "\t-c, --convert \t\t\t Convert Signature Format Version 10 to 11.\n" @@ -1085,6 +1684,11 @@ rsgtutil_usage(void) "\t\tKSI = Guardtime KSI Library\n" "\t-s, --show-verified \t\t Also show correctly verified blocks.\n" "\t-P , --publications-server \t Sets the publications server.\n" + "\t-E , --extend-server \t Sets the extension server.\n" + "\t-u , --userid \t Sets the userid used (Needed for the extension server).\n" + "\t-k , --userkey \t Sets the userkey used (Needed for the extension server).\n" + "\t-o , --output \t Sets an output filename (EXTRACT Mode only).\n" + "\t-A, --append \t\t\t Append extracted output to file (EXTRACT Mode only).\n" "\t-v, --verbose \t\t\t Verbose output.\n" "\t-d, --debug \t\t\t Debug (developer) output.\n" ); @@ -1097,7 +1701,7 @@ main(int argc, char *argv[]) int opt; while(1) { - opt = getopt_long(argc, argv, "aBcdDeHPstTvV", long_options, NULL); + opt = getopt_long(argc, argv, "aABcdDeEhkoPstTuvVx", long_options, NULL); if(opt == -1) break; switch(opt) { @@ -1148,6 +1752,30 @@ main(int argc, char *argv[]) #endif #ifdef ENABLEKSI rsksi_read_puburl = optarg; +#endif + break; + case 'E': +#ifdef ENABLEGT + rsgt_extend_puburl = optarg; +#endif +#ifdef ENABLEKSI + rsksi_extend_puburl = optarg; +#endif + break; + case 'u': +#ifdef ENABLEGT + rsgt_userid = optarg; +#endif +#ifdef ENABLEKSI + rsksi_userid = optarg; +#endif + break; + case 'k': +#ifdef ENABLEGT + rsgt_userkey = optarg; +#endif +#ifdef ENABLEKSI + rsksi_userkey = optarg; #endif break; case 'T': @@ -1162,6 +1790,16 @@ main(int argc, char *argv[]) case 'c': mode = MD_CONVERT; break; + case 'x': + mode = MD_EXTRACT; + linenumbers = optarg; + break; + case 'o': + outputfile = optarg; + break; + case 'A': + append = 1; + break; case 'h': case '?': rsgtutil_usage(); diff --git a/tools/rsgtutil.rst b/tools/rsgtutil.rst index 6b2e08884..ca574c662 100644 --- a/tools/rsgtutil.rst +++ b/tools/rsgtutil.rst @@ -66,15 +66,44 @@ OPTIONS Select "conversion" mode. This converts signature files from Version 10 to 11. The original file will automatically be backed up. +-x, --extract + Extract Lines (separated by comma) from the input logfile and creates + hash chains needed to verify the integrity of the extracted records. + If no output file is specified, a new file with the same name as the + inputfile + ".out" will be created. + -v, --verbose Select verbose mode. Most importantly, hashes and signatures are printed in full length (can be **very** lengthy) rather than the usual abbreviation. +-o , --output + Sets an output filename. Optional for EXTRACT operation mode. If no output + filename is configured, rsgtutil we create a new file appending ".out". + +-A, --append + Append extracted output to an existing file. Optional for EXTRACT + operation mode. By appending to an existing outputfile, it is possible to + extract loglines from multiple input sources into one file. -P , --publications-server - Sets the publications server. If not set but required by the operation a - default server is used. The default server is not necessarily optimal - in regard to performance and reliability. + Sets the publications server used to verify the signature. + When verifying a KSI signature, the parameter is mandatory. + When verifying a GT signature, the parameter is optional. If not set but + required by the operation a default server is used. The default server + is not necessarily optimal in regard to performance and reliability. + +-E , --extend-server + Sets the extender server. Only needs to be set when using the extend mode + and a custom extend-server is needed. When extending KSI signatures, the + parameter is mandatory. + +-u , --userid + Sets the userid used for the extension server. The option only + applies in the extend mode. + +-k , --userkey + Sets the userkey used for the extension server. The option only + applies in the extend mode. -h, --help Shows short help for the utility. @@ -116,12 +145,12 @@ verify ------ This mode does not work with stdin. On the command line, the *log* file names -are specified. The corresponding *signature* files (ending on ".gtsig") must also -be preset at the same location as the log file. In verify mode, both the log -and signature file is read and the validity of the log file checked. If verification -errors are detected these are printed and processing of the file aborted. By default, -each file is verified individually, without taking cross-file hash chains into -account (so the order of files on the command line does not matter). +are specified. The corresponding *signature* files (ending on ".gtsig"/".ksisig") +must also be preset at the same location as the log file. In verify mode, both the +log and signature file is read and the validity of the log file checked. If +verification errors are detected these are printed and processing of the file +aborted. By default, each file is verified individually, without taking cross-file +hash chains into account (so the order of files on the command line does not matter). Note that the actual amount of what can be verified depends on the parameters with which the signature file was written. If record and tree hashes are present, they @@ -131,6 +160,27 @@ not present, only the block signature itself is verified. By default, only errors are printed. To also print successful verifications, use the **--show-verified** option. +convert +------- + +As the binary file format has changed between LOGSIG10 and LOGSIG11, it might be +necessary to convert old signature files in order to be able to verify them with +the current version of rsgtutil. This conversion needs to be done once, and will +automatically create a new .*sig file and backup the old file in case of success. + +extract +------- + +Extract single or multiple lines from a given input logfile. The lines have to be +separated by comma, for example ./rsgtutil --extract 1,2 security.log will +extract line 1 and 2 from security.log. +The corresponding *signature* file (ending on ".ksisig") needs to be present +at the same location and will be needed during the process of creating hash chains. +The hash chains will be written into their own .ksisig file (RECSIG11) including +the signature blocks. If no outputfile name is specified, a new file with the same +name as the inputfile appending ".out" will be created. +The --append option can be used to append results to an existing outputfile if +loglines from multiple input sources have to be combined into one extraction file. extend ------ @@ -189,6 +239,13 @@ This dumps the content of the signature file "logfile.gtsig". The actual log file is not being processed and does not even need to be present. +**rsgtutil --extract 1,2,3,4 logfile** + +This exports loglines 1, 2, 3 and 4 into a new file "logfile.out". +The actual log file combined with the signature file will be processed +to create and export corresponding hash chains. + + SEE ALSO ======== **rsyslogd(8)** diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index a2ee2bc88..a42870fca 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -3,18 +3,18 @@ * because it was either written from scratch by me (rgerhards) or * contributors who agreed to ASL 2.0. * - * Copyright 2004-2015 Rainer Gerhards and Adiscon + * Copyright 2004-2016 Rainer Gerhards and Adiscon * * This file is part of rsyslog. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -143,8 +143,8 @@ setsid(void) rsRetVal // TODO: make static queryLocalHostname(void) { - uchar *LocalHostName; - uchar *LocalDomain; + uchar *LocalHostName = NULL; + uchar *LocalDomain = NULL; uchar *LocalFQDNName; DEFiRet; @@ -163,8 +163,12 @@ queryLocalHostname(void) glbl.SetLocalHostName(LocalHostName); glbl.SetLocalDomain(LocalDomain); glbl.GenerateLocalHostNameProperty(); + LocalHostName = NULL; /* handed over */ + LocalDomain = NULL; /* handed over */ finalize_it: + free(LocalHostName); + free(LocalDomain); RETiRet; } @@ -957,13 +961,13 @@ hdlr_sigttin() } static void -hdlr_enable(int signal, void (*hdlr)()) +hdlr_enable(int sig, void (*hdlr)()) { struct sigaction sigAct; memset(&sigAct, 0, sizeof (sigAct)); sigemptyset(&sigAct.sa_mask); sigAct.sa_handler = hdlr; - sigaction(signal, &sigAct, NULL); + sigaction(sig, &sigAct, NULL); } static void @@ -1161,7 +1165,7 @@ initAll(int argc, char **argv) doFork = 0; break; case 'N': /* enable config verify mode */ - iConfigVerify = atoi(arg); + iConfigVerify = (arg == NULL) ? 0 : atoi(arg); break; case 'q': /* add hostname if DNS resolving has failed */ fprintf (stderr, "rsyslogd: the -q command line option will go away " @@ -1186,18 +1190,25 @@ initAll(int argc, char **argv) } break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ + if(arg == NULL) { + /* note this case should already be handled by getopt, + * but we want to keep the static analyzer happy. + */ + fprintf(stderr, "-T options needs a parameter\n"); + exit(1); + } if(chroot(arg) != 0) { perror("chroot"); exit(1); } break; case 'u': /* misc user settings */ - iHelperUOpt = atoi(arg); + iHelperUOpt = (arg == NULL) ? 0 : atoi(arg); if(iHelperUOpt & 0x01) { fprintf (stderr, "rsyslogd: the -u command line option will go away " "soon.\n" "For the 0x01 bit, please use the " - "global(net.parseHostnamdAndTag=\"off\") " + "global(parser.parseHostnameAndTag=\"off\") " "configuration parameter instead.\n"); glbl.SetParseHOSTNAMEandTAG(0); } @@ -1300,7 +1311,7 @@ initAll(int argc, char **argv) if(ourConf->globals.bLogStatusMsgs) { char bufStartUpMsg[512]; - snprintf(bufStartUpMsg, sizeof(bufStartUpMsg)/sizeof(char), + snprintf(bufStartUpMsg, sizeof(bufStartUpMsg), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] start", (int) glblGetOurPid()); @@ -1419,7 +1430,7 @@ doHUP(void) char buf[512]; if(ourConf->globals.bLogStatusMsgs) { - snprintf(buf, sizeof(buf) / sizeof(char), + snprintf(buf, sizeof(buf), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed", (int) glblGetOurPid()); @@ -1483,6 +1494,7 @@ static void mainloop(void) { struct timeval tvSelectTimeout; + time_t tTime; BEGINfunc /* first check if we have any internal messages queued and spit them out. */ @@ -1507,6 +1519,9 @@ mainloop(void) janitorRun(); + datetime.GetTime(&tTime); + checkGoneAwaySenders(tTime); + if(bHadHUP) { doHUP(); bHadHUP = 0; @@ -1553,7 +1568,7 @@ deinitAll(void) /* and THEN send the termination log message (see long comment above) */ if(bFinished && runConf->globals.bLogStatusMsgs) { - (void) snprintf(buf, sizeof(buf) / sizeof(char), + (void) snprintf(buf, sizeof(buf), " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"]" " exiting on signal %d.", (int) glblGetOurPid(), bFinished); diff --git a/tools/syncdemo.c b/tools/syncdemo.c index 6b76c53ee..e452ebf2c 100644 --- a/tools/syncdemo.c +++ b/tools/syncdemo.c @@ -329,7 +329,7 @@ dispRuntime(unsigned rt) { static char *fmtbuf; - fmtbuf = malloc(32 * sizeof(char)); + fmtbuf = malloc(32); snprintf(fmtbuf, 32, "%u.%03.3u", rt / 1000, rt % 1000); return(fmtbuf); diff --git a/tools/syslogd.c b/tools/syslogd.c index 3bda97dbc..2d67d891a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -168,7 +168,7 @@ char **syslogd_crunch_list(char *list) */ count = 0; while ((q=strchr(p, LIST_DELIMITER))) { - result[count] = (char *) MALLOC((q - p + 1) * sizeof(char)); + result[count] = (char *) MALLOC(q - p + 1); if (result[count] == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ @@ -179,7 +179,7 @@ char **syslogd_crunch_list(char *list) count++; } if ((result[count] = \ - (char *)MALLOC(sizeof(char) * strlen(p) + 1)) == NULL) { + (char *)MALLOC(strlen(p) + 1)) == NULL) { printf ("Sorry, can't get enough memory, exiting.\n"); exit(0); /* safe exit, because only called during startup */ }