Merge pull request #2044 from rgerhards/i-2037

Travis CI: add support for cron jobs & Coverity scan

.travis.yml

@@ -10,15 +10,21 @@ compiler:
 
 services: mysql
 
+env:
+  global:
+    secure: DsTuVAZg5omWT652Xnz/ZG20nJ5ShdTDXPQA01HscUhjAzcuMYsCPp889QoLip+3O5yYH6/oH2WprQA6ZQQMt+7idNRo6ennu7tPQiHEBm/lK2Yq1W6r6tOb1zmSalbIVWV2cGTdKOOuHAatgmp/L70WjOlCrtLvPT+JQ9AkzLU=
+
 addons:
   apt:
+    sources:
+    - sourceline: 'deb http://download.opensuse.org/repositories/network:/messaging:/zeromq:/git-draft/xUbuntu_14.04/ ./'
+      key_url: 'http://download.opensuse.org/repositories/network:/messaging:/zeromq:/git-draft/xUbuntu_12.04/Release.key'
     packages:
     - build-essential
     - automake
     - pkg-config
     - libtool
     - autoconf
-    #- autoconf-archive
     - autotools-dev
     - gdb
     - valgrind
@@ -38,6 +44,9 @@ addons:
     - openjdk-7-jdk
     - wget
     - libkrb5-dev
+    - libsodium-dev
+    - libczmq-dev
+    #- autoconf-archive
     #- libgrok1
     #- libgrok-dev
     #- faketime
@@ -49,7 +58,6 @@ matrix:
      - compiler: "gcc"
      - compiler: "clang"
   include:
-       # test for OS X
      - os: osx
        compiler: "clang"
        env: CFLAGS="-g "
@@ -93,6 +101,11 @@ matrix:
        dist: trusty
        env: DEBUGLESS="YES", CFLAGS="-g -O2 -std=c99 -Werror -W -Wall -Wextra -Wformat-security -Wshadow -Wcast-align -Wpointer-arith -Wmissing-format-attribute -Wno-unused-function"
 
+     - os: linux
+       compiler: "gcc"
+       dist: trusty
+       env: DO_COVERITY="YES",DO_CRON="YES", KAFKA="YES", GROK="YES"
+
 services:
   - elasticsearch
 

CI/check_line_length.sh

@@ -6,4 +6,7 @@ cd codestyle
 gcc --std=c99 stylecheck.c -o stylecheck
 cd ../..
 find -name "*.[ch]" | xargs _tmp_stylecheck/codestyle/stylecheck -w -f -l 130 && \
-	_tmp_stylecheck/codestyle/stylecheck plugins/imfile/imfile.c
+	_tmp_stylecheck/codestyle/stylecheck plugins/imfile/imfile.c -l 130
+# Note: we do stricter checks for some code sources that have been
+# sufficiently cleaned up. That after the "&&" part of the statement.
+rm -rf codestyle

tests/travis/install.sh

@@ -3,7 +3,7 @@ source /etc/lsb-release
 # the following packages are not yet available via travis package
 sudo apt-get install -qq faketime libdbd-mysql libmongo-client-dev autoconf-archive
 if [ "x$GROK" == "xYES" ]; then sudo apt-get install -qq libgrok1 libgrok-dev ; fi
-sudo apt-get install -qq --force-yes libestr-dev librelp-dev libfastjson-dev liblogging-stdlog-dev libksi1 libksi1-dev \
+sudo apt-get install -qq --force-yes libestr-dev librelp-dev libfastjson-dev liblogging-stdlog-dev \
 	liblognorm-dev \
 	libcurl4-gnutls-dev
 sudo apt-get install -qq python-docutils
@@ -12,7 +12,8 @@ if [ "$DISTRIB_CODENAME" == "trusty" ] || [ "$DISTRIB_CODENAME" == "precise" ];
 	set -ex
 	WANT_MAXMIND=1.2.0
 	curl -Ls https://github.com/maxmind/libmaxminddb/releases/download/${WANT_MAXMIND}/libmaxminddb-${WANT_MAXMIND}.tar.gz | tar -xz
-	(cd libmaxminddb-${WANT_MAXMIND} ; ./configure --prefix=/usr CC=gcc CFLAGS="-Wall -Wextra -g -pipe -std=gnu99"  > /dev/null ; sudo make install &> /dev/null)
+	(cd libmaxminddb-${WANT_MAXMIND} ; ./configure --prefix=/usr CC=gcc CFLAGS="-Wall -Wextra -g -pipe -std=gnu99"  > /dev/null ; make -j2  >/dev/null ; sudo make install > /dev/null)
+	rm -rf libmaxminddb-${WANT_MAXMIND} # get rid of source, e.g. for line length check
 	set +x
 else
 	sudo apt-get install -qq libmaxminddb-dev
@@ -22,7 +23,8 @@ fi
 if [ "x$KAFKA" == "xYES" ]; then 
 	sudo apt-get install -qq liblz4-dev
 	git clone https://github.com/edenhill/librdkafka
-	(unset CFLAGS; cd librdkafka ; ./configure --prefix=/usr --CFLAGS="-g" > /dev/null ; make  > /dev/null ; sudo make install > /dev/null)
+	(unset CFLAGS; cd librdkafka ; ./configure --prefix=/usr --CFLAGS="-g" > /dev/null ; make -j2  > /dev/null ; sudo make install > /dev/null)
+	rm -rf librdkafka # get rid of source, e.g. for line length check
 fi
 #if [ "x$KAFKA" == "xYES" ]; then sudo apt-get install -qq librdkafka-dev ; fi
 

tests/travis/run-cron.sh

@@ -0,0 +1,49 @@
+# this perform a travis cron job
+echo Travis under cron detected, currently no action!
+#set -x
+
+# download coverity tool
+mkdir coverity
+cd coverity
+wget --no-verbose http://build.rsyslog.com/CI/cov-analysis.tar.gz
+if [ $? -ne 0 ]; then
+	echo Download Coverity analysis tool failed!
+	exit 1
+fi
+tar xzf cov*.tar.gz
+rm -f cov*.tar.gz
+export PATH="coverity/$(ls -d cov*)/bin:$PATH"
+cd ..
+# Coverity scan tool installed
+
+# we need Guardtime libksi here, otherwise we cannot check the KSI component
+git clone https://github.com/guardtime/libksi.git
+cd libksi
+autoreconf -fvi
+./configure --prefix=/usr
+make -j
+sudo make install
+cd ..
+
+# prep rsyslog for submission
+autoreconf -vfi
+# explicit ./configure as this needs to be consistent accross all builds
+# for Coverity
+./configure -enable-testbench --enable-imdiag --enable-imfile --enable-impstats --enable-imptcp --enable-mmanon --enable-mmaudit --enable-mmfields --enable-mmjsonparse --enable-mmpstrucdata --enable-mmsequence --enable-mmutf8fix --enable-mail --enable-omprog --enable-omruleset --enable-omstdout --enable-omuxsock --enable-pmaixforwardedfrom --enable-pmciscoios --enable-pmcisconames --enable-pmlastmsg --enable-pmsnare --enable-libgcrypt --enable-mmnormalize --disable-omudpspoof --enable-relp=yes --disable-snmp --disable-mmsnmptrapd --enable-gnutls --enable-mysql=yes --enable-usertools=yes --enable-ksi-ls12 --enable-libdbi --enable-pgsql --enable-omhttpfs --enable-elasticsearch --enable-valgrind --enable-ommongodb --enable-omamqp1=no --enable-imjournal --enable-omjournal --enable-compile-warnings=error --enable-testbench --enable-compile-warnings=yes --without-valgrind-testbench --enable-omrelp-default-port=13515 --disable-liblogging-stdlog --enable-mmrm1stspace -enable-omkafka --enable-imkafka --enable-mmdblookup --enable-omczmq --enable-imczmq --enable-mmcount --enable-ommongodb
+make clean
+cov-build --dir cov-int make -j4
+tar czf rsyslog.tgz cov-int
+ls -l rsyslog.tgz
+# we make this FAIL to not thrash our allowance if things go wrong!
+curl --form token=$COVERITY_TOKEN \
+  --form email=rgerhards@adiscon.com \
+  --form file=@rsyslog.tgz \
+  --form version="master branch head" \
+  --form description="$(git log -1|head -1)" \
+  https://scan.coverity.com/builds?project=rsyslog%2Frsyslog
+CURL_RESULT=$?
+echo curl returned $CURL_RESULT
+if [ $CURL_RESULT -ne 0 ]; then
+	echo Upload to Coverity failed, curl returned $CURL_RESULT
+	exit 1
+fi

tests/travis/run.sh

@@ -8,16 +8,39 @@ set -e  # abort on first failure
 echo "DISTRIB_CODENAME: $DISTRIB_CODENAME"
 echo "CLANG:            $CLANG"
 
+echo "****************************** BEGIN ACTUAL SCRIPT STEP ******************************"
+source tests/travis/install.sh
+source /etc/lsb-release
+
+# first handle cron builds (most importantly Coverity)
+
+#if [ "$DO_COVERITY" == "YES" ]; then
+#	source tests/travis/run-cron.sh
+#exit
+#fi
+
+# cron job?
+if [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then
+	if [ "$DO_CRON" == "YES" ]; then
+		source tests/travis/run-cron.sh
+	fi
+	exit
+fi
+if [ "$DO_CRON" == "YES" ]; then
+	echo cron job not executed under non-cron run
+	exit 0 # this must not run under PRs
+fi
+
 # first check code style. We do this only when STAT_AN is enabled,
 # so that we do not do it in each and every run. While once is sufficient,
 # STAT_AN for now gives us sufficient runtime reduction.
 if [ "x$STAT_AN" == "xYES" ] ; then CI/check_line_length.sh ; fi
 
 
-echo "****************************** BEGIN ACTUAL SCRIPT STEP ******************************"
-source tests/travis/install.sh
-source /etc/lsb-release
-
+#
+# ACTUAL MAIN CI PART OF THE SCRIPT
+# This is to be executed for each PR
+#
 
 # we turn off leak sanitizer at this time because it reports some
 # pretty irrelevant problems in startup code. In the longer term,