bugfix core: potential hang on rsyslog termination

The root cause was a deadlock during worker startup. This could
happen for example when a DA queue needed to persist data during
shutdown.

Fail condition:
* startup request for a new worker
* initialization of that worker
* immediate detection that the worker can or must shutdown
* main thread waiting for worker running state, which it skips,
  and so the main thread hangs inside a loop

closes https://github.com/rsyslog/rsyslog/issues/3094

runtime/queue.c

@@ -12,7 +12,7 @@
  * function names - this makes it really hard to read and does not provide much
  * benefit, at least I (now) think so...
  *
- * Copyright 2008-2017 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2008-2018 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of the rsyslog runtime library.
  *

runtime/wtp.c

@@ -480,7 +480,10 @@ wtpStartWrkr(wtp_t *pThis)
 	 */
 	do {
 		d_pthread_cond_wait(&pThis->condThrdInitDone, &pThis->mutWtp);
-	} while(wtiGetState(pWti) != WRKTHRD_RUNNING);
+	} while((iState = wtiGetState(pWti)) == WRKTHRD_INITIALIZING);
+	DBGPRINTF("%s: new worker finished initialization with state %d, num workers now %d\n",
+		wtpGetDbgHdr(pThis), iState,
+		ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd));
 
 finalize_it:
 	d_pthread_mutex_unlock(&pThis->mutWtp);

tests/diag.sh

@@ -552,6 +552,7 @@ function wait_shutdown() {
 		   echo "manual cleanup."
 		   echo "TRYING TO capture status via gdb from hanging process"
 		   gdb ../tools/rsyslogd <<< "attach $out_pid
+set pagination off
 inf thr
 thread apply all bt
 quit"