rsyslog/rsyslog
1
0
Fork 0
mirror of https://github.com/rsyslog/rsyslog.git synced 2026-06-19 14:02:56 +02:00
Code Issues Packages Projects Releases Wiki Activity
rsyslog/runtime/lookup.c
Rainer Gerhards da0b5b8baa
lookup: report active table reloads as pending 
Why:
The de-flake campaign exposed lookup-table reload tests that could resume
after HUP while the reload worker was still installing the replacement
table. The wait helper saw no pending reload and injected messages against
stale or stub lookup state.

Impact:
Lookup reload waiters now observe the full reload lifecycle.

Before/After:
Before, only queued reload requests were pending; after, an active reload
also remains pending until the table swap completes.

Technical Overview:
Track the interval after the reloader consumes do_reload but before
lookupDoReload() returns. lookupPendingReloadCount() now treats that
interval as pending, so imdiag's AwaitLookupTableReload command cannot
return while a reload is still applying. Initialize and clear the new state
alongside the existing reloader flags to keep startup, activation, and
shutdown state consistent.

Validation:
- ./autogen.sh --enable-debug --enable-testbench --enable-imdiag --enable-omstdout
- make -j$(nproc) check TESTS=""
- ./tests/array_lookup_table.sh
- ./tests/lookup_table_bad_configs.sh
- git diff --check
- Ubuntu 26.04 dev container focused lookup reload run, 9/9 pass:
  array_lookup_table.sh array_lookup_table-vg.sh
  array_lookup_table_misuse-vg.sh lookup_table_bad_configs.sh
  lookup_table_bad_configs-vg.sh lookup_table_rscript_reload.sh
  lookup_table_rscript_reload-vg.sh
  lookup_table_rscript_reload_without_stub.sh
  lookup_table_rscript_reload_without_stub-vg.sh

With the help of AI-Agents: OpenAI Codex
2026-05-27 16:16:54 +02:00

1248 lines
43 KiB
C
Raw Blame History

/* lookup.c
* Support for lookup tables in RainerScript.
*
* Copyright 2013-2023 Adiscon GmbH.
*
* This file is part of the rsyslog runtime library.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* -or-
* see COPYING.ASL20 in the source distribution
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <json.h>
#include <assert.h>
#include "rsyslog.h"
#include "srUtils.h"
#include "errmsg.h"
#include "lookup.h"
#include "msg.h"
#include "rsconf.h"
#include "dirty.h"
#include "unicode-helper.h"
#include "regexp.h"
PRAGMA_IGNORE_Wdeprecated_declarations
/* definitions for objects we access */
DEFobjStaticHelpers;
DEFobjCurrIf(glbl)
#ifdef FEATURE_REGEXP
DEFobjCurrIf(regexp)
#endif
/* forward definitions */
static rsRetVal lookupReadFile(lookup_t *pThis, const uchar *name, const uchar *filename);
static void lookupDestruct(lookup_t *pThis);
/* static data */
/* tables for interfacing with the v6 config system (as far as we need to) */
static struct cnfparamdescr modpdescr[] = {{"name", eCmdHdlrString, CNFPARAM_REQUIRED},
{"file", eCmdHdlrString, CNFPARAM_REQUIRED},
{"reloadOnHUP", eCmdHdlrBinary, 0}};
static struct cnfparamblk modpblk = {CNFPARAMBLK_VERSION, sizeof(modpdescr) / sizeof(struct cnfparamdescr), modpdescr};
/* internal data-types */
typedef struct uint32_index_val_s {
uint32_t index;
uchar *val;
} uint32_index_val_t;
const char *reloader_prefix = "lkp:";
static void *lookupTableReloader(void *self);
static void lookupStopReloader(lookup_ref_t *pThis);
#ifdef HAVE_PTHREAD_SETNAME_NP
static void lookupSetReloaderThreadName(const lookup_ref_t *const pThis) {
#if defined(__NetBSD__) || defined(__APPLE__)
char *reloader_thd_name = NULL;
int thd_name_len;
#else
char reloader_thd_name[16];
#endif
int ret;
#if defined(__NetBSD__) || defined(__APPLE__)
thd_name_len = asprintf(&reloader_thd_name, "%s%s", reloader_prefix, (char *)pThis->name);
if (thd_name_len < 0) {
return;
}
#else
snprintf(reloader_thd_name, sizeof(reloader_thd_name), "%s%s", reloader_prefix, (char *)pThis->name);
#endif
#if defined(__NetBSD__)
ret = pthread_setname_np(pthread_self(), "%s", reloader_thd_name);
#elif defined(__APPLE__)
ret = pthread_setname_np(reloader_thd_name);
#else
ret = pthread_setname_np(pthread_self(), reloader_thd_name);
#endif
if (ret != 0) {
DBGPRINTF("pthread_setname_np failed, not setting lookup reloader thread name for '%s'\n", reloader_thd_name);
}
#if defined(__NetBSD__) || defined(__APPLE__)
free(reloader_thd_name);
#endif
}
#endif
static void lookupAppend(lookup_ref_t *pThis) {
pThis->next = NULL;
if (loadConf->lu_tabs.root == NULL) {
loadConf->lu_tabs.root = pThis;
} else {
loadConf->lu_tabs.last->next = pThis;
}
loadConf->lu_tabs.last = pThis;
}
/* create a new lookup table object */
static rsRetVal lookupNew(lookup_ref_t **ppThis) {
lookup_ref_t *pThis = NULL;
lookup_t *t = NULL;
DEFiRet;
CHKmalloc(pThis = calloc(1, sizeof(lookup_ref_t)));
CHKmalloc(t = calloc(1, sizeof(lookup_t)));
pThis->do_reload = pThis->is_reloading = pThis->do_stop = 0;
pThis->reload_on_hup = 1; /*DO reload on HUP (default)*/
pThis->next = NULL;
pThis->self = t;
*ppThis = pThis;
finalize_it:
if (iRet != RS_RET_OK) {
LogError(errno, iRet, "a lookup table could not be initialized");
free(t);
free(pThis);
}
RETiRet;
}
/* activate a lookup table entry once rsyslog is ready to do so */
static rsRetVal lookupActivateTable(lookup_ref_t *pThis) {
DEFiRet;
int initialized = 0;
DBGPRINTF("lookupActivateTable called\n");
CHKiConcCtrl(pthread_rwlock_init(&pThis->rwlock, NULL));
pThis->rwlock_initialized = 1;
initialized++; /*1*/
CHKiConcCtrl(pthread_mutex_init(&pThis->reloader_mut, NULL));
pThis->reloader_mut_initialized = 1;
initialized++; /*2*/
CHKiConcCtrl(pthread_cond_init(&pThis->run_reloader, NULL));
pThis->run_reloader_initialized = 1;
initialized++; /*3*/
CHKiConcCtrl(pthread_attr_init(&pThis->reloader_thd_attr));
pThis->reloader_attr_initialized = 1;
initialized++; /*4*/
pThis->do_reload = pThis->is_reloading = pThis->do_stop = 0;
CHKiConcCtrl(pthread_create(&pThis->reloader, &pThis->reloader_thd_attr, lookupTableReloader, pThis));
pThis->reloader_started = 1;
initialized++; /*5*/
finalize_it:
if (iRet != RS_RET_OK) {
LogError(errno, iRet,
"a lookup table could not be activated: "
"failed at init-step %d (please enable debug logs for details)",
initialized);
if (pThis->reloader_started) {
lookupStopReloader(pThis);
pThis->reloader_started = 0;
}
if (pThis->reloader_attr_initialized) {
pthread_attr_destroy(&pThis->reloader_thd_attr);
pThis->reloader_attr_initialized = 0;
}
if (pThis->run_reloader_initialized) {
pthread_cond_destroy(&pThis->run_reloader);
pThis->run_reloader_initialized = 0;
}
if (pThis->reloader_mut_initialized) {
pthread_mutex_destroy(&pThis->reloader_mut);
pThis->reloader_mut_initialized = 0;
}
if (pThis->rwlock_initialized) {
pthread_rwlock_destroy(&pThis->rwlock);
pThis->rwlock_initialized = 0;
}
}
RETiRet;
}
/*must be called with reloader_mut acquired*/
static void ATTR_NONNULL() freeStubValueForReloadFailure(lookup_ref_t *const pThis) {
if (pThis->stub_value_for_reload_failure != NULL) {
free(pThis->stub_value_for_reload_failure);
pThis->stub_value_for_reload_failure = NULL;
}
}
static void lookupStopReloader(lookup_ref_t *pThis) {
pthread_mutex_lock(&pThis->reloader_mut);
freeStubValueForReloadFailure(pThis);
pThis->do_reload = 0;
pThis->is_reloading = 0;
pThis->do_stop = 1;
pthread_cond_signal(&pThis->run_reloader);
pthread_mutex_unlock(&pThis->reloader_mut);
pthread_join(pThis->reloader, NULL);
}
static void lookupRefDestruct(lookup_ref_t *pThis) {
if (pThis->reloader_started) {
lookupStopReloader(pThis);
pThis->reloader_started = 0;
}
if (pThis->reloader_mut_initialized) pthread_mutex_destroy(&pThis->reloader_mut);
if (pThis->run_reloader_initialized) pthread_cond_destroy(&pThis->run_reloader);
if (pThis->reloader_attr_initialized) pthread_attr_destroy(&pThis->reloader_thd_attr);
if (pThis->rwlock_initialized) pthread_rwlock_destroy(&pThis->rwlock);
lookupDestruct(pThis->self);
free(pThis->name);
free(pThis->filename);
free(pThis);
}
static void destructTable_str(lookup_t *pThis) {
uint32_t i = 0;
lookup_string_tab_entry_t *entries = pThis->table.str->entries;
for (i = 0; i < pThis->nmemb; i++) {
free(entries[i].key);
}
free(entries);
free(pThis->table.str);
}
static void destructTable_arr(lookup_t *pThis) {
free(pThis->table.arr->interned_val_refs);
free(pThis->table.arr);
}
static void destructTable_sparseArr(lookup_t *pThis) {
free(pThis->table.sprsArr->entries);
free(pThis->table.sprsArr);
}
#ifdef FEATURE_REGEXP
static void destructTable_regex(lookup_t *pThis) {
uint32_t i;
int regfree_available = 0;
lookup_regex_tab_entry_t *entries;
if (pThis == NULL || pThis->table.regex == NULL) return;
entries = pThis->table.regex->entries;
if (objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) regfree_available = 1;
if (entries != NULL) {
for (i = 0; i < pThis->nmemb; ++i) {
if (entries[i].is_compiled) {
if (regfree_available) {
regexp.regfree(&entries[i].regex);
} else {
LogError(0, RS_RET_INTERNAL_ERROR, "regexp module missing; compiled regex cannot be freed");
}
}
free(entries[i].regex_str);
entries[i].regex_str = NULL;
}
}
free(entries);
free(pThis->table.regex);
}
#endif
static void lookupDestruct(lookup_t *pThis) {
uint32_t i;
if (pThis == NULL) return;
if (pThis->type == STRING_LOOKUP_TABLE) {
destructTable_str(pThis);
} else if (pThis->type == ARRAY_LOOKUP_TABLE) {
destructTable_arr(pThis);
} else if (pThis->type == SPARSE_ARRAY_LOOKUP_TABLE) {
destructTable_sparseArr(pThis);
#ifdef FEATURE_REGEXP
} else if (pThis->type == REGEX_LOOKUP_TABLE) {
destructTable_regex(pThis);
#endif
} else if (pThis->type == STUBBED_LOOKUP_TABLE) {
/*nothing to be done*/
}
for (i = 0; i < pThis->interned_val_count; i++) {
free(pThis->interned_vals[i]);
}
free(pThis->interned_vals);
free(pThis->nomatch);
free(pThis);
}
static void __attribute__((noinline)) lookupRefFreeUnlinked(lookup_ref_t *pThis) {
if (pThis == NULL) return;
lookupDestruct(pThis->self);
free(pThis->name);
free(pThis->filename);
free(pThis);
}
static void __attribute__((noinline)) lookupRefDropTable(lookup_ref_t *pThis) {
lookupDestruct(pThis->self);
pThis->self = NULL;
}
void lookupInitCnf(lookup_tables_t *lu_tabs) {
lu_tabs->root = NULL;
lu_tabs->last = NULL;
}
void lookupDestroyCnf(void) {
lookup_ref_t *luref, *luref_next;
for (luref = runConf->lu_tabs.root; luref != NULL;) {
luref_next = luref->next;
lookupRefDestruct(luref);
luref = luref_next;
}
}
/* comparison function for qsort() */
static int qs_arrcmp_strtab(const void *s1, const void *s2) {
return ustrcmp(((lookup_string_tab_entry_t *)s1)->key, ((lookup_string_tab_entry_t *)s2)->key);
}
static int qs_arrcmp_ustrs(const void *s1, const void *s2) {
return ustrcmp(*(uchar **)s1, *(uchar **)s2);
}
static int qs_arrcmp_uint32_index_val(const void *s1, const void *s2) {
uint32_t first_value = ((uint32_index_val_t *)s1)->index;
uint32_t second_value = ((uint32_index_val_t *)s2)->index;
if (first_value < second_value) {
return -1;
}
if (first_value > second_value) {
return 1;
}
return 0;
}
static int qs_arrcmp_sprsArrtab(const void *s1, const void *s2) {
uint32_t first_value = ((lookup_sparseArray_tab_entry_t *)s1)->key;
uint32_t second_value = ((lookup_sparseArray_tab_entry_t *)s2)->key;
if (first_value < second_value) {
return -1;
}
if (first_value > second_value) {
return 1;
}
return 0;
}
/* comparison function for bsearch() and string array compare
* this is for the string lookup table type
*/
static int bs_arrcmp_strtab(const void *s1, const void *s2) {
return strcmp((char *)s1, (char *)((lookup_string_tab_entry_t *)s2)->key);
}
static int bs_arrcmp_str(const void *s1, const void *s2) {
return ustrcmp((uchar *)s1, *(uchar **)s2);
}
static int bs_arrcmp_sprsArrtab(const void *s1, const void *s2) {
uint32_t key = *(uint32_t *)s1;
uint32_t array_member_value = ((lookup_sparseArray_tab_entry_t *)s2)->key;
if (key < array_member_value) {
return -1;
}
if (key > array_member_value) {
return 1;
}
return 0;
}
static inline const char *defaultVal(lookup_t *pThis) {
return (pThis->nomatch == NULL) ? "" : (const char *)pThis->nomatch;
}
/* lookup_fn for different types of tables */
static es_str_t *lookupKey_stub(lookup_t *pThis, lookup_key_t __attribute__((unused)) key) {
return es_newStrFromCStr((char *)pThis->nomatch, ustrlen(pThis->nomatch));
}
static es_str_t *lookupKey_str(lookup_t *pThis, lookup_key_t key) {
lookup_string_tab_entry_t *entry;
const char *r;
if (pThis->nmemb == 0) {
entry = NULL;
} else {
assert(pThis->table.str->entries);
entry = bsearch(key.k_str, pThis->table.str->entries, pThis->nmemb, sizeof(lookup_string_tab_entry_t),
bs_arrcmp_strtab);
}
if (entry == NULL) {
r = defaultVal(pThis);
} else {
r = (const char *)entry->interned_val_ref;
}
return es_newStrFromCStr(r, strlen(r));
}
static es_str_t *lookupKey_arr(lookup_t *pThis, lookup_key_t key) {
const char *r;
uint32_t uint_key = key.k_uint;
if ((pThis->nmemb == 0) || (uint_key < pThis->table.arr->first_key)) {
r = defaultVal(pThis);
} else {
uint32_t idx = uint_key - pThis->table.arr->first_key;
if (idx >= pThis->nmemb) {
r = defaultVal(pThis);
} else {
r = (char *)pThis->table.arr->interned_val_refs[idx];
}
}
return es_newStrFromCStr(r, strlen(r));
}
typedef int(comp_fn_t)(const void *s1, const void *s2);
static void *bsearch_lte(const void *key, const void *base, size_t nmemb, size_t size, comp_fn_t *comp_fn) {
size_t l, u, idx;
const void *p;
int comparison;
l = 0;
u = nmemb;
if (l == u) {
return NULL;
}
while (l < u) {
idx = (l + u) / 2;
p = (void *)(((const char *)base) + (idx * size));
comparison = (*comp_fn)(key, p);
if (comparison < 0)
u = idx;
else if (comparison > 0)
l = idx + 1;
else
return (void *)p;
}
if (comparison < 0) {
if (idx == 0) {
return NULL;
}
idx--;
}
return (void *)(((const char *)base) + (idx * size));
}
static es_str_t *lookupKey_sprsArr(lookup_t *pThis, lookup_key_t key) {
lookup_sparseArray_tab_entry_t *entry;
const char *r;
if (pThis->nmemb == 0) {
entry = NULL;
} else {
entry = bsearch_lte(&key.k_uint, pThis->table.sprsArr->entries, pThis->nmemb,
sizeof(lookup_sparseArray_tab_entry_t), bs_arrcmp_sprsArrtab);
}
if (entry == NULL) {
r = defaultVal(pThis);
} else {
r = (const char *)entry->interned_val_ref;
}
return es_newStrFromCStr(r, strlen(r));
}
#ifdef FEATURE_REGEXP
static es_str_t *lookupKey_regex(lookup_t *pThis, lookup_key_t key) {
const char *r = defaultVal(pThis);
for (uint32_t i = 0; i < pThis->nmemb; ++i) {
if (regexp.regexec(&pThis->table.regex->entries[i].regex, (char *)key.k_str, 0, NULL, 0) == 0) {
r = (const char *)pThis->table.regex->entries[i].interned_val_ref;
break;
}
}
return es_newStrFromCStr(r, strlen(r));
}
#endif
/* builders for different table-types */
#define NO_INDEX_ERROR(type, name) \
LogError(0, RS_RET_INVALID_VALUE, \
"'%s' lookup table named: '%s' has record(s) without 'index' " \
"field", \
type, name); \
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
static rsRetVal build_StringTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
uint32_t i;
struct json_object *jrow, *jindex, *jvalue;
uchar *value, *canonicalValueRef;
DEFiRet;
pThis->table.str = NULL;
CHKmalloc(pThis->table.str = calloc(1, sizeof(lookup_string_tab_t)));
if (pThis->nmemb > 0) {
CHKmalloc(pThis->table.str->entries = calloc(pThis->nmemb, sizeof(lookup_string_tab_entry_t)));
for (i = 0; i < pThis->nmemb; i++) {
jrow = json_object_array_get_idx(jtab, i);
fjson_object_object_get_ex(jrow, "index", &jindex);
fjson_object_object_get_ex(jrow, "value", &jvalue);
if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
NO_INDEX_ERROR("string", name);
}
CHKmalloc(pThis->table.str->entries[i].key = ustrdup((uchar *)json_object_get_string(jindex)));
value = (uchar *)json_object_get_string(jvalue);
uchar **found = (uchar **)bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *),
bs_arrcmp_str);
if (found == NULL) {
LogError(0, RS_RET_INTERNAL_ERROR,
"lookup.c:build_StringTable(): "
"internal error, bsearch returned NULL for '%s'",
value);
ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
}
// I give up, I see no way to remove false positive -- rgerhards, 2017-10-24
#ifndef __clang_analyzer__
canonicalValueRef = *found;
if (canonicalValueRef == NULL) {
LogError(0, RS_RET_INTERNAL_ERROR,
"lookup.c:build_StringTable(): "
"internal error, canonicalValueRef returned from bsearch "
"is NULL for '%s'",
value);
ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
}
pThis->table.str->entries[i].interned_val_ref = canonicalValueRef;
#endif
}
qsort(pThis->table.str->entries, pThis->nmemb, sizeof(lookup_string_tab_entry_t), qs_arrcmp_strtab);
}
pThis->lookup = lookupKey_str;
pThis->key_type = LOOKUP_KEY_TYPE_STRING;
finalize_it:
RETiRet;
}
static rsRetVal build_ArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
uint32_t i;
struct json_object *jrow, *jindex, *jvalue;
uchar *canonicalValueRef;
uint32_t prev_index, _index;
uint8_t prev_index_set;
uint32_index_val_t *indexes = NULL;
DEFiRet;
prev_index_set = 0;
pThis->table.arr = NULL;
CHKmalloc(pThis->table.arr = calloc(1, sizeof(lookup_array_tab_t)));
if (pThis->nmemb > 0) {
CHKmalloc(indexes = calloc(pThis->nmemb, sizeof(uint32_index_val_t)));
CHKmalloc(pThis->table.arr->interned_val_refs = calloc(pThis->nmemb, sizeof(uchar *)));
for (i = 0; i < pThis->nmemb; i++) {
jrow = json_object_array_get_idx(jtab, i);
fjson_object_object_get_ex(jrow, "index", &jindex);
fjson_object_object_get_ex(jrow, "value", &jvalue);
if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
NO_INDEX_ERROR("array", name);
}
int64_t index_val = json_object_get_int64(jindex);
if (index_val < 0 || index_val > UINT32_MAX) {
LogError(0, RS_RET_INVALID_VALUE,
"'array' lookup table name: '%s' has index '%lld' outside uint32 range", name,
(long long)index_val);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
indexes[i].index = (uint32_t)index_val;
indexes[i].val = (uchar *)json_object_get_string(jvalue);
}
qsort(indexes, pThis->nmemb, sizeof(uint32_index_val_t), qs_arrcmp_uint32_index_val);
for (i = 0; i < pThis->nmemb; i++) {
_index = indexes[i].index;
if (prev_index_set == 0) {
prev_index = _index;
prev_index_set = 1;
pThis->table.arr->first_key = _index;
} else {
if (_index != ++prev_index) {
LogError(0, RS_RET_INVALID_VALUE,
"'array' lookup table name: '%s' "
"has non-contiguous members between index '%d' and '%d'",
name, prev_index, _index);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
}
uchar *const *const canonicalValueRef_ptr = bsearch(
indexes[i].val, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
if (canonicalValueRef_ptr == NULL) {
LogError(0, RS_RET_ERR,
"BUG: canonicalValueRef not found in "
"build_ArrayTable(), %s:%d",
__FILE__, __LINE__);
ABORT_FINALIZE(RS_RET_ERR);
}
canonicalValueRef = *canonicalValueRef_ptr;
assert(canonicalValueRef != NULL);
pThis->table.arr->interned_val_refs[i] = canonicalValueRef;
}
}
pThis->lookup = lookupKey_arr;
pThis->key_type = LOOKUP_KEY_TYPE_UINT;
finalize_it:
free(indexes);
RETiRet;
}
static rsRetVal build_SparseArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
uint32_t i;
struct json_object *jrow, *jindex, *jvalue;
uchar *value, *canonicalValueRef;
DEFiRet;
pThis->table.str = NULL;
CHKmalloc(pThis->table.sprsArr = calloc(1, sizeof(lookup_sparseArray_tab_t)));
if (pThis->nmemb > 0) {
CHKmalloc(pThis->table.sprsArr->entries = calloc(pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t)));
for (i = 0; i < pThis->nmemb; i++) {
jrow = json_object_array_get_idx(jtab, i);
fjson_object_object_get_ex(jrow, "index", &jindex);
fjson_object_object_get_ex(jrow, "value", &jvalue);
if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
NO_INDEX_ERROR("sparseArray", name);
}
int64_t index_val = json_object_get_int64(jindex);
if (index_val < 0 || index_val > UINT32_MAX) {
LogError(0, RS_RET_INVALID_VALUE,
"'sparseArray' lookup table name: '%s' has index '%lld' outside uint32 range", name,
(long long)index_val);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
pThis->table.sprsArr->entries[i].key = (uint32_t)index_val;
value = (uchar *)json_object_get_string(jvalue);
uchar *const *const canonicalValueRef_ptr =
bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
if (canonicalValueRef_ptr == NULL) {
LogError(0, RS_RET_ERR,
"BUG: canonicalValueRef not found in "
"build_SparseArrayTable(), %s:%d",
__FILE__, __LINE__);
ABORT_FINALIZE(RS_RET_ERR);
}
canonicalValueRef = *canonicalValueRef_ptr;
assert(canonicalValueRef != NULL);
pThis->table.sprsArr->entries[i].interned_val_ref = canonicalValueRef;
}
qsort(pThis->table.sprsArr->entries, pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t),
qs_arrcmp_sprsArrtab);
}
pThis->lookup = lookupKey_sprsArr;
pThis->key_type = LOOKUP_KEY_TYPE_UINT;
finalize_it:
RETiRet;
}
#ifdef FEATURE_REGEXP
static rsRetVal build_RegexTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
uint32_t i;
struct json_object *jrow, *jregex, *jtag;
uchar *value, *canonicalValueRef;
const char *regex_str;
DEFiRet;
pThis->table.regex = NULL;
CHKmalloc(pThis->table.regex = calloc(1, sizeof(lookup_regex_tab_t)));
if (pThis->nmemb > 0) {
CHKmalloc(pThis->table.regex->entries = calloc(pThis->nmemb, sizeof(lookup_regex_tab_entry_t)));
for (i = 0; i < pThis->nmemb; i++) {
jrow = json_object_array_get_idx(jtab, i);
fjson_object_object_get_ex(jrow, "regex", &jregex);
fjson_object_object_get_ex(jrow, "tag", &jtag);
if (jregex == NULL || jtag == NULL || json_object_is_type(jregex, json_type_null) ||
json_object_is_type(jtag, json_type_null)) {
LogError(0, RS_RET_INVALID_VALUE,
"'regex' lookup table named: '%s' has record(s) without required "
"fields",
name);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
regex_str = json_object_get_string(jregex);
CHKmalloc(pThis->table.regex->entries[i].regex_str = ustrdup((const uchar *)regex_str));
value = (uchar *)json_object_get_string(jtag);
uchar *const *const canonicalValueRef_ptr =
bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
if (canonicalValueRef_ptr == NULL) {
LogError(0, RS_RET_ERR, "BUG: canonicalValueRef not found in build_RegexTable(), %s:%d", __FILE__,
__LINE__);
ABORT_FINALIZE(RS_RET_ERR);
}
canonicalValueRef = *canonicalValueRef_ptr;
assert(canonicalValueRef != NULL);
pThis->table.regex->entries[i].interned_val_ref = canonicalValueRef;
int err;
if (objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
if ((err = regexp.regcomp(&pThis->table.regex->entries[i].regex, regex_str,
REG_EXTENDED | REG_NOSUB)) != 0) {
char errbuf[256];
regexp.regerror(err, &pThis->table.regex->entries[i].regex, errbuf, sizeof(errbuf));
LogError(0, RS_RET_INVALID_VALUE, "error compiling regex '%s' in lookup table '%s': %s", regex_str,
name, errbuf);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
} else {
pThis->table.regex->entries[i].is_compiled = 1;
}
} else {
LogError(0, RS_RET_INTERNAL_ERROR, "regex library could not be loaded");
ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
}
}
}
pThis->lookup = lookupKey_regex;
pThis->key_type = LOOKUP_KEY_TYPE_STRING;
finalize_it:
RETiRet;
}
#endif
static rsRetVal lookupBuildStubbedTable(lookup_t *pThis, const uchar *stub_val) {
DEFiRet;
CHKmalloc(pThis->nomatch = ustrdup(stub_val));
pThis->lookup = lookupKey_stub;
pThis->type = STUBBED_LOOKUP_TABLE;
pThis->key_type = LOOKUP_KEY_TYPE_NONE;
finalize_it:
RETiRet;
}
static rsRetVal lookupBuildTable_v1(lookup_t *pThis, struct json_object *jroot, const uchar *name) {
struct json_object *jnomatch, *jtype, *jtab;
struct json_object *jrow, *jvalue;
const char *table_type, *nomatch_value;
const char *value_key;
const uchar **all_values;
const uchar *curr, *prev;
uint32_t i, j;
uint32_t uniq_values;
DEFiRet;
all_values = NULL;
fjson_object_object_get_ex(jroot, "nomatch", &jnomatch);
fjson_object_object_get_ex(jroot, "type", &jtype);
fjson_object_object_get_ex(jroot, "table", &jtab);
if (jtab == NULL || !json_object_is_type(jtab, json_type_array)) {
LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' has invalid table definition", name);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
pThis->nmemb = json_object_array_length(jtab);
table_type = json_object_get_string(jtype);
if (table_type == NULL) {
table_type = "string";
}
value_key = (strcmp(table_type, "regex") == 0) ? "tag" : "value";
CHKmalloc(all_values = malloc(pThis->nmemb * sizeof(uchar *)));
/* before actual table can be loaded, prepare all-value list and remove duplicates*/
for (i = 0; i < pThis->nmemb; i++) {
jrow = json_object_array_get_idx(jtab, i);
fjson_object_object_get_ex(jrow, value_key, &jvalue);
if (jvalue == NULL || json_object_is_type(jvalue, json_type_null)) {
LogError(0, RS_RET_INVALID_VALUE,
"'%s' lookup table named: '%s' has record(s) "
"without '%s' field",
table_type, name, value_key);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
all_values[i] = (const uchar *)json_object_get_string(jvalue);
}
qsort(all_values, pThis->nmemb, sizeof(uchar *), qs_arrcmp_ustrs);
uniq_values = 1;
for (i = 1; i < pThis->nmemb; i++) {
curr = all_values[i];
prev = all_values[i - 1];
if (ustrcmp(prev, curr) != 0) {
uniq_values++;
}
}
if (pThis->nmemb > 0) {
CHKmalloc(pThis->interned_vals = malloc(uniq_values * sizeof(uchar *)));
j = 0;
CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[0]));
for (i = 1; i < pThis->nmemb; ++i) {
curr = all_values[i];
prev = all_values[i - 1];
if (ustrcmp(prev, curr) != 0) {
CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[i]));
}
}
pThis->interned_val_count = uniq_values;
}
/* uniq values captured (sorted) */
nomatch_value = json_object_get_string(jnomatch);
if (nomatch_value != NULL) {
CHKmalloc(pThis->nomatch = (uchar *)strdup(nomatch_value));
}
if (strcmp(table_type, "array") == 0) {
pThis->type = ARRAY_LOOKUP_TABLE;
CHKiRet(build_ArrayTable(pThis, jtab, name));
} else if (strcmp(table_type, "sparseArray") == 0) {
pThis->type = SPARSE_ARRAY_LOOKUP_TABLE;
CHKiRet(build_SparseArrayTable(pThis, jtab, name));
#ifdef FEATURE_REGEXP
} else if (strcmp(table_type, "regex") == 0) {
pThis->type = REGEX_LOOKUP_TABLE;
CHKiRet(build_RegexTable(pThis, jtab, name));
#endif
} else if (strcmp(table_type, "string") == 0) {
pThis->type = STRING_LOOKUP_TABLE;
CHKiRet(build_StringTable(pThis, jtab, name));
} else {
LogError(0, RS_RET_INVALID_VALUE,
"lookup table named: '%s' uses unupported "
"type: '%s'",
name, table_type);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
finalize_it:
if (all_values != NULL) free(all_values);
RETiRet;
}
static rsRetVal lookupBuildTable(lookup_t *pThis, struct json_object *jroot, const uchar *name) {
struct json_object *jversion;
int version = 1;
DEFiRet;
fjson_object_object_get_ex(jroot, "version", &jversion);
if (jversion != NULL && !json_object_is_type(jversion, json_type_null)) {
version = json_object_get_int(jversion);
} else {
LogError(0, RS_RET_INVALID_VALUE,
"lookup table named: '%s' doesn't specify version "
"(will use default value: %d)",
name, version);
}
if (version == 1) {
CHKiRet(lookupBuildTable_v1(pThis, jroot, name));
} else {
LogError(0, RS_RET_INVALID_VALUE,
"lookup table named: '%s' uses unsupported "
"version: %d",
name, version);
ABORT_FINALIZE(RS_RET_INVALID_VALUE);
}
finalize_it:
RETiRet;
}
/* find a lookup table. This is a naive O(n) algo, but this really
* doesn't matter as it is called only a few times during config
* load. The function returns either a pointer to the requested
* table or NULL, if not found.
*/
lookup_ref_t *ATTR_NONNULL() lookupFindTable(uchar *name) {
lookup_ref_t *curr;
for (curr = loadConf->lu_tabs.root; curr != NULL; curr = curr->next) {
if (!ustrcmp(curr->name, name)) break;
}
return curr;
}
/* this reloads a lookup table. This is done while the engine is running,
* as such the function must ensure proper locking and proper order of
* operations (so that nothing can interfere). If the table cannot be loaded,
* the old table is continued to be used.
*/
static rsRetVal lookupReloadOrStub(lookup_ref_t *pThis, const uchar *stub_val) {
lookup_t *newlu, *oldlu; /* dummy to be able to use support functions without
affecting current settings. */
DEFiRet;
oldlu = pThis->self;
newlu = NULL;
DBGPRINTF("reload requested for lookup table '%s'\n", pThis->name);
CHKmalloc(newlu = calloc(1, sizeof(lookup_t)));
if (stub_val == NULL) {
CHKiRet(lookupReadFile(newlu, pThis->name, pThis->filename));
} else {
CHKiRet(lookupBuildStubbedTable(newlu, stub_val));
}
/* all went well, copy over data members */
pthread_rwlock_wrlock(&pThis->rwlock);
pThis->self = newlu;
pthread_rwlock_unlock(&pThis->rwlock);
finalize_it:
if (iRet != RS_RET_OK) {
if (stub_val == NULL) {
LogError(0, RS_RET_INTERNAL_ERROR, "lookup table '%s' could not be reloaded from file '%s'", pThis->name,
pThis->filename);
} else {
LogError(0, RS_RET_INTERNAL_ERROR, "lookup table '%s' could not be stubbed with value '%s'", pThis->name,
stub_val);
}
lookupDestruct(newlu);
} else {
if (stub_val == NULL) {
LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' reloaded from file '%s'", pThis->name, pThis->filename);
} else {
LogError(0, RS_RET_OK, "lookup table '%s' stubbed with value '%s'", pThis->name, stub_val);
}
lookupDestruct(oldlu);
}
RETiRet;
}
static rsRetVal lookupDoStub(lookup_ref_t *pThis, const uchar *stub_val) {
int already_stubbed = 0;
DEFiRet;
pthread_rwlock_rdlock(&pThis->rwlock);
if (pThis->self->type == STUBBED_LOOKUP_TABLE && ustrcmp(pThis->self->nomatch, stub_val) == 0) already_stubbed = 1;
pthread_rwlock_unlock(&pThis->rwlock);
if (!already_stubbed) {
LogError(0, RS_RET_OK, "stubbing lookup table '%s' with value '%s'", pThis->name, stub_val);
CHKiRet(lookupReloadOrStub(pThis, stub_val));
} else {
LogError(0, RS_RET_OK, "lookup table '%s' is already stubbed with value '%s'", pThis->name, stub_val);
}
finalize_it:
RETiRet;
}
static uint8_t lookupIsReloadPending(lookup_ref_t *pThis) {
uint8_t reload_pending;
pthread_mutex_lock(&pThis->reloader_mut);
reload_pending = pThis->do_reload || pThis->is_reloading;
pthread_mutex_unlock(&pThis->reloader_mut);
return reload_pending;
}
/* note: stub_val_if_reload_fails may or may not be NULL */
rsRetVal ATTR_NONNULL(1) lookupReload(lookup_ref_t *const pThis, const uchar *const stub_val_if_reload_fails) {
uint8_t locked = 0;
int lock_errno = 0;
DEFiRet;
assert(pThis != NULL);
if ((lock_errno = pthread_mutex_trylock(&pThis->reloader_mut)) == 0) {
locked = 1;
/*so it doesn't leak memory in situation where 2 reload requests are issued back to back*/
freeStubValueForReloadFailure(pThis);
if (stub_val_if_reload_fails != NULL) {
CHKmalloc(pThis->stub_value_for_reload_failure = ustrdup(stub_val_if_reload_fails));
}
pThis->do_reload = 1;
pthread_cond_signal(&pThis->run_reloader);
} else {
LogError(lock_errno, RS_RET_INTERNAL_ERROR,
"attempt to trigger "
"reload of lookup table '%s' failed (not stubbing)",
pThis->name);
ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
/* we can choose to stub the table here, but it'll hurt because
the table reloader may take time to complete the reload
and stubbing because of a concurrent reload message may
not be desirable (except in very tightly controled environments
where reload-triggering messages pushed are timed accurately
and an idempotency-filter is used to reject re-deliveries) */
}
finalize_it:
if (locked) {
pthread_mutex_unlock(&pThis->reloader_mut);
}
RETiRet;
}
static rsRetVal ATTR_NONNULL(1) lookupDoReload(lookup_ref_t *pThis, uchar *stub_val) {
DEFiRet;
iRet = lookupReloadOrStub(pThis, NULL);
if ((iRet != RS_RET_OK) && (stub_val != NULL)) {
iRet = lookupDoStub(pThis, stub_val);
}
free(stub_val);
RETiRet;
}
void *lookupTableReloader(void *self) {
lookup_ref_t *pThis = (lookup_ref_t *)self;
#ifdef HAVE_PTHREAD_SETNAME_NP
lookupSetReloaderThreadName(pThis);
#endif
pthread_mutex_lock(&pThis->reloader_mut);
while (1) {
if (pThis->do_stop) {
break;
} else if (pThis->do_reload) {
uchar *stub_val = pThis->stub_value_for_reload_failure;
pThis->stub_value_for_reload_failure = NULL;
pThis->do_reload = 0;
pThis->is_reloading = 1;
pthread_mutex_unlock(&pThis->reloader_mut);
lookupDoReload(pThis, stub_val);
pthread_mutex_lock(&pThis->reloader_mut);
pThis->is_reloading = 0;
} else {
pthread_cond_wait(&pThis->run_reloader, &pThis->reloader_mut);
}
}
pthread_mutex_unlock(&pThis->reloader_mut);
return NULL;
}
/* reload all lookup tables on HUP */
void lookupDoHUP(void) {
lookup_ref_t *luref;
for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
if (luref->reload_on_hup) {
lookupReload(luref, NULL);
}
}
}
/* activate lookup table system config
* most importantly, this means tarting the lookup table reloader thread in the
* right process space - it is a difference if we fork or not!
*/
void lookupActivateConf(void) {
DBGPRINTF("lookup tables: activate config \n");
lookup_ref_t *luref;
for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
DBGPRINTF("lookup actiate: processing %p\n", luref);
lookupActivateTable(luref);
}
DBGPRINTF("lookup tables: activate done\n");
}
uint lookupPendingReloadCount(void) {
uint pending_reload_count = 0;
lookup_ref_t *luref;
for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
if (lookupIsReloadPending(luref)) {
pending_reload_count++;
}
}
return pending_reload_count;
}
/* returns either a pointer to the value (read only!) or NULL
* if either the key could not be found or an error occurred.
* Note that an estr_t object is returned. The caller is
* responsible for freeing it.
*/
es_str_t *lookupKey(lookup_ref_t *pThis, lookup_key_t key) {
es_str_t *estr;
pthread_rwlock_rdlock(&pThis->rwlock);
estr = lookupKeyLocked(pThis, key);
pthread_rwlock_unlock(&pThis->rwlock);
return estr;
}
/* caller must hold pThis->rwlock */
es_str_t *lookupKeyLocked(lookup_ref_t *pThis, lookup_key_t key) {
lookup_t *t;
t = pThis->self;
return t->lookup(t, key);
}
/* note: widely-deployed json_c 0.9 does NOT support incremental
* parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS,
* we read the file into one big memory buffer and parse it at once.
* While this is not very elegant, it will not pose any real issue
* for "reasonable" lookup tables (and "unreasonably" large ones
* will probably have other issues as well...).
*/
static rsRetVal ATTR_NONNULL()
lookupReadFile(lookup_t *const pThis, const uchar *const name, const uchar *const filename) {
struct json_tokener *tokener = NULL;
struct json_object *json = NULL;
char *iobuf = NULL;
int fd = -1;
ssize_t nread;
struct stat sb;
DEFiRet;
if ((fd = open((const char *)filename, O_RDONLY)) == -1) {
LogError(errno, RS_RET_FILE_NOT_FOUND, "lookup table file '%s' could not be opened", filename);
ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
}
if (fstat(fd, &sb) == -1) {
LogError(errno, RS_RET_FILE_NOT_FOUND, "lookup table file '%s' stat failed", filename);
ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
}
if (sb.st_size == 0) {
LogError(0, RS_RET_JSON_PARSE_ERR, "lookup table file '%s' is empty", filename);
ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR);
}
CHKmalloc(iobuf = malloc(sb.st_size));
tokener = json_tokener_new();
nread = read(fd, iobuf, sb.st_size);
if (nread != (ssize_t)sb.st_size) {
LogError(errno, RS_RET_READ_ERR, "lookup table file '%s' read error", filename);
ABORT_FINALIZE(RS_RET_READ_ERR);
}
json = json_tokener_parse_ex(tokener, iobuf, sb.st_size);
if (json == NULL) {
LogError(0, RS_RET_JSON_PARSE_ERR, "lookup table file '%s' json parsing error", filename);
ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR);
}
free(iobuf); /* early free to sever resources*/
iobuf = NULL; /* make sure no double-free */
/* got json object, now populate our own in-memory structure */
CHKiRet(lookupBuildTable(pThis, json, name));
finalize_it:
if (fd != -1) {
close(fd);
}
free(iobuf);
if (tokener != NULL) json_tokener_free(tokener);
if (json != NULL) json_object_put(json);
RETiRet;
}
rsRetVal lookupTableDefProcessCnf(struct cnfobj *o) {
struct cnfparamvals *pvals;
lookup_ref_t *lu;
uchar *name = NULL;
uchar *filename = NULL;
int reload_on_hup = 1;
int lu_linked = 0;
short i;
DEFiRet;
lu = NULL;
pvals = nvlstGetParams(o->nvlst, &modpblk, NULL);
if (pvals == NULL) {
ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
}
DBGPRINTF("lookupTableDefProcessCnf params:\n");
cnfparamsPrint(&modpblk, pvals);
for (i = 0; i < modpblk.nParams; ++i) {
if (!pvals[i].bUsed) continue;
if (!strcmp(modpblk.descr[i].name, "file")) {
CHKmalloc(filename = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
} else if (!strcmp(modpblk.descr[i].name, "name")) {
CHKmalloc(name = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
} else if (!strcmp(modpblk.descr[i].name, "reloadOnHUP")) {
reload_on_hup = (pvals[i].val.d.n != 0);
} else {
dbgprintf(
"lookup_table: program error, non-handled "
"param '%s'\n",
modpblk.descr[i].name);
}
}
if (name == NULL || filename == NULL) {
iRet = RS_RET_INTERNAL_ERROR;
LogError(0, iRet, "internal error: lookup table name or file not set albeit being mandatory");
ABORT_FINALIZE(iRet);
}
if (lookupFindTable(name) != NULL) {
LogError(0, RS_RET_CONFIG_ERROR, "lookup_table: duplicate name '%s' in current config set", name);
ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
}
CHKiRet(lookupNew(&lu));
lu->name = name;
name = NULL;
lu->filename = filename;
filename = NULL;
lu->reload_on_hup = reload_on_hup;
const uchar *const lu_name = lu->name; /* we need a const to keep TSAN happy :-( */
const uchar *const lu_filename = lu->filename; /* we need a const to keep TSAN happy :-( */
lookupAppend(lu);
lu_linked = 1;
CHKiRet(lookupReadFile(lu->self, lu_name, lu_filename));
LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' loaded from file '%s'", lu_name, lu->filename);
finalize_it:
free(name);
free(filename);
cnfparamvalsDestruct(pvals, &modpblk);
if (iRet != RS_RET_OK && lu != NULL) {
if (lu_linked) {
lookupRefDropTable(lu);
} else {
lookupRefFreeUnlinked(lu);
}
}
RETiRet;
}
void lookupClassExit(void) {
objRelease(glbl, CORE_COMPONENT);
}
rsRetVal lookupClassInit(void) {
DEFiRet;
CHKiRet(objGetObjInterface(&obj));
CHKiRet(objUse(glbl, CORE_COMPONENT));
finalize_it:
RETiRet;
}
Reference in New Issue View Git Blame Copy Permalink