rsyslog/runtime/lookup.c

/* lookup.c
 * Support for lookup tables in RainerScript.
 *
 * Copyright 2013-2023 Adiscon GmbH.
 *
 * This file is part of the rsyslog runtime library.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *		 http://www.apache.org/licenses/LICENSE-2.0
 *		 -or-
 *		 see COPYING.ASL20 in the source distribution
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <json.h>
#include <assert.h>

#include "rsyslog.h"
#include "srUtils.h"
#include "errmsg.h"
#include "lookup.h"
#include "msg.h"
#include "rsconf.h"
#include "dirty.h"
#include "unicode-helper.h"
#include "regexp.h"

PRAGMA_IGNORE_Wdeprecated_declarations
    /* definitions for objects we access */
    DEFobjStaticHelpers;
DEFobjCurrIf(glbl)
#ifdef FEATURE_REGEXP
    DEFobjCurrIf(regexp)
#endif

    /* forward definitions */
    static rsRetVal lookupReadFile(lookup_t *pThis, const uchar *name, const uchar *filename);
static void lookupDestruct(lookup_t *pThis);

/* static data */
/* tables for interfacing with the v6 config system (as far as we need to) */
static struct cnfparamdescr modpdescr[] = {{"name", eCmdHdlrString, CNFPARAM_REQUIRED},
                                           {"file", eCmdHdlrString, CNFPARAM_REQUIRED},
                                           {"reloadOnHUP", eCmdHdlrBinary, 0}};
static struct cnfparamblk modpblk = {CNFPARAMBLK_VERSION, sizeof(modpdescr) / sizeof(struct cnfparamdescr), modpdescr};

/* internal data-types */
typedef struct uint32_index_val_s {
    uint32_t index;
    uchar *val;
} uint32_index_val_t;

const char *reloader_prefix = "lkp_tbl_reloader:";

static void *lookupTableReloader(void *self);

static void lookupStopReloader(lookup_ref_t *pThis);


/* create a new lookup table object AND include it in our list of
 * lookup tables.
 */
static rsRetVal lookupNew(lookup_ref_t **ppThis) {
    lookup_ref_t *pThis = NULL;
    lookup_t *t = NULL;
    DEFiRet;

    CHKmalloc(pThis = calloc(1, sizeof(lookup_ref_t)));
    CHKmalloc(t = calloc(1, sizeof(lookup_t)));
    pThis->do_reload = pThis->do_stop = 0;
    pThis->reload_on_hup = 1; /*DO reload on HUP (default)*/

    pThis->next = NULL;
    if (loadConf->lu_tabs.root == NULL) {
        loadConf->lu_tabs.root = pThis;
    } else {
        loadConf->lu_tabs.last->next = pThis;
    }
    loadConf->lu_tabs.last = pThis;

    pThis->self = t;

    *ppThis = pThis;
finalize_it:
    if (iRet != RS_RET_OK) {
        LogError(errno, iRet, "a lookup table could not be initialized");
        free(t);
        free(pThis);
    }
    RETiRet;
}

/* activate a lookup table entry once rsyslog is ready to do so */
static rsRetVal lookupActivateTable(lookup_ref_t *pThis) {
    DEFiRet;
    int initialized = 0;

    DBGPRINTF("lookupActivateTable called\n");
    CHKiConcCtrl(pthread_rwlock_init(&pThis->rwlock, NULL));
    initialized++; /*1*/
    CHKiConcCtrl(pthread_mutex_init(&pThis->reloader_mut, NULL));
    initialized++; /*2*/
    CHKiConcCtrl(pthread_cond_init(&pThis->run_reloader, NULL));
    initialized++; /*3*/
    CHKiConcCtrl(pthread_attr_init(&pThis->reloader_thd_attr));
    initialized++; /*4*/
    pThis->do_reload = pThis->do_stop = 0;
    CHKiConcCtrl(pthread_create(&pThis->reloader, &pThis->reloader_thd_attr, lookupTableReloader, pThis));
    initialized++; /*5*/


finalize_it:
    if (iRet != RS_RET_OK) {
        LogError(errno, iRet,
                 "a lookup table could not be activated: "
                 "failed at init-step %d (please enable debug logs for details)",
                 initialized);
        /* Can not happen with current code, but might occur in the future when
         * an error-condition as added after step 5. If we leave it in, Coverity
         * scan complains. So we comment it out but do not remove the code.
         * Triggered by CID 185426
        if (initialized > 4) lookupStopReloader(pThis);
        */
        if (initialized > 3) pthread_attr_destroy(&pThis->reloader_thd_attr);
        if (initialized > 2) pthread_cond_destroy(&pThis->run_reloader);
        if (initialized > 1) pthread_mutex_destroy(&pThis->reloader_mut);
        if (initialized > 0) pthread_rwlock_destroy(&pThis->rwlock);
    }
    RETiRet;
}

/*must be called with reloader_mut acquired*/
static void ATTR_NONNULL() freeStubValueForReloadFailure(lookup_ref_t *const pThis) {
    if (pThis->stub_value_for_reload_failure != NULL) {
        free(pThis->stub_value_for_reload_failure);
        pThis->stub_value_for_reload_failure = NULL;
    }
}

static void lookupStopReloader(lookup_ref_t *pThis) {
    pthread_mutex_lock(&pThis->reloader_mut);
    freeStubValueForReloadFailure(pThis);
    pThis->do_reload = 0;
    pThis->do_stop = 1;
    pthread_cond_signal(&pThis->run_reloader);
    pthread_mutex_unlock(&pThis->reloader_mut);
    pthread_join(pThis->reloader, NULL);
}

static void lookupRefDestruct(lookup_ref_t *pThis) {
    lookupStopReloader(pThis);
    pthread_mutex_destroy(&pThis->reloader_mut);
    pthread_cond_destroy(&pThis->run_reloader);
    pthread_attr_destroy(&pThis->reloader_thd_attr);

    pthread_rwlock_destroy(&pThis->rwlock);
    lookupDestruct(pThis->self);
    free(pThis->name);
    free(pThis->filename);
    free(pThis);
}

static void destructTable_str(lookup_t *pThis) {
    uint32_t i = 0;
    lookup_string_tab_entry_t *entries = pThis->table.str->entries;
    for (i = 0; i < pThis->nmemb; i++) {
        free(entries[i].key);
    }
    free(entries);
    free(pThis->table.str);
}


static void destructTable_arr(lookup_t *pThis) {
    free(pThis->table.arr->interned_val_refs);
    free(pThis->table.arr);
}

static void destructTable_sparseArr(lookup_t *pThis) {
    free(pThis->table.sprsArr->entries);
    free(pThis->table.sprsArr);
}

#ifdef FEATURE_REGEXP
static void destructTable_regex(lookup_t *pThis) {
    uint32_t i;
    int regfree_available = 0;
    lookup_regex_tab_entry_t *entries;

    if (pThis == NULL || pThis->table.regex == NULL) return;

    entries = pThis->table.regex->entries;
    if (objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) regfree_available = 1;

    if (entries != NULL) {
        for (i = 0; i < pThis->nmemb; ++i) {
            if (entries[i].is_compiled) {
                if (regfree_available) {
                    regexp.regfree(&entries[i].regex);
                } else {
                    LogError(0, RS_RET_INTERNAL_ERROR, "regexp module missing; compiled regex cannot be freed");
                }
            }
            free(entries[i].regex_str);
            entries[i].regex_str = NULL;
        }
    }
    free(entries);
    free(pThis->table.regex);
}
#endif

static void lookupDestruct(lookup_t *pThis) {
    uint32_t i;

    if (pThis == NULL) return;

    if (pThis->type == STRING_LOOKUP_TABLE) {
        destructTable_str(pThis);
    } else if (pThis->type == ARRAY_LOOKUP_TABLE) {
        destructTable_arr(pThis);
    } else if (pThis->type == SPARSE_ARRAY_LOOKUP_TABLE) {
        destructTable_sparseArr(pThis);
#ifdef FEATURE_REGEXP
    } else if (pThis->type == REGEX_LOOKUP_TABLE) {
        destructTable_regex(pThis);
#endif
    } else if (pThis->type == STUBBED_LOOKUP_TABLE) {
        /*nothing to be done*/
    }

    for (i = 0; i < pThis->interned_val_count; i++) {
        free(pThis->interned_vals[i]);
    }
    free(pThis->interned_vals);
    free(pThis->nomatch);
    free(pThis);
}

void lookupInitCnf(lookup_tables_t *lu_tabs) {
    lu_tabs->root = NULL;
    lu_tabs->last = NULL;
}

void lookupDestroyCnf(void) {
    lookup_ref_t *luref, *luref_next;
    for (luref = runConf->lu_tabs.root; luref != NULL;) {
        luref_next = luref->next;
        lookupRefDestruct(luref);
        luref = luref_next;
    }
}

/* comparison function for qsort() */
static int qs_arrcmp_strtab(const void *s1, const void *s2) {
    return ustrcmp(((lookup_string_tab_entry_t *)s1)->key, ((lookup_string_tab_entry_t *)s2)->key);
}

static int qs_arrcmp_ustrs(const void *s1, const void *s2) {
    return ustrcmp(*(uchar **)s1, *(uchar **)s2);
}

static int qs_arrcmp_uint32_index_val(const void *s1, const void *s2) {
    uint32_t first_value = ((uint32_index_val_t *)s1)->index;
    uint32_t second_value = ((uint32_index_val_t *)s2)->index;
    if (first_value < second_value) {
        return -1;
    }
    return first_value - second_value;
}

static int qs_arrcmp_sprsArrtab(const void *s1, const void *s2) {
    uint32_t first_value = ((lookup_sparseArray_tab_entry_t *)s1)->key;
    uint32_t second_value = ((lookup_sparseArray_tab_entry_t *)s2)->key;
    if (first_value < second_value) {
        return -1;
    }
    return first_value - second_value;
}

/* comparison function for bsearch() and string array compare
 * this is for the string lookup table type
 */
static int bs_arrcmp_strtab(const void *s1, const void *s2) {
    return strcmp((char *)s1, (char *)((lookup_string_tab_entry_t *)s2)->key);
}

static int bs_arrcmp_str(const void *s1, const void *s2) {
    return ustrcmp((uchar *)s1, *(uchar **)s2);
}

static int bs_arrcmp_sprsArrtab(const void *s1, const void *s2) {
    uint32_t key = *(uint32_t *)s1;
    uint32_t array_member_value = ((lookup_sparseArray_tab_entry_t *)s2)->key;
    if (key < array_member_value) {
        return -1;
    }
    return key - array_member_value;
}

static inline const char *defaultVal(lookup_t *pThis) {
    return (pThis->nomatch == NULL) ? "" : (const char *)pThis->nomatch;
}

/* lookup_fn for different types of tables */
static es_str_t *lookupKey_stub(lookup_t *pThis, lookup_key_t __attribute__((unused)) key) {
    return es_newStrFromCStr((char *)pThis->nomatch, ustrlen(pThis->nomatch));
}

static es_str_t *lookupKey_str(lookup_t *pThis, lookup_key_t key) {
    lookup_string_tab_entry_t *entry;
    const char *r;
    if (pThis->nmemb == 0) {
        entry = NULL;
    } else {
        assert(pThis->table.str->entries);
        entry = bsearch(key.k_str, pThis->table.str->entries, pThis->nmemb, sizeof(lookup_string_tab_entry_t),
                        bs_arrcmp_strtab);
    }
    if (entry == NULL) {
        r = defaultVal(pThis);
    } else {
        r = (const char *)entry->interned_val_ref;
    }
    return es_newStrFromCStr(r, strlen(r));
}

static es_str_t *lookupKey_arr(lookup_t *pThis, lookup_key_t key) {
    const char *r;
    uint32_t uint_key = key.k_uint;
    if ((pThis->nmemb == 0) || (uint_key < pThis->table.arr->first_key)) {
        r = defaultVal(pThis);
    } else {
        uint32_t idx = uint_key - pThis->table.arr->first_key;
        if (idx >= pThis->nmemb) {
            r = defaultVal(pThis);
        } else {
            r = (char *)pThis->table.arr->interned_val_refs[idx];
        }
    }

    return es_newStrFromCStr(r, strlen(r));
}

typedef int(comp_fn_t)(const void *s1, const void *s2);

static void *bsearch_lte(const void *key, const void *base, size_t nmemb, size_t size, comp_fn_t *comp_fn) {
    size_t l, u, idx;
    const void *p;
    int comparison;

    l = 0;
    u = nmemb;
    if (l == u) {
        return NULL;
    }
    while (l < u) {
        idx = (l + u) / 2;
        p = (void *)(((const char *)base) + (idx * size));
        comparison = (*comp_fn)(key, p);
        if (comparison < 0)
            u = idx;
        else if (comparison > 0)
            l = idx + 1;
        else
            return (void *)p;
    }
    if (comparison < 0) {
        if (idx == 0) {
            return NULL;
        }
        idx--;
    }
    return (void *)(((const char *)base) + (idx * size));
}

static es_str_t *lookupKey_sprsArr(lookup_t *pThis, lookup_key_t key) {
    lookup_sparseArray_tab_entry_t *entry;
    const char *r;
    if (pThis->nmemb == 0) {
        entry = NULL;
    } else {
        entry = bsearch_lte(&key.k_uint, pThis->table.sprsArr->entries, pThis->nmemb,
                            sizeof(lookup_sparseArray_tab_entry_t), bs_arrcmp_sprsArrtab);
    }

    if (entry == NULL) {
        r = defaultVal(pThis);
    } else {
        r = (const char *)entry->interned_val_ref;
    }
    return es_newStrFromCStr(r, strlen(r));
}

#ifdef FEATURE_REGEXP
static es_str_t *lookupKey_regex(lookup_t *pThis, lookup_key_t key) {
    const char *r = defaultVal(pThis);

    for (uint32_t i = 0; i < pThis->nmemb; ++i) {
        if (regexp.regexec(&pThis->table.regex->entries[i].regex, (char *)key.k_str, 0, NULL, 0) == 0) {
            r = (const char *)pThis->table.regex->entries[i].interned_val_ref;
            break;
        }
    }
    return es_newStrFromCStr(r, strlen(r));
}
#endif

/* builders for different table-types */

#define NO_INDEX_ERROR(type, name)                                          \
    LogError(0, RS_RET_INVALID_VALUE,                                       \
             "'%s' lookup table named: '%s' has record(s) without 'index' " \
             "field",                                                       \
             type, name);                                                   \
    ABORT_FINALIZE(RS_RET_INVALID_VALUE);

static rsRetVal build_StringTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
    uint32_t i;
    struct json_object *jrow, *jindex, *jvalue;
    uchar *value, *canonicalValueRef;
    DEFiRet;

    pThis->table.str = NULL;
    CHKmalloc(pThis->table.str = calloc(1, sizeof(lookup_string_tab_t)));
    if (pThis->nmemb > 0) {
        CHKmalloc(pThis->table.str->entries = calloc(pThis->nmemb, sizeof(lookup_string_tab_entry_t)));

        for (i = 0; i < pThis->nmemb; i++) {
            jrow = json_object_array_get_idx(jtab, i);
            fjson_object_object_get_ex(jrow, "index", &jindex);
            fjson_object_object_get_ex(jrow, "value", &jvalue);
            if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
                NO_INDEX_ERROR("string", name);
            }
            CHKmalloc(pThis->table.str->entries[i].key = ustrdup((uchar *)json_object_get_string(jindex)));
            value = (uchar *)json_object_get_string(jvalue);
            uchar **found = (uchar **)bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *),
                                              bs_arrcmp_str);
            if (found == NULL) {
                LogError(0, RS_RET_INTERNAL_ERROR,
                         "lookup.c:build_StringTable(): "
                         "internal error, bsearch returned NULL for '%s'",
                         value);
                ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
            }
// I give up, I see no way to remove false positive -- rgerhards, 2017-10-24
#ifndef __clang_analyzer__
            canonicalValueRef = *found;
            if (canonicalValueRef == NULL) {
                LogError(0, RS_RET_INTERNAL_ERROR,
                         "lookup.c:build_StringTable(): "
                         "internal error, canonicalValueRef returned from bsearch "
                         "is NULL for '%s'",
                         value);
                ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
            }
            pThis->table.str->entries[i].interned_val_ref = canonicalValueRef;
#endif
        }
        qsort(pThis->table.str->entries, pThis->nmemb, sizeof(lookup_string_tab_entry_t), qs_arrcmp_strtab);
    }

    pThis->lookup = lookupKey_str;
    pThis->key_type = LOOKUP_KEY_TYPE_STRING;
finalize_it:
    RETiRet;
}

static rsRetVal build_ArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
    uint32_t i;
    struct json_object *jrow, *jindex, *jvalue;
    uchar *canonicalValueRef;
    uint32_t prev_index, _index;
    uint8_t prev_index_set;
    uint32_index_val_t *indexes = NULL;
    DEFiRet;

    prev_index_set = 0;

    pThis->table.arr = NULL;
    CHKmalloc(pThis->table.arr = calloc(1, sizeof(lookup_array_tab_t)));
    if (pThis->nmemb > 0) {
        CHKmalloc(indexes = calloc(pThis->nmemb, sizeof(uint32_index_val_t)));
        CHKmalloc(pThis->table.arr->interned_val_refs = calloc(pThis->nmemb, sizeof(uchar *)));

        for (i = 0; i < pThis->nmemb; i++) {
            jrow = json_object_array_get_idx(jtab, i);
            fjson_object_object_get_ex(jrow, "index", &jindex);
            fjson_object_object_get_ex(jrow, "value", &jvalue);
            if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
                NO_INDEX_ERROR("array", name);
            }
            indexes[i].index = (uint32_t)json_object_get_int(jindex);
            indexes[i].val = (uchar *)json_object_get_string(jvalue);
        }
        qsort(indexes, pThis->nmemb, sizeof(uint32_index_val_t), qs_arrcmp_uint32_index_val);
        for (i = 0; i < pThis->nmemb; i++) {
            _index = indexes[i].index;
            if (prev_index_set == 0) {
                prev_index = _index;
                prev_index_set = 1;
                pThis->table.arr->first_key = _index;
            } else {
                if (_index != ++prev_index) {
                    LogError(0, RS_RET_INVALID_VALUE,
                             "'array' lookup table name: '%s' "
                             "has non-contiguous members between index '%d' and '%d'",
                             name, prev_index, _index);
                    ABORT_FINALIZE(RS_RET_INVALID_VALUE);
                }
            }
            uchar *const *const canonicalValueRef_ptr = bsearch(
                indexes[i].val, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
            if (canonicalValueRef_ptr == NULL) {
                LogError(0, RS_RET_ERR,
                         "BUG: canonicalValueRef not found in "
                         "build_ArrayTable(), %s:%d",
                         __FILE__, __LINE__);
                ABORT_FINALIZE(RS_RET_ERR);
            }
            canonicalValueRef = *canonicalValueRef_ptr;
            assert(canonicalValueRef != NULL);
            pThis->table.arr->interned_val_refs[i] = canonicalValueRef;
        }
    }

    pThis->lookup = lookupKey_arr;
    pThis->key_type = LOOKUP_KEY_TYPE_UINT;

finalize_it:
    free(indexes);
    RETiRet;
}

static rsRetVal build_SparseArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
    uint32_t i;
    struct json_object *jrow, *jindex, *jvalue;
    uchar *value, *canonicalValueRef;
    DEFiRet;

    pThis->table.str = NULL;
    CHKmalloc(pThis->table.sprsArr = calloc(1, sizeof(lookup_sparseArray_tab_t)));
    if (pThis->nmemb > 0) {
        CHKmalloc(pThis->table.sprsArr->entries = calloc(pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t)));

        for (i = 0; i < pThis->nmemb; i++) {
            jrow = json_object_array_get_idx(jtab, i);
            fjson_object_object_get_ex(jrow, "index", &jindex);
            fjson_object_object_get_ex(jrow, "value", &jvalue);
            if (jindex == NULL || json_object_is_type(jindex, json_type_null)) {
                NO_INDEX_ERROR("sparseArray", name);
            }
            pThis->table.sprsArr->entries[i].key = (uint32_t)json_object_get_int(jindex);
            value = (uchar *)json_object_get_string(jvalue);
            uchar *const *const canonicalValueRef_ptr =
                bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
            if (canonicalValueRef_ptr == NULL) {
                LogError(0, RS_RET_ERR,
                         "BUG: canonicalValueRef not found in "
                         "build_SparseArrayTable(), %s:%d",
                         __FILE__, __LINE__);
                ABORT_FINALIZE(RS_RET_ERR);
            }
            canonicalValueRef = *canonicalValueRef_ptr;
            assert(canonicalValueRef != NULL);
            pThis->table.sprsArr->entries[i].interned_val_ref = canonicalValueRef;
        }
        qsort(pThis->table.sprsArr->entries, pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t),
              qs_arrcmp_sprsArrtab);
    }

    pThis->lookup = lookupKey_sprsArr;
    pThis->key_type = LOOKUP_KEY_TYPE_UINT;

finalize_it:
    RETiRet;
}

#ifdef FEATURE_REGEXP
static rsRetVal build_RegexTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) {
    uint32_t i;
    struct json_object *jrow, *jregex, *jtag;
    uchar *value, *canonicalValueRef;
    const char *regex_str;
    DEFiRet;

    pThis->table.regex = NULL;
    CHKmalloc(pThis->table.regex = calloc(1, sizeof(lookup_regex_tab_t)));
    if (pThis->nmemb > 0) {
        CHKmalloc(pThis->table.regex->entries = calloc(pThis->nmemb, sizeof(lookup_regex_tab_entry_t)));

        for (i = 0; i < pThis->nmemb; i++) {
            jrow = json_object_array_get_idx(jtab, i);
            fjson_object_object_get_ex(jrow, "regex", &jregex);
            fjson_object_object_get_ex(jrow, "tag", &jtag);
            if (jregex == NULL || jtag == NULL || json_object_is_type(jregex, json_type_null) ||
                json_object_is_type(jtag, json_type_null)) {
                LogError(0, RS_RET_INVALID_VALUE,
                         "'regex' lookup table named: '%s' has record(s) without required "
                         "fields",
                         name);
                ABORT_FINALIZE(RS_RET_INVALID_VALUE);
            }
            regex_str = json_object_get_string(jregex);
            CHKmalloc(pThis->table.regex->entries[i].regex_str = ustrdup((const uchar *)regex_str));
            value = (uchar *)json_object_get_string(jtag);
            uchar *const *const canonicalValueRef_ptr =
                bsearch(value, pThis->interned_vals, pThis->interned_val_count, sizeof(uchar *), bs_arrcmp_str);
            if (canonicalValueRef_ptr == NULL) {
                LogError(0, RS_RET_ERR, "BUG: canonicalValueRef not found in build_RegexTable(), %s:%d", __FILE__,
                         __LINE__);
                ABORT_FINALIZE(RS_RET_ERR);
            }
            canonicalValueRef = *canonicalValueRef_ptr;
            assert(canonicalValueRef != NULL);
            pThis->table.regex->entries[i].interned_val_ref = canonicalValueRef;

            int err;
            if (objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
                if ((err = regexp.regcomp(&pThis->table.regex->entries[i].regex, regex_str,
                                          REG_EXTENDED | REG_NOSUB)) != 0) {
                    char errbuf[256];
                    regexp.regerror(err, &pThis->table.regex->entries[i].regex, errbuf, sizeof(errbuf));
                    LogError(0, RS_RET_INVALID_VALUE, "error compiling regex '%s' in lookup table '%s': %s", regex_str,
                             name, errbuf);
                    ABORT_FINALIZE(RS_RET_INVALID_VALUE);
                } else {
                    pThis->table.regex->entries[i].is_compiled = 1;
                }
            } else {
                LogError(0, RS_RET_INTERNAL_ERROR, "regex library could not be loaded");
                ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
            }
        }
    }

    pThis->lookup = lookupKey_regex;
    pThis->key_type = LOOKUP_KEY_TYPE_STRING;

finalize_it:
    RETiRet;
}
#endif

static rsRetVal lookupBuildStubbedTable(lookup_t *pThis, const uchar *stub_val) {
    DEFiRet;

    CHKmalloc(pThis->nomatch = ustrdup(stub_val));
    pThis->lookup = lookupKey_stub;
    pThis->type = STUBBED_LOOKUP_TABLE;
    pThis->key_type = LOOKUP_KEY_TYPE_NONE;

finalize_it:
    RETiRet;
}

static rsRetVal lookupBuildTable_v1(lookup_t *pThis, struct json_object *jroot, const uchar *name) {
    struct json_object *jnomatch, *jtype, *jtab;
    struct json_object *jrow, *jvalue;
    const char *table_type, *nomatch_value;
    const char *value_key;
    const uchar **all_values;
    const uchar *curr, *prev;
    uint32_t i, j;
    uint32_t uniq_values;

    DEFiRet;
    all_values = NULL;

    fjson_object_object_get_ex(jroot, "nomatch", &jnomatch);
    fjson_object_object_get_ex(jroot, "type", &jtype);
    fjson_object_object_get_ex(jroot, "table", &jtab);
    if (jtab == NULL || !json_object_is_type(jtab, json_type_array)) {
        LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' has invalid table definition", name);
        ABORT_FINALIZE(RS_RET_INVALID_VALUE);
    }
    pThis->nmemb = json_object_array_length(jtab);
    table_type = json_object_get_string(jtype);
    if (table_type == NULL) {
        table_type = "string";
    }
    value_key = (strcmp(table_type, "regex") == 0) ? "tag" : "value";

    CHKmalloc(all_values = malloc(pThis->nmemb * sizeof(uchar *)));

    /* before actual table can be loaded, prepare all-value list and remove duplicates*/
    for (i = 0; i < pThis->nmemb; i++) {
        jrow = json_object_array_get_idx(jtab, i);
        fjson_object_object_get_ex(jrow, value_key, &jvalue);
        if (jvalue == NULL || json_object_is_type(jvalue, json_type_null)) {
            LogError(0, RS_RET_INVALID_VALUE,
                     "'%s' lookup table named: '%s' has record(s) "
                     "without '%s' field",
                     table_type, name, value_key);
            ABORT_FINALIZE(RS_RET_INVALID_VALUE);
        }
        all_values[i] = (const uchar *)json_object_get_string(jvalue);
    }
    qsort(all_values, pThis->nmemb, sizeof(uchar *), qs_arrcmp_ustrs);
    uniq_values = 1;
    for (i = 1; i < pThis->nmemb; i++) {
        curr = all_values[i];
        prev = all_values[i - 1];
        if (ustrcmp(prev, curr) != 0) {
            uniq_values++;
        }
    }

    if (pThis->nmemb > 0) {
        CHKmalloc(pThis->interned_vals = malloc(uniq_values * sizeof(uchar *)));
        j = 0;
        CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[0]));
        for (i = 1; i < pThis->nmemb; ++i) {
            curr = all_values[i];
            prev = all_values[i - 1];
            if (ustrcmp(prev, curr) != 0) {
                CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[i]));
            }
        }
        pThis->interned_val_count = uniq_values;
    }
    /* uniq values captured (sorted) */

    nomatch_value = json_object_get_string(jnomatch);
    if (nomatch_value != NULL) {
        CHKmalloc(pThis->nomatch = (uchar *)strdup(nomatch_value));
    }

    if (strcmp(table_type, "array") == 0) {
        pThis->type = ARRAY_LOOKUP_TABLE;
        CHKiRet(build_ArrayTable(pThis, jtab, name));
    } else if (strcmp(table_type, "sparseArray") == 0) {
        pThis->type = SPARSE_ARRAY_LOOKUP_TABLE;
        CHKiRet(build_SparseArrayTable(pThis, jtab, name));
#ifdef FEATURE_REGEXP
    } else if (strcmp(table_type, "regex") == 0) {
        pThis->type = REGEX_LOOKUP_TABLE;
        CHKiRet(build_RegexTable(pThis, jtab, name));
#endif
    } else if (strcmp(table_type, "string") == 0) {
        pThis->type = STRING_LOOKUP_TABLE;
        CHKiRet(build_StringTable(pThis, jtab, name));
    } else {
        LogError(0, RS_RET_INVALID_VALUE,
                 "lookup table named: '%s' uses unupported "
                 "type: '%s'",
                 name, table_type);
        ABORT_FINALIZE(RS_RET_INVALID_VALUE);
    }
finalize_it:
    if (all_values != NULL) free(all_values);
    RETiRet;
}

static rsRetVal lookupBuildTable(lookup_t *pThis, struct json_object *jroot, const uchar *name) {
    struct json_object *jversion;
    int version = 1;

    DEFiRet;

    fjson_object_object_get_ex(jroot, "version", &jversion);
    if (jversion != NULL && !json_object_is_type(jversion, json_type_null)) {
        version = json_object_get_int(jversion);
    } else {
        LogError(0, RS_RET_INVALID_VALUE,
                 "lookup table named: '%s' doesn't specify version "
                 "(will use default value: %d)",
                 name, version);
    }
    if (version == 1) {
        CHKiRet(lookupBuildTable_v1(pThis, jroot, name));
    } else {
        LogError(0, RS_RET_INVALID_VALUE,
                 "lookup table named: '%s' uses unsupported "
                 "version: %d",
                 name, version);
        ABORT_FINALIZE(RS_RET_INVALID_VALUE);
    }

finalize_it:
    RETiRet;
}


/* find a lookup table. This is a naive O(n) algo, but this really
 * doesn't matter as it is called only a few times during config
 * load. The function returns either a pointer to the requested
 * table or NULL, if not found.
 */
lookup_ref_t *ATTR_NONNULL() lookupFindTable(uchar *name) {
    lookup_ref_t *curr;

    for (curr = loadConf->lu_tabs.root; curr != NULL; curr = curr->next) {
        if (!ustrcmp(curr->name, name)) break;
    }
    return curr;
}


/* this reloads a lookup table. This is done while the engine is running,
 * as such the function must ensure proper locking and proper order of
 * operations (so that nothing can interfere). If the table cannot be loaded,
 * the old table is continued to be used.
 */
static rsRetVal lookupReloadOrStub(lookup_ref_t *pThis, const uchar *stub_val) {
    lookup_t *newlu, *oldlu; /* dummy to be able to use support functions without
                                affecting current settings. */
    DEFiRet;

    oldlu = pThis->self;
    newlu = NULL;

    DBGPRINTF("reload requested for lookup table '%s'\n", pThis->name);
    CHKmalloc(newlu = calloc(1, sizeof(lookup_t)));
    if (stub_val == NULL) {
        CHKiRet(lookupReadFile(newlu, pThis->name, pThis->filename));
    } else {
        CHKiRet(lookupBuildStubbedTable(newlu, stub_val));
    }
    /* all went well, copy over data members */
    pthread_rwlock_wrlock(&pThis->rwlock);
    pThis->self = newlu;
    pthread_rwlock_unlock(&pThis->rwlock);
finalize_it:
    if (iRet != RS_RET_OK) {
        if (stub_val == NULL) {
            LogError(0, RS_RET_INTERNAL_ERROR, "lookup table '%s' could not be reloaded from file '%s'", pThis->name,
                     pThis->filename);
        } else {
            LogError(0, RS_RET_INTERNAL_ERROR, "lookup table '%s' could not be stubbed with value '%s'", pThis->name,
                     stub_val);
        }
        lookupDestruct(newlu);
    } else {
        if (stub_val == NULL) {
            LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' reloaded from file '%s'", pThis->name, pThis->filename);
        } else {
            LogError(0, RS_RET_OK, "lookup table '%s' stubbed with value '%s'", pThis->name, stub_val);
        }
        lookupDestruct(oldlu);
    }
    RETiRet;
}

static rsRetVal lookupDoStub(lookup_ref_t *pThis, const uchar *stub_val) {
    int already_stubbed = 0;
    DEFiRet;
    pthread_rwlock_rdlock(&pThis->rwlock);
    if (pThis->self->type == STUBBED_LOOKUP_TABLE && ustrcmp(pThis->self->nomatch, stub_val) == 0) already_stubbed = 1;
    pthread_rwlock_unlock(&pThis->rwlock);
    if (!already_stubbed) {
        LogError(0, RS_RET_OK, "stubbing lookup table '%s' with value '%s'", pThis->name, stub_val);
        CHKiRet(lookupReloadOrStub(pThis, stub_val));
    } else {
        LogError(0, RS_RET_OK, "lookup table '%s' is already stubbed with value '%s'", pThis->name, stub_val);
    }
finalize_it:
    RETiRet;
}

static uint8_t lookupIsReloadPending(lookup_ref_t *pThis) {
    uint8_t reload_pending;
    pthread_mutex_lock(&pThis->reloader_mut);
    reload_pending = pThis->do_reload;
    pthread_mutex_unlock(&pThis->reloader_mut);
    return reload_pending;
}

/* note: stub_val_if_reload_fails may or may not be NULL */
rsRetVal ATTR_NONNULL(1) lookupReload(lookup_ref_t *const pThis, const uchar *const stub_val_if_reload_fails) {
    uint8_t locked = 0;
    int lock_errno = 0;
    DEFiRet;
    assert(pThis != NULL);
    if ((lock_errno = pthread_mutex_trylock(&pThis->reloader_mut)) == 0) {
        locked = 1;
        /*so it doesn't leak memory in situation where 2 reload requests are issued back to back*/
        freeStubValueForReloadFailure(pThis);
        if (stub_val_if_reload_fails != NULL) {
            CHKmalloc(pThis->stub_value_for_reload_failure = ustrdup(stub_val_if_reload_fails));
        }
        pThis->do_reload = 1;
        pthread_cond_signal(&pThis->run_reloader);
    } else {
        LogError(lock_errno, RS_RET_INTERNAL_ERROR,
                 "attempt to trigger "
                 "reload of lookup table '%s' failed (not stubbing)",
                 pThis->name);
        ABORT_FINALIZE(RS_RET_INTERNAL_ERROR);
        /* we can choose to stub the table here, but it'll hurt because
           the table reloader may take time to complete the reload
           and stubbing because of a concurrent reload message may
           not be desirable (except in very tightly controled environments
           where reload-triggering messages pushed are timed accurately
           and an idempotency-filter is used to reject re-deliveries) */
    }
finalize_it:
    if (locked) {
        pthread_mutex_unlock(&pThis->reloader_mut);
    }
    RETiRet;
}

static rsRetVal ATTR_NONNULL() lookupDoReload(lookup_ref_t *pThis) {
    DEFiRet;
    iRet = lookupReloadOrStub(pThis, NULL);
    if ((iRet != RS_RET_OK) && (pThis->stub_value_for_reload_failure != NULL)) {
        iRet = lookupDoStub(pThis, pThis->stub_value_for_reload_failure);
    }
    freeStubValueForReloadFailure(pThis);
    RETiRet;
}

void *lookupTableReloader(void *self) {
    lookup_ref_t *pThis = (lookup_ref_t *)self;
    pthread_mutex_lock(&pThis->reloader_mut);
    while (1) {
        if (pThis->do_stop) {
            break;
        } else if (pThis->do_reload) {
            lookupDoReload(pThis);
            pThis->do_reload = 0;
        } else {
            pthread_cond_wait(&pThis->run_reloader, &pThis->reloader_mut);
        }
    }
    pthread_mutex_unlock(&pThis->reloader_mut);
    return NULL;
}

/* reload all lookup tables on HUP */
void lookupDoHUP(void) {
    lookup_ref_t *luref;
    for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
        if (luref->reload_on_hup) {
            lookupReload(luref, NULL);
        }
    }
}

/* activate lookup table system config
 * most importantly, this means tarting the lookup table reloader thread in the
 * right process space - it is a difference if we fork or not!
 */
void lookupActivateConf(void) {
    DBGPRINTF("lookup tables: activate config \n");
    lookup_ref_t *luref;
    for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
        DBGPRINTF("lookup actiate: processing %p\n", luref);
        lookupActivateTable(luref);
    }
    DBGPRINTF("lookup tables: activate done\n");
}

uint lookupPendingReloadCount(void) {
    uint pending_reload_count = 0;
    lookup_ref_t *luref;
    for (luref = runConf->lu_tabs.root; luref != NULL; luref = luref->next) {
        if (lookupIsReloadPending(luref)) {
            pending_reload_count++;
        }
    }
    return pending_reload_count;
}


/* returns either a pointer to the value (read only!) or NULL
 * if either the key could not be found or an error occurred.
 * Note that an estr_t object is returned. The caller is
 * responsible for freeing it.
 */
es_str_t *lookupKey(lookup_ref_t *pThis, lookup_key_t key) {
    es_str_t *estr;
    lookup_t *t;
    pthread_rwlock_rdlock(&pThis->rwlock);
    t = pThis->self;
    estr = t->lookup(t, key);
    pthread_rwlock_unlock(&pThis->rwlock);
    return estr;
}


/* note: widely-deployed json_c 0.9 does NOT support incremental
 * parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS,
 * we read the file into one big memory buffer and parse it at once.
 * While this is not very elegant, it will not pose any real issue
 * for "reasonable" lookup tables (and "unreasonably" large ones
 * will probably have other issues as well...).
 */
static rsRetVal ATTR_NONNULL()
    lookupReadFile(lookup_t *const pThis, const uchar *const name, const uchar *const filename) {
    struct json_tokener *tokener = NULL;
    struct json_object *json = NULL;
    char *iobuf = NULL;
    int fd = -1;
    ssize_t nread;
    struct stat sb;
    DEFiRet;


    if ((fd = open((const char *)filename, O_RDONLY)) == -1) {
        LogError(errno, RS_RET_FILE_NOT_FOUND, "lookup table file '%s' could not be opened", filename);
        ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
    }

    if (fstat(fd, &sb) == -1) {
        LogError(errno, RS_RET_FILE_NOT_FOUND, "lookup table file '%s' stat failed", filename);
        ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
    }

    CHKmalloc(iobuf = malloc(sb.st_size));

    tokener = json_tokener_new();
    nread = read(fd, iobuf, sb.st_size);
    if (nread != (ssize_t)sb.st_size) {
        LogError(errno, RS_RET_READ_ERR, "lookup table file '%s' read error", filename);
        ABORT_FINALIZE(RS_RET_READ_ERR);
    }

    json = json_tokener_parse_ex(tokener, iobuf, sb.st_size);
    if (json == NULL) {
        LogError(0, RS_RET_JSON_PARSE_ERR, "lookup table file '%s' json parsing error", filename);
        ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR);
    }
    free(iobuf); /* early free to sever resources*/
    iobuf = NULL; /* make sure no double-free */

    /* got json object, now populate our own in-memory structure */
    CHKiRet(lookupBuildTable(pThis, json, name));

finalize_it:
    if (fd != -1) {
        close(fd);
    }
    free(iobuf);
    if (tokener != NULL) json_tokener_free(tokener);
    if (json != NULL) json_object_put(json);
    RETiRet;
}


rsRetVal lookupTableDefProcessCnf(struct cnfobj *o) {
    struct cnfparamvals *pvals;
    lookup_ref_t *lu;
    short i;
#ifdef HAVE_PTHREAD_SETNAME_NP
    char *reloader_thd_name = NULL;
    int thd_name_len = 0;
#endif
    DEFiRet;
    lu = NULL;

    pvals = nvlstGetParams(o->nvlst, &modpblk, NULL);
    if (pvals == NULL) {
        ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
    }
    DBGPRINTF("lookupTableDefProcessCnf params:\n");
    cnfparamsPrint(&modpblk, pvals);

    CHKiRet(lookupNew(&lu));

    for (i = 0; i < modpblk.nParams; ++i) {
        if (!pvals[i].bUsed) continue;
        if (!strcmp(modpblk.descr[i].name, "file")) {
            CHKmalloc(lu->filename = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(modpblk.descr[i].name, "name")) {
            CHKmalloc(lu->name = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(modpblk.descr[i].name, "reloadOnHUP")) {
            lu->reload_on_hup = (pvals[i].val.d.n != 0);
        } else {
            dbgprintf(
                "lookup_table: program error, non-handled "
                "param '%s'\n",
                modpblk.descr[i].name);
        }
    }
    const uchar *const lu_name = lu->name; /* we need a const to keep TSAN happy :-( */
    const uchar *const lu_filename = lu->filename; /* we need a const to keep TSAN happy :-( */
    if (lu_name == NULL || lu_filename == NULL) {
        iRet = RS_RET_INTERNAL_ERROR;
        LogError(0, iRet, "internal error: lookup table name not set albeit being mandatory");
        ABORT_FINALIZE(iRet);
    }
#ifdef HAVE_PTHREAD_SETNAME_NP
    thd_name_len = ustrlen(lu_name) + strlen(reloader_prefix) + 1;
    CHKmalloc(reloader_thd_name = malloc(thd_name_len));
    strcpy(reloader_thd_name, reloader_prefix);
    strcpy(reloader_thd_name + strlen(reloader_prefix), (char *)lu_name);
    reloader_thd_name[thd_name_len - 1] = '\0';
    #if defined(__NetBSD__)
    pthread_setname_np(lu->reloader, "%s", reloader_thd_name);
    #elif defined(__APPLE__)
    pthread_setname_np(reloader_thd_name);  // must check
    #else
    pthread_setname_np(lu->reloader, reloader_thd_name);
    #endif
#endif
    CHKiRet(lookupReadFile(lu->self, lu_name, lu_filename));
    LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' loaded from file '%s'", lu_name, lu->filename);

finalize_it:
#ifdef HAVE_PTHREAD_SETNAME_NP
    free(reloader_thd_name);
#endif
    cnfparamvalsDestruct(pvals, &modpblk);
    if (iRet != RS_RET_OK) {
        if (lu != NULL) {
            lookupDestruct(lu->self);
            lu->self = NULL;
        }
    }
    RETiRet;
}

void lookupClassExit(void) {
    objRelease(glbl, CORE_COMPONENT);
}

rsRetVal lookupClassInit(void) {
    DEFiRet;
    CHKiRet(objGetObjInterface(&obj));
    CHKiRet(objUse(glbl, CORE_COMPONENT));
finalize_it:
    RETiRet;
}