rsyslog/rsyslog
1
0
Fork 0
mirror of https://github.com/rsyslog/rsyslog.git synced 2025-12-18 16:50:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1357 from GrayTShirt/mmdb-stylecheck

Browse Source 
mmdblookup module cleanup
This commit is contained in:
Rainer Gerhards 2017-01-23 15:37:49 +01:00 committed by GitHub
commit e318309c61
7 changed files with 199 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
contrib/mmdblookup/README.md
View File

@ -1,42 +1,24 @@
# rsyslog-maxminddb
# Rsyslog - MMDBLookup
MaxMindDB is the new file format for storing information about IP addresses in a highly optimized, flexible database format. GeoIP2 Databases are available in the MaxMind DB format.
Fast geoip lookups straight from Rsyslog.
MaxMindDB vs GeoIP:
[Maxmind](https://www.maxmind.com/en/home) provides free and pay-for memmory mapped ip-lookup databases.
The free city-level database is about 22 mB, and can be found on the [geolite page](http://dev.maxmind.com/geoip/geoip2/geolite2/).
1. 4 ~ 6 faster
2. MaxMindDB Writer
## Compile
1. download and install libfastjson > 0.99.3 from https://github.com/rgerhards/libfastjson/commit/c437cad46af1998e3ad2dafa058c9e2c715df261
```
git clone https://github.com/rgerhards/libfastjson
```
2. download rsyslog source
```
git clone https://github.com/rsyslog/rsyslog.git
```
3. copy the code into rsyslog contrib path：
```
cp -r src/contrib/mmdblookup ../rsyslog/contrib/
cp src/configure.ac ../rsyslog/
cp src/Makefile.am ../rsyslog/
cp src/libfastjson.env ../rsyslog/
# cp src/tests ../rsyslog/
```
4. configure
1. Ensure you have [libfastjson](https://github.com/rgerhards/libfastjson/) installed, check your package manager or install from source.
2. Ensure you have [libmaxminddb](https://github.com/maxmind/libmaxminddb) installed, check your package manager.
3. configure
```
export PKG_CONFIG_PATH=/lib64/pkgconfig/
yum install -y libestr liblogging libmaxminddb-devel
yum install -y git-core valgrind autoconf automake flex bison json-c-devel libuuid-devel libgcrypt-devel zlib-devel openssl-devel libcurl-devel gnutls-devel mysql-devel postgresql-devel libdbi-dbd-mysql libdbi-devel net-snmp-devel
yum install -y autoconf automake flex bison json-c-devel libuuid-devel libgcrypt-devel zlib-devel openssl-devel libcurl-devel gnutls-devel
cd ../rsyslog
source libfastjson.env
autoconf
./configure --enable-mmdblookup --enable-mmjsonparse --***
./autogen.sh --enable-mmdblookup --enable-mmjsonparse --***
make
make install
```
## Usage
@ -66,11 +48,10 @@ ruleset ( name="test"){
action( type="omfwd" Target="10.211.55.3" port="514" Protocol="tcp" template="clientlog" )
stop
}
}
```
### test
### Testing
```
cat /root/a
@ -89,4 +70,3 @@ get the result from logstash-input:
"port" => 58199
}
```

2
contrib/mmdblookup/libfastjson.env
View File

@ -1,2 +0,0 @@
export JSON_C_CFLAGS=-I/usr/local/include/libfastjson
export JSON_C_LIBS="-L/usr/local/lib -Wl,-rpath,/usr/local/lib -lfastjson"

73
contrib/mmdblookup/mmdblookup.c
View File

@ -51,7 +51,6 @@ DEFobjCurrIf(errmsg);
DEF_OMOD_STATIC_DATA
/* config variables */
typedef struct _instanceData {
char *pszKey;
char *pszMmdbFile;
@ -67,25 +66,34 @@ typedef struct wrkrInstanceData {
} wrkrInstanceData_t;
struct modConfData_s {
rsconf_t *pConf; /* our overall config object */
/* our overall config object */
rsconf_t *pConf;
};
static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */
static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */
/* modConf ptr to use for the current load process */
static modConfData_t *loadModConf = NULL;
/* modConf ptr to use for the current exec process */
static modConfData_t *runModConf = NULL;
/* tables for interfacing with the v6 config system */
/* action (instance) parameters */
/* tables for interfacing with the v6 config system
* action (instance) parameters */
static struct cnfparamdescr actpdescr[] = {
{ "key", eCmdHdlrGetWord, 0 },
{ "mmdbfile", eCmdHdlrGetWord, 0 },
{ "fields", eCmdHdlrArray, 0 },
};
static struct cnfparamblk actpblk =
{ CNFPARAMBLK_VERSION,
static struct cnfparamblk actpblk = {
CNFPARAMBLK_VERSION,
sizeof(actpdescr)/sizeof(struct cnfparamdescr),
actpdescr
};
/* protype functions */
void str_split(char **membuf);
BEGINbeginCnfLoad
CODESTARTbeginCnfLoad
loadModConf = pModConf;
@ -123,7 +131,7 @@ CODESTARTcreateWrkrInstance
dbgprintf(" IO error: %s\n", strerror(errno));
}
errmsg.LogError(0, RS_RET_SUSPENDED, "can not initialize maxminddb");
// ABORT_FINALIZE(RS_RET_SUSPENDED);
/* ABORT_FINALIZE(RS_RET_SUSPENDED); */
}
ENDcreateWrkrInstance
@ -179,10 +187,9 @@ CODESTARTnewActInst
if (!strcmp(actpblk.descr[i].name, "fields")) {
pData->fieldList.nmemb = pvals[i].val.d.ar->nmemb;
CHKmalloc(pData->fieldList.name = malloc(sizeof(uchar *) * pData->fieldList.nmemb));
for(int j = 0 ; j < pvals[i].val.d.ar->nmemb ; ++j) {
for (int j = 0; j < pvals[i].val.d.ar->nmemb; ++j)
pData->fieldList.name[j] = (uchar*)es_str2cstr(pvals[i].val.d.ar->arr[j], NULL);
}
}
dbgprintf("mmdblookup: program error, non-handled"
" param '%s'\n", actpblk.descr[i].name);
}
@ -210,29 +217,27 @@ ENDdbgPrintInstInfo
BEGINtryResume
CODESTARTtryResume
ENDtryResume
void str_split(char **membuf){
char *buf = *membuf;
char tempbuf[strlen(buf)];
memset(tempbuf, 0, strlen(buf));
while (*buf++ != '\0') {
if (*buf == '\n' || *buf == '\t' || *buf == ' '){
if (*buf == '\n' || *buf == '\t' || *buf == ' ')
continue;
}
else {
if (*buf == '<') {
char *p = strchr(buf, '>');
buf = buf + (int)(p - buf);
strcat(tempbuf, ",");
}
else if( *buf == '}'){
} else if (*buf == '}')
strcat(tempbuf, "},");
}
else{
else
strncat(tempbuf, buf, 1);
}
}
}
tempbuf[strlen(tempbuf) + 1] = '\n';
memset(*membuf, 0, strlen(*membuf)) ;
@ -240,16 +245,14 @@ void str_split(char **membuf){
}
BEGINdoAction
msg_t *pMsg;
BEGINdoAction_NoStrings
smsg_t **ppMsg = (smsg_t **) pMsgData;
smsg_t *pMsg = ppMsg[0];
struct json_object *json = NULL;
struct json_object *keyjson = NULL;
char *pszValue;
instanceData *const pData = pWrkrData->pData;
CODESTARTdoAction
pMsg = (msg_t*) ppString[0];
json = json_object_new_object();
/* key is given, so get the property json */
@ -265,7 +268,6 @@ CODESTARTdoAction
/* key found, so get the value */
pszValue = (char*)json_object_get_string(keyjson);
int gai_err, mmdb_err;
MMDB_lookup_result_s result = MMDB_lookup_string(&pWrkrData->mmdb, pszValue, &gai_err, &mmdb_err);
@ -288,9 +290,9 @@ CODESTARTdoAction
ABORT_FINALIZE(RS_RET_OK);
}
FILE *memstream;
char *membuf;
size_t memlen;
char *membuf;
FILE *memstream;
memstream = open_memstream(&membuf, &memlen);
if (entry_data_list != NULL && memstream != NULL) {
@ -314,16 +316,17 @@ CODESTARTdoAction
struct json_object *json1[5] = {NULL};
json_object *temp_json = total_json;
json_object *sub_obj = temp_json;
int j = 0;
char *path[10] = {NULL};
char *sep = "!";
const char *sep = "!";
char *s = strtok(buf, sep);
for (; s != NULL; j++) {
path[j] = s;
s = strtok(NULL, sep);
json_object *sub_obj = json_object_object_get(temp_json, path[j]);
json_object_object_get_ex(temp_json, path[j], &sub_obj);
temp_json = sub_obj;
}
@ -333,20 +336,17 @@ CODESTARTdoAction
json1[j] = json_object_new_object();
json_object_object_add(json1[j], path[j], temp_json);
temp_json = json1[j];
}
else {
} else
json_object_object_add(json, path[j], temp_json);
}
}
}
finalize_it:
if(json) {
if (json)
msgAddJSON(pMsg, (uchar *)JSON_IPLOOKUP_NAME, json, 0, 0);
}
ENDdoAction
@ -378,12 +378,11 @@ CODEqueryEtryPt_STD_CONF2_QUERIES
ENDqueryEtryPt
BEGINmodInit()
CODESTARTmodInit
*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
/* we only support the current interface specification */
*ipIFVersProvided = CURR_MOD_IF_VERSION;
CODEmodInit_QueryRegCFSLineHdlr
dbgprintf("mmdblookup: module compiled with rsyslog version %s.\n", VERSION);
CHKiRet(objUse(errmsg, CORE_COMPONENT));
ENDmodInit

8
tests/Makefile.am
View File

@ -476,10 +476,10 @@ TESTS += \
json_var_case.sh
endif
if ENABLE_MMDBLOOKUP
TESTS += \
mmdb.sh
endif
#if ENABLE_MMDBLOOKUP
#TESTS += \
# mmdb.sh
#endif
if ENABLE_GNUTLS
TESTS += \

0
tests/mmdb.sh Normal file → Executable file
View File

10
tests/travis/install.sh
View File

@ -8,6 +8,16 @@ sudo apt-get install -qq --force-yes libestr-dev librelp-dev libfastjson-dev lib
libcurl4-gnutls-dev
sudo apt-get install -qq python-docutils
if [ "$DISTRIB_CODENAME" == "trusty" ] || [ "$DISTRIB_CODENAME" == "precise" ]; then
set -ex
WANT_MAXMIND=1.2.0
curl -Ls https://github.com/maxmind/libmaxminddb/releases/download/${WANT_MAXMIND}/libmaxminddb-${WANT_MAXMIND}.tar.gz | tar -xz
(cd libmaxminddb-${WANT_MAXMIND} ; ./configure --prefix=/usr CC=gcc CFLAGS="-Wall -Wextra -g -pipe -std=gnu99" ; sudo make install)
set +x
else
sudo apt-get install -qq libmaxminddb-dev
fi
if [ "x$ESTEST" == "xYES" ]; then sudo apt-get install -qq elasticsearch ; fi
if [ "$DISTRIB_CODENAME" == "trusty" ]; then sudo apt-get install -qq libhiredis-dev; export HIREDIS_OPT="--enable-omhiredis"; fi
if [ "$DISTRIB_CODENAME" == "trusty" ]; then sudo apt-get install -qq libsystemd-journal-dev; export JOURNAL_OPT="--enable-imjournal --enable-omjournal"; fi

2
tests/travis/run.sh
View File

@ -38,7 +38,7 @@ if [ "x$ESTEST" == "xYES" ]; then export ES_TEST_CONFIGURE_OPT="--enable-elastic
if [ "$CC" == "clang" ] && [ "$DISTRIB_CODENAME" == "trusty" ]; then export CC="clang-3.6"; fi
$CC -v
env
export CONFIG_FLAGS="--prefix=/opt/rsyslog --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --enable-silent-rules --libdir=/usr/lib64 --docdir=/usr/share/doc/rsyslog --disable-generate-man-pages --enable-testbench --enable-imdiag --enable-imfile --enable-impstats --enable-mmrm1stspace --enable-imptcp --enable-mmanon --enable-mmaudit --enable-mmfields --enable-mmjsonparse --enable-mmpstrucdata --enable-mmsequence --enable-mmutf8fix --enable-mail --enable-omprog --enable-omruleset --enable-omstdout --enable-omuxsock --enable-pmaixforwardedfrom --enable-pmciscoios --enable-pmcisconames --enable-pmlastmsg --enable-pmsnare --enable-libgcrypt --enable-mmnormalize --disable-omudpspoof --enable-relp --disable-snmp --disable-mmsnmptrapd --enable-gnutls --enable-mysql --enable-mysql-tests --enable-usertools --enable-gt-ksi --enable-libdbi --enable-pgsql --enable-omhttpfs --enable-elasticsearch --enable-valgrind --enable-ommongodb --enable-omamqp1 --enable-omrelp-default-port=13515 --enable-omtcl $JOURNAL_OPT $HIREDIS_OPT $ENABLE_KAFKA $NO_VALGRIND $GROK $ES_TEST_CONFIGURE_OPT $CONFIGURE_FLAGS"
export CONFIG_FLAGS="--prefix=/opt/rsyslog --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --enable-silent-rules --libdir=/usr/lib64 --docdir=/usr/share/doc/rsyslog --disable-generate-man-pages --enable-testbench --enable-imdiag --enable-imfile --enable-impstats --enable-mmrm1stspace --enable-imptcp --enable-mmanon --enable-mmaudit --enable-mmfields --enable-mmjsonparse --enable-mmpstrucdata --enable-mmsequence --enable-mmutf8fix --enable-mail --enable-omprog --enable-omruleset --enable-omstdout --enable-omuxsock --enable-pmaixforwardedfrom --enable-pmciscoios --enable-pmcisconames --enable-pmlastmsg --enable-pmsnare --enable-libgcrypt --enable-mmnormalize --disable-omudpspoof --enable-relp --disable-snmp --disable-mmsnmptrapd --enable-gnutls --enable-mysql --enable-mysql-tests --enable-usertools --enable-gt-ksi --enable-libdbi --enable-pgsql --enable-omhttpfs --enable-elasticsearch --enable-valgrind --enable-ommongodb --enable-omamqp1 --enable-omrelp-default-port=13515 --enable-omtcl --enable-mmdblookup $JOURNAL_OPT $HIREDIS_OPT $ENABLE_KAFKA $NO_VALGRIND $GROK $ES_TEST_CONFIGURE_OPT $CONFIGURE_FLAGS"
./configure $CONFIG_FLAGS
export USE_AUTO_DEBUG="off" # set to "on" to enable this for travis
make -j