rsyslog/rsyslog
1
0
Fork 0
mirror of https://github.com/rsyslog/rsyslog.git synced 2025-12-13 08:20:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6316 from alorbach/cursor/add-regex-support-for-trailing-data-removal-default-1d66

Browse Source 
mmsnareparse: add parameter ignoreTrailingPattern.regex
This commit is contained in:
Rainer Gerhards 2025-11-24 17:20:19 +01:00 committed by GitHub
commit c327c574b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 303 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/source/configuration/modules/mmsnareparse.rst
View File

@ -285,7 +285,8 @@ Parameters
"``definition.json``", "string", "``unset``", "Inline JSON descriptor following the same schema as ``definition.file``. Processed after the file-based overrides." "``definition.json``", "string", "``unset``", "Inline JSON descriptor following the same schema as ``definition.file``. Processed after the file-based overrides."
"``runtime.config``", "string", "``unset``", "Persistent runtime configuration file. Supports the definition schema plus ``options`` such as ``enable_debug`` and ``enable_fallback``." "``runtime.config``", "string", "``unset``", "Persistent runtime configuration file. Supports the definition schema plus ``options`` such as ``enable_debug`` and ``enable_fallback``."
"``validation.mode`` / ``validation_mode``", "string", "``permissive``", "Selects parser strictness: ``permissive`` ignores issues, ``moderate`` records warnings, ``strict`` aborts when thresholds are exceeded." "``validation.mode`` / ``validation_mode``", "string", "``permissive``", "Selects parser strictness: ``permissive`` ignores issues, ``moderate`` records warnings, ``strict`` aborts when thresholds are exceeded."
"``ignoreTrailingPattern``", "string", "``unset``", "Pattern that marks the start of a trailing extra-data section to be ignored during parsing. When set, the parser searches for this pattern in trailing positions (after the last tab-separated token). If found, the message is truncated at that point before parsing, and the truncated extra-data section is stored in the ``!extradata_section`` message property. This is useful for removing non-standard trailing enrichment data that may be added by third-party enrichers. The pattern is a literal string match (not a regex)." "``ignoreTrailingPattern``", "string", "``unset``", "Pattern that marks the start of a trailing extra-data section to be ignored during parsing. When set, the parser searches for this pattern in trailing positions (after the last tab-separated token). If found, the message is truncated at the last tab character (removing the entire last token), and the truncated extra-data section (including any dynamic data like numbers preceding the pattern) is stored in the ``!extradata_section`` message property. This is useful for removing non-standard trailing custom data that may be added by third-party enrichers. The pattern is a literal string match (not a regex). This parameter is mutually exclusive with ``ignoreTrailingPattern.regex``."
"``ignoreTrailingPattern.regex``", "string", "``unset``", "POSIX extended regular expression that marks the start of a trailing extra-data section to be ignored during parsing. When set, the parser applies the regex to the trailing token (content after the last tab-separated token). If the regex matches, the message is truncated at the last tab character (removing the entire last token), and the truncated extra-data section is stored in the ``!extradata_section`` message property. This is useful for removing non-standard trailing custom data with dynamic prefixes (e.g., random integers) that may be added by third-party enrichers. Example: ``^[0-9]+\\s+custom_section:`` matches a number followed by whitespace and then the marker. This parameter is mutually exclusive with ``ignoreTrailingPattern``."
Extracted fields Extracted fields
---------------- ----------------

243
plugins/mmsnareparse/mmsnareparse.c
View File

@ -29,6 +29,7 @@
#include <time.h> #include <time.h>
#include <arpa/inet.h> #include <arpa/inet.h>
#include <netinet/in.h> #include <netinet/in.h>
#include <regex.h>
#include "conf.h" #include "conf.h"
#include "datetime.h" #include "datetime.h"
@ -442,6 +443,8 @@ typedef struct _instanceData {
sbool emitRawPayload; sbool emitRawPayload;
sbool emitDebugJson; sbool emitDebugJson;
uchar *ignoreTrailingPattern; uchar *ignoreTrailingPattern;
regex_t ignoreTrailingPattern_preg; /* compiled regex for ignoreTrailingPattern.regex */
sbool ignoreTrailingPattern_isRegex; /* flag indicating if regex mode is enabled */
validation_context_t validationTemplate; validation_context_t validationTemplate;
section_descriptor_t *sectionDescriptors; section_descriptor_t *sectionDescriptors;
size_t sectionDescriptorCount; size_t sectionDescriptorCount;
@ -494,6 +497,7 @@ struct modConfData_s {
char *definitionJson; char *definitionJson;
char *runtimeConfigFile; char *runtimeConfigFile;
uchar *ignoreTrailingPattern; uchar *ignoreTrailingPattern;
uchar *ignoreTrailingPatternRegex;
validation_context_t validationTemplate; validation_context_t validationTemplate;
}; };
static modConfData_t *loadModConf = NULL; static modConfData_t *loadModConf = NULL;
@ -5083,69 +5087,128 @@ static char *detect_and_truncate_trailing_extradata(instanceData *pData, char *m
return NULL; return NULL;
} }
const char *pattern = (const char *)pData->ignoreTrailingPattern;
size_t patternLen = strlen(pattern);
if (patternLen == 0) {
return NULL;
}
/* Find the last tab character to identify the end of the last token */ /* Find the last tab character to identify the end of the last token */
char *lastTab = strrchr(mutableMsg, '\t'); char *lastTab = strrchr(mutableMsg, '\t');
if (lastTab == NULL) { if (lastTab == NULL) {
/* No tabs found - this is an edge case as SNARE format normally uses tab-separated values. /* No tabs found - this is an edge case as SNARE format normally uses tab-separated values.
* However, we still attempt to remove trailing enrichment sections from malformed or * Valid Snare messages will always have tabs and be handled in the else block below,
* where truncation is safely anchored to the last token.
*
* This fallback path attempts to remove trailing enrichment sections from malformed or
* non-standard messages. Be conservative: only truncate if pattern appears in the * non-standard messages. Be conservative: only truncate if pattern appears in the
* trailing portion (last 20% or last 200 chars, whichever is smaller) to avoid * trailing portion (last 20% or last 200 chars, whichever is smaller) to avoid
* accidentally removing legitimate message content. */ * accidentally removing legitimate message content. */
size_t msgLenVal = strlen(mutableMsg); size_t msgLenVal = strlen(mutableMsg);
if (msgLenVal < patternLen) { if (pData->ignoreTrailingPattern_isRegex) {
return NULL; /* For regex mode, search within the trailing portion (last 20% or last 200 chars) */
} size_t trailingSearchLen = msgLenVal / 5; /* Last 20% */
size_t trailingSearchLen = msgLenVal / 5; /* Last 20% */ if (trailingSearchLen > 200) {
if (trailingSearchLen > 200) { trailingSearchLen = 200;
trailingSearchLen = 200; }
} if (trailingSearchLen > msgLenVal) {
if (trailingSearchLen < patternLen) { trailingSearchLen = msgLenVal;
trailingSearchLen = patternLen; }
} char *searchStart = mutableMsg + msgLenVal - trailingSearchLen;
/* Search backwards from end within the trailing portion only */ if (searchStart < mutableMsg) {
char *searchStart = mutableMsg + msgLenVal - trailingSearchLen; searchStart = mutableMsg;
for (char *searchPos = mutableMsg + msgLenVal - patternLen; searchPos >= searchStart; searchPos--) { }
if (memcmp(searchPos, pattern, patternLen) == 0) { /* Try regex match on the trailing portion */
const int isMatch = !regexec(&pData->ignoreTrailingPattern_preg, searchStart, 0, NULL, 0);
if (isMatch) {
/* Found pattern in trailing position - save it and truncate before it */ /* Found pattern in trailing position - save it and truncate before it */
extradataSection = strdup(searchPos); extradataSection = strdup(searchStart);
if (extradataSection == NULL) { if (extradataSection == NULL) {
return NULL; /* out of memory */ return NULL; /* out of memory */
} }
*searchPos = '\0'; *searchStart = '\0';
if (msgLen != NULL) { if (msgLen != NULL) {
*msgLen = searchPos - mutableMsg; *msgLen = searchStart - mutableMsg;
} }
return extradataSection; return extradataSection;
} }
} else {
/* Static pattern mode */
const char *pattern = (const char *)pData->ignoreTrailingPattern;
size_t patternLen = strlen(pattern);
if (patternLen == 0) {
return NULL;
}
if (msgLenVal < patternLen) {
return NULL;
}
size_t trailingSearchLen = msgLenVal / 5; /* Last 20% */
if (trailingSearchLen > 200) {
trailingSearchLen = 200;
}
if (trailingSearchLen < patternLen) {
trailingSearchLen = patternLen;
}
/* Search backwards from end within the trailing portion only */
char *searchStart = mutableMsg + msgLenVal - trailingSearchLen;
for (char *searchPos = mutableMsg + msgLenVal - patternLen; searchPos >= searchStart; searchPos--) {
if (memcmp(searchPos, pattern, patternLen) == 0) {
/* Found pattern in trailing position - save it and truncate before it */
extradataSection = strdup(searchPos);
if (extradataSection == NULL) {
return NULL; /* out of memory */
}
*searchPos = '\0';
if (msgLen != NULL) {
*msgLen = searchPos - mutableMsg;
}
return extradataSection;
}
}
} }
return NULL; return NULL;
} }
/* Pattern must appear after the last tab to be considered trailing */ /* Pattern must appear after the last tab to be considered trailing */
char *searchStart = lastTab + 1; char *searchStart = lastTab + 1;
char *patternPos = strstr(searchStart, pattern);
if (patternPos != NULL) { if (pData->ignoreTrailingPattern_isRegex) {
/* Pattern found in trailing position - truncate at the start of the last token /* Regex mode: apply regexec to the trailing token */
* (after the last tab) to remove the entire enrichment section including any const int isMatch = !regexec(&pData->ignoreTrailingPattern_preg, searchStart, 0, NULL, 0);
* preceding content in that token (e.g., dynamic numbers before the pattern) */ if (isMatch) {
/* Save the extra-data section before truncating */ /* Pattern found in trailing position - truncate at the start of the last token
extradataSection = strdup(searchStart); * (after the last tab) to remove the entire enrichment section including any
if (extradataSection == NULL) { * preceding content in that token (e.g., dynamic numbers before the pattern) */
return NULL; /* Save the extra-data section before truncating */
extradataSection = strdup(searchStart);
if (extradataSection == NULL) {
return NULL;
}
/* Now truncate at the last tab */
*lastTab = '\0';
if (msgLen != NULL) {
*msgLen = lastTab - mutableMsg;
}
return extradataSection;
} }
/* Now truncate at the last tab */ } else {
*lastTab = '\0'; /* Static pattern mode: use strstr */
if (msgLen != NULL) { const char *pattern = (const char *)pData->ignoreTrailingPattern;
*msgLen = lastTab - mutableMsg; /* Defensive check: ensure pattern is not empty to avoid unintended matches */
if (strlen(pattern) > 0) {
char *patternPos = strstr(searchStart, pattern);
if (patternPos != NULL) {
/* Pattern found in trailing position - truncate at the start of the last token
* (after the last tab) to remove the entire enrichment section including any
* preceding content in that token (e.g., dynamic numbers before the pattern) */
/* Save the extra-data section before truncating */
extradataSection = strdup(searchStart);
if (extradataSection == NULL) {
return NULL;
}
/* Now truncate at the last tab */
*lastTab = '\0';
if (msgLen != NULL) {
*msgLen = lastTab - mutableMsg;
}
return extradataSection;
}
} }
return extradataSection;
} }
return NULL; return NULL;
@ -5257,22 +5320,29 @@ static rsRetVal process_message(instanceData *pData, smsg_t *pMsg, uchar *msgTex
DEF_OMOD_STATIC_DATA; DEF_OMOD_STATIC_DATA;
static struct cnfparamdescr modpdescr[] = {{"definition.file", eCmdHdlrString, 0}, static struct cnfparamdescr modpdescr[] = {
{"definition.file", eCmdHdlrString, 0}, {"definition.json", eCmdHdlrString, 0},
{"runtime.config", eCmdHdlrString, 0}, {"validation.mode", eCmdHdlrString, 0},
{"ignoreTrailingPattern", eCmdHdlrString, 0}, {"ignoreTrailingPattern.regex", eCmdHdlrString, 0}};
static struct cnfparamblk modpblk = {CNFPARAMBLK_VERSION, ARRAY_SIZE(modpdescr), modpdescr};
static struct cnfparamdescr actpdescr[] = {{"container", eCmdHdlrString, 0},
{"rootpath", eCmdHdlrString, 0},
{"template", eCmdHdlrGetWord, 0},
{"enable.network", eCmdHdlrBinary, 0},
{"enable.laps", eCmdHdlrBinary, 0},
{"enable.tls", eCmdHdlrBinary, 0},
{"enable.wdac", eCmdHdlrBinary, 0},
{"emit.rawpayload", eCmdHdlrBinary, 0},
{"emit.debugjson", eCmdHdlrBinary, 0},
{"debugjson", eCmdHdlrBinary, 0},
{"definition.file", eCmdHdlrString, 0},
{"definition.json", eCmdHdlrString, 0}, {"definition.json", eCmdHdlrString, 0},
{"runtime.config", eCmdHdlrString, 0}, {"runtime.config", eCmdHdlrString, 0},
{"validation.mode", eCmdHdlrString, 0}, {"validation.mode", eCmdHdlrString, 0},
{"ignoreTrailingPattern", eCmdHdlrString, 0}}; {"validation_mode", eCmdHdlrString, 0},
static struct cnfparamblk modpblk = {CNFPARAMBLK_VERSION, ARRAY_SIZE(modpdescr), modpdescr}; {"ignoreTrailingPattern", eCmdHdlrString, 0},
{"ignoreTrailingPattern.regex", eCmdHdlrString, 0}};
static struct cnfparamdescr actpdescr[] = {
{"container", eCmdHdlrString, 0}, {"rootpath", eCmdHdlrString, 0},
{"template", eCmdHdlrGetWord, 0}, {"enable.network", eCmdHdlrBinary, 0},
{"enable.laps", eCmdHdlrBinary, 0}, {"enable.tls", eCmdHdlrBinary, 0},
{"enable.wdac", eCmdHdlrBinary, 0}, {"emit.rawpayload", eCmdHdlrBinary, 0},
{"emit.debugjson", eCmdHdlrBinary, 0}, {"debugjson", eCmdHdlrBinary, 0},
{"definition.file", eCmdHdlrString, 0}, {"definition.json", eCmdHdlrString, 0},
{"runtime.config", eCmdHdlrString, 0}, {"validation.mode", eCmdHdlrString, 0},
{"validation_mode", eCmdHdlrString, 0}, {"ignoreTrailingPattern", eCmdHdlrString, 0}};
static struct cnfparamblk actpblk = {CNFPARAMBLK_VERSION, ARRAY_SIZE(actpdescr), actpdescr}; static struct cnfparamblk actpblk = {CNFPARAMBLK_VERSION, ARRAY_SIZE(actpdescr), actpdescr};
BEGINbeginCnfLoad BEGINbeginCnfLoad
@ -5287,6 +5357,8 @@ BEGINbeginCnfLoad
pModConf->runtimeConfigFile = NULL; pModConf->runtimeConfigFile = NULL;
free(pModConf->ignoreTrailingPattern); free(pModConf->ignoreTrailingPattern);
pModConf->ignoreTrailingPattern = NULL; pModConf->ignoreTrailingPattern = NULL;
free(pModConf->ignoreTrailingPatternRegex);
pModConf->ignoreTrailingPatternRegex = NULL;
init_validation_context(&pModConf->validationTemplate); init_validation_context(&pModConf->validationTemplate);
ENDbeginCnfLoad ENDbeginCnfLoad
@ -5346,10 +5418,23 @@ BEGINsetModCnf
} }
free(loadModConf->ignoreTrailingPattern); free(loadModConf->ignoreTrailingPattern);
loadModConf->ignoreTrailingPattern = (uchar *)value; loadModConf->ignoreTrailingPattern = (uchar *)value;
} else if (!strcmp(modpblk.descr[i].name, "ignoreTrailingPattern.regex")) {
char *value = es_str2cstr(pvals[i].val.d.estr, NULL);
if (value == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
free(loadModConf->ignoreTrailingPatternRegex);
loadModConf->ignoreTrailingPatternRegex = (uchar *)value;
} else { } else {
dbgprintf("mmsnareparse: unhandled module parameter '%s'\n", modpblk.descr[i].name); dbgprintf("mmsnareparse: unhandled module parameter '%s'\n", modpblk.descr[i].name);
} }
} }
/* Check mutual exclusivity */
if (loadModConf->ignoreTrailingPattern != NULL && loadModConf->ignoreTrailingPatternRegex != NULL) {
LogError(0, RS_RET_PARAM_NOT_PERMITTED,
"mmsnareparse: ignoreTrailingPattern and ignoreTrailingPattern.regex are mutually exclusive");
ABORT_FINALIZE(RS_RET_PARAM_NOT_PERMITTED);
}
finalize_it: finalize_it:
if (pvals != NULL) cnfparamvalsDestruct(pvals, &modpblk); if (pvals != NULL) cnfparamvalsDestruct(pvals, &modpblk);
ENDsetModCnf ENDsetModCnf
@ -5378,6 +5463,8 @@ BEGINfreeCnf
pModConf->runtimeConfigFile = NULL; pModConf->runtimeConfigFile = NULL;
free(pModConf->ignoreTrailingPattern); free(pModConf->ignoreTrailingPattern);
pModConf->ignoreTrailingPattern = NULL; pModConf->ignoreTrailingPattern = NULL;
free(pModConf->ignoreTrailingPatternRegex);
pModConf->ignoreTrailingPatternRegex = NULL;
} }
ENDfreeCnf ENDfreeCnf
@ -5397,6 +5484,9 @@ BEGINfreeInstance
CODESTARTfreeInstance; CODESTARTfreeInstance;
free(pData->container); free(pData->container);
free(pData->ignoreTrailingPattern); free(pData->ignoreTrailingPattern);
if (pData->ignoreTrailingPattern_isRegex) {
regfree(&pData->ignoreTrailingPattern_preg);
}
free_runtime_tables(pData); free_runtime_tables(pData);
free_runtime_config(&pData->runtimeConfig); free_runtime_config(&pData->runtimeConfig);
ENDfreeInstance ENDfreeInstance
@ -5417,6 +5507,8 @@ static inline void setInstParamDefaults(instanceData *pData) {
pData->emitRawPayload = 1; pData->emitRawPayload = 1;
pData->emitDebugJson = 0; pData->emitDebugJson = 0;
pData->ignoreTrailingPattern = NULL; pData->ignoreTrailingPattern = NULL;
pData->ignoreTrailingPattern_isRegex = 0;
memset(&pData->ignoreTrailingPattern_preg, 0, sizeof(regex_t));
init_validation_context(&pData->validationTemplate); init_validation_context(&pData->validationTemplate);
init_runtime_config(&pData->runtimeConfig); init_runtime_config(&pData->runtimeConfig);
pData->sectionDescriptors = NULL; pData->sectionDescriptors = NULL;
@ -5436,6 +5528,8 @@ BEGINnewActInst
char *definitionJson = NULL; char *definitionJson = NULL;
char *runtimeConfigFile = NULL; char *runtimeConfigFile = NULL;
char *templateName = NULL; char *templateName = NULL;
sbool hasStaticPattern = 0;
sbool hasRegexPattern = 0;
CODESTARTnewActInst; CODESTARTnewActInst;
if ((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) { if ((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) {
LogError(0, RS_RET_MISSING_CNFPARAMS, "mmsnareparse: missing configuration parameters"); LogError(0, RS_RET_MISSING_CNFPARAMS, "mmsnareparse: missing configuration parameters");
@ -5455,6 +5549,14 @@ BEGINnewActInst
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
} }
} }
if (loadModConf->ignoreTrailingPatternRegex != NULL) {
free(pData->ignoreTrailingPattern);
pData->ignoreTrailingPattern = (uchar *)strdup((char *)loadModConf->ignoreTrailingPatternRegex);
if (pData->ignoreTrailingPattern == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
pData->ignoreTrailingPattern_isRegex = 1;
}
} }
for (i = 0; i < (int)actpblk.nParams; ++i) { for (i = 0; i < (int)actpblk.nParams; ++i) {
if (!pvals[i].bUsed) continue; if (!pvals[i].bUsed) continue;
@ -5496,12 +5598,51 @@ BEGINnewActInst
CHKiRet(set_validation_mode(pData, mode)); CHKiRet(set_validation_mode(pData, mode));
free(mode); free(mode);
} else if (!strcmp(actpblk.descr[i].name, "ignoreTrailingPattern")) { } else if (!strcmp(actpblk.descr[i].name, "ignoreTrailingPattern")) {
hasStaticPattern = 1;
char *value = es_str2cstr(pvals[i].val.d.estr, NULL); char *value = es_str2cstr(pvals[i].val.d.estr, NULL);
if (value == NULL) { if (value == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
} }
free(pData->ignoreTrailingPattern); free(pData->ignoreTrailingPattern);
pData->ignoreTrailingPattern = (uchar *)value; pData->ignoreTrailingPattern = (uchar *)value;
pData->ignoreTrailingPattern_isRegex = 0;
} else if (!strcmp(actpblk.descr[i].name, "ignoreTrailingPattern.regex")) {
hasRegexPattern = 1;
char *value = es_str2cstr(pvals[i].val.d.estr, NULL);
if (value == NULL) {
ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
}
free(pData->ignoreTrailingPattern);
pData->ignoreTrailingPattern = (uchar *)value;
pData->ignoreTrailingPattern_isRegex = 1;
}
}
/* Check mutual exclusivity - both action parameters cannot be set */
if (hasStaticPattern && hasRegexPattern) {
LogError(0, RS_RET_PARAM_NOT_PERMITTED,
"mmsnareparse: ignoreTrailingPattern and ignoreTrailingPattern.regex are mutually exclusive");
ABORT_FINALIZE(RS_RET_PARAM_NOT_PERMITTED);
}
/* Also check if module config and action config conflict */
if (loadModConf != NULL) {
if ((loadModConf->ignoreTrailingPattern != NULL && hasRegexPattern) ||
(loadModConf->ignoreTrailingPatternRegex != NULL && hasStaticPattern)) {
LogError(0, RS_RET_PARAM_NOT_PERMITTED,
"mmsnareparse: ignoreTrailingPattern and ignoreTrailingPattern.regex are mutually exclusive");
ABORT_FINALIZE(RS_RET_PARAM_NOT_PERMITTED);
}
}
/* Compile regex if regex mode is enabled */
if (pData->ignoreTrailingPattern != NULL && pData->ignoreTrailingPattern_isRegex == 1) {
const int errcode =
regcomp(&pData->ignoreTrailingPattern_preg, (char *)pData->ignoreTrailingPattern, REG_EXTENDED);
if (errcode != 0) {
char errbuff[512];
/* POSIX: Use NULL as regex argument after regcomp failure for portability */
regerror(errcode, NULL, errbuff, sizeof(errbuff));
LogError(0, RS_RET_ERR, "mmsnareparse: error compiling ignoreTrailingPattern.regex '%s': %s",
(char *)pData->ignoreTrailingPattern, errbuff);
ABORT_FINALIZE(RS_RET_ERR);
} }
} }
CODE_STD_STRING_REQUESTnewActInst(1); CODE_STD_STRING_REQUESTnewActInst(1);

4
tests/Makefile.am
View File

@ -514,7 +514,8 @@ TESTS += \
mmsnareparse-kerberos.sh \ mmsnareparse-kerberos.sh \
mmsnareparse-custom.sh \ mmsnareparse-custom.sh \
mmsnareparse-realworld-4624-4634-5140.sh \ mmsnareparse-realworld-4624-4634-5140.sh \
mmsnareparse-trailing-extradata.sh mmsnareparse-trailing-extradata.sh \
mmsnareparse-trailing-extradata-regex.sh
if HAVE_VALGRIND if HAVE_VALGRIND
TESTS += \ TESTS += \
mmsnareparse-comprehensive-vg.sh mmsnareparse-comprehensive-vg.sh
@ -2097,6 +2098,7 @@ EXTRA_DIST= \
mmsnareparse-syslog.sh \ mmsnareparse-syslog.sh \
mmsnareparse-value-types.sh \ mmsnareparse-value-types.sh \
mmsnareparse-trailing-extradata.sh \ mmsnareparse-trailing-extradata.sh \
mmsnareparse-trailing-extradata-regex.sh \
mmexternal-InvldProg-vg.sh \ mmexternal-InvldProg-vg.sh \
nested-call-shutdown.sh \ nested-call-shutdown.sh \
1.rstest 2.rstest 3.rstest err1.rstest \ 1.rstest 2.rstest 3.rstest err1.rstest \

101
tests/mmsnareparse-trailing-extradata-regex.sh Executable file
View File

@ -0,0 +1,101 @@
#!/bin/bash
# Validate mmsnareparse parsing with trailing extra-data section truncation using regex.
# This test verifies regex support for dynamic numeric prefixes in trailing custom data.
unset RSYSLOG_DYNNAME
. ${srcdir:=.}/diag.sh init
generate_conf
add_conf '
module(load="../plugins/mmsnareparse/.libs/mmsnareparse")
template(name="outfmt" type="list") {
property(name="$!win!Event!EventID")
constant(value=",")
property(name="$!win!Event!Channel")
constant(value=",")
property(name="$!win!EventData!EventType")
constant(value=",")
property(name="$!win!EventData!TargetObject")
constant(value=",")
property(name="$!win!EventData!User")
constant(value=",")
property(name="$!extradata_section")
constant(value="\n")
}
action(type="mmsnareparse"
definition.file="../plugins/mmsnareparse/sysmon_definitions.json"
ignoreTrailingPattern.regex="^[0-9]+[[:space:]]+custom_section:")
action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt")
'
startup
# Test case 1: Standard number prefix (3385599)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input1
<14>Mar 22 08:47:23 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20977 Mon Mar 22 08:47:23 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:23.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000000} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 3385599 custom_section: fromhost-ip=192.168.45.217
MSG
# Test case 2: Different number (12345)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input2
<14>Mar 22 08:47:24 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20978 Mon Mar 22 08:47:24 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:24.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000001} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 12345 custom_section: fromhost-ip=192.168.45.218
MSG
# Test case 3: Single digit (9)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input3
<14>Mar 22 08:47:25 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20979 Mon Mar 22 08:47:25 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:25.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000002} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 9 custom_section: fromhost-ip=192.168.45.219
MSG
# Test case 4: Large number (999999999)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input4
<14>Mar 22 08:47:26 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20980 Mon Mar 22 08:47:26 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:26.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000003} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 999999999 custom_section: fromhost-ip=192.168.45.220
MSG
# Test case 5: Multiple spaces (should still match)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input5
<14>Mar 22 08:47:27 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20981 Mon Mar 22 08:47:27 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:27.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000004} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 42 custom_section: fromhost-ip=192.168.45.221
MSG
# Test case 6: Zero-padded number (000123)
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input6
<14>Mar 22 08:47:28 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20982 Mon Mar 22 08:47:28 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:28.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000005} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 000123 custom_section: fromhost-ip=192.168.45.222
MSG
injectmsg_file ${RSYSLOG_DYNNAME}.input1
injectmsg_file ${RSYSLOG_DYNNAME}.input2
injectmsg_file ${RSYSLOG_DYNNAME}.input3
injectmsg_file ${RSYSLOG_DYNNAME}.input4
injectmsg_file ${RSYSLOG_DYNNAME}.input5
injectmsg_file ${RSYSLOG_DYNNAME}.input6
shutdown_when_empty
wait_shutdown
# Test that Event ID 13 (Registry value set) is parsed correctly for all test cases
# The ignoreTrailingPattern.regex should remove the custom sections with dynamic
# numeric prefixes from the message before parsing, so they should NOT appear in any
# parsed fields. However, the truncated content is stored in the !extradata_section property.
# This test verifies:
# 1. Parsing works correctly (EventID=13, Channel, EventType=SetValue, TargetObject, User)
# 2. The custom sections with various numeric prefixes are removed from parsing
# 3. The truncated content (including the numeric prefix) is stored in !extradata_section property
# Test case 1: Standard number (3385599)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,3385599 custom_section: fromhost-ip=192.168.45.217' $RSYSLOG_OUT_LOG
# Test case 2: Different number (12345)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,12345 custom_section: fromhost-ip=192.168.45.218' $RSYSLOG_OUT_LOG
# Test case 3: Single digit (9)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,9 custom_section: fromhost-ip=192.168.45.219' $RSYSLOG_OUT_LOG
# Test case 4: Large number (999999999)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,999999999 custom_section: fromhost-ip=192.168.45.220' $RSYSLOG_OUT_LOG
# Test case 5: Multiple spaces (42 custom_section:)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,42 custom_section: fromhost-ip=192.168.45.221' $RSYSLOG_OUT_LOG
# Test case 6: Zero-padded number (000123)
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,000123 custom_section: fromhost-ip=192.168.45.222' $RSYSLOG_OUT_LOG
exit_test

10
tests/mmsnareparse-trailing-extradata.sh
View File

@ -25,13 +25,13 @@ template(name="outfmt" type="list") {
action(type="mmsnareparse" action(type="mmsnareparse"
definition.file="../plugins/mmsnareparse/sysmon_definitions.json" definition.file="../plugins/mmsnareparse/sysmon_definitions.json"
ignoreTrailingPattern="enrichment_section:") ignoreTrailingPattern="custom_section:")
action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt")
' '
startup startup
cat <<'MSG' > ${RSYSLOG_DYNNAME}.input cat <<'MSG' > ${RSYSLOG_DYNNAME}.input
<14>Mar 22 08:47:23 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20977 Mon Mar 22 08:47:23 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:23.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000000} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 3385599 enrichment_section: fromhost-ip=192.168.45.217 <14>Mar 22 08:47:23 testhost MSWinEventLog 1 Microsoft-Windows-Sysmon/Operational 20977 Mon Mar 22 08:47:23 2025 13 Windows SYSTEM User SetValue testhost Registry value set (rule: RegistryEvent) Registry value set: RuleName: Default RegistryEvent EventType: SetValue UtcTime: 2025-03-22 08:47:23.284 ProcessGuid: {fd4d0da6-d589-6916-eb03-000000000000} ProcessId: 4 Image: System TargetObject: HKLM\System\CurrentControlSet\Services\TestService\ImagePath Details: "C:\Program Files\TestAgent\TestService.exe" User: NT AUTHORITY\SYSTEM 3385599 custom_section: fromhost-ip=192.168.45.217
MSG MSG
injectmsg_file ${RSYSLOG_DYNNAME}.input injectmsg_file ${RSYSLOG_DYNNAME}.input
@ -39,17 +39,17 @@ shutdown_when_empty
wait_shutdown wait_shutdown
# Test that Event ID 13 (Registry value set) is parsed correctly # Test that Event ID 13 (Registry value set) is parsed correctly
# The ignoreTrailingPattern should remove "enrichment_section: fromhost-ip=192.168.45.217" # The ignoreTrailingPattern should remove "custom_section: fromhost-ip=192.168.45.217"
# from the message before parsing, so it should NOT appear in any parsed fields. # from the message before parsing, so it should NOT appear in any parsed fields.
# However, the truncated content is stored in the !extradata_section property. # However, the truncated content is stored in the !extradata_section property.
# This test verifies: # This test verifies:
# 1. Parsing works correctly (EventID=13, Channel, EventType=SetValue, TargetObject, User) # 1. Parsing works correctly (EventID=13, Channel, EventType=SetValue, TargetObject, User)
# 2. The enrichment section is removed from parsing (doesn't affect parsed fields) # 2. The custom section is removed from parsing (doesn't affect parsed fields)
# 3. The truncated content is stored in !extradata_section property (tests with-tabs code path) # 3. The truncated content is stored in !extradata_section property (tests with-tabs code path)
# #
# NOTE: A critical bug in the no-tabs code path (lines 5111-5121 in mmsnareparse.c) was fixed # NOTE: A critical bug in the no-tabs code path (lines 5111-5121 in mmsnareparse.c) was fixed
# where strdup was called AFTER truncation, resulting in an empty string. The fix reverses # where strdup was called AFTER truncation, resulting in an empty string. The fix reverses
# the order: copy first, then truncate. This is now consistent with the with-tabs path. # the order: copy first, then truncate. This is now consistent with the with-tabs path.
content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,3385599 enrichment_section: fromhost-ip=192.168.45.217' $RSYSLOG_OUT_LOG content_check '13,Microsoft-Windows-Sysmon/Operational,SetValue,HKLM\System\CurrentControlSet\Services\TestService\ImagePath,NT AUTHORITY\SYSTEM,3385599 custom_section: fromhost-ip=192.168.45.217' $RSYSLOG_OUT_LOG
exit_test exit_test