rsyslog/rsyslog
1
0
Fork 0
mirror of https://github.com/rsyslog/rsyslog.git synced 2025-12-15 15:10:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

omelasticsearch - retry: set rawmsg to data from original request

Browse Source 
https://github.com/rsyslog/rsyslog/issues/3573
Previously, when constructing the message to submit for a retry
for an original request, if the original request did not contain
the field `message`, the system property `rawmsg` was set to
the entire metadata + data from the original request.  This was
causing problems with Elasticsearch.  This patch changes
the code so that the `rawmsg` will be set to only the data part
of the original request if there is no `message` field.
This commit is contained in:
Rich Megginson 2019-03-29 11:08:22 -06:00
parent d56541a7a1
commit 37edda0a8d
2 changed files with 43 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/omelasticsearch/omelasticsearch.c
View File

@ -1138,7 +1138,8 @@ createMsgFromRequest(const char *request, context *ctx, smsg_t **msg, fjson_obje
const size_t msgLen = (size_t)json_object_get_string_len(jo_msg); const size_t msgLen = (size_t)json_object_get_string_len(jo_msg);
MsgSetRawMsg(*msg, rawmsg, msgLen); MsgSetRawMsg(*msg, rawmsg, msgLen);
} else { } else {
MsgSetRawMsg(*msg, request, strlen(request)); /* use entire data part of request as rawmsg */
MsgSetRawMsg(*msg, datastart, datalen);
} }
MsgSetMSGoffs(*msg, 0); /* we do not have a header... */ MsgSetMSGoffs(*msg, 0); /* we do not have a header... */
MsgSetTAG(*msg, (const uchar *)"omes", 4); MsgSetTAG(*msg, (const uchar *)"omes", 4);

64
tests/es-bulk-retry.sh
View File

@ -22,17 +22,28 @@ module(load="../plugins/impstats/.libs/impstats" interval="1"
log.file="'$RSYSLOG_DYNNAME'.spool/es-stats.log" log.syslog="off" format="cee") log.file="'$RSYSLOG_DYNNAME'.spool/es-stats.log" log.syslog="off" format="cee")
set $.msgnum = field($msg, 58, 2); set $.msgnum = field($msg, 58, 2);
set $.testval = cnum($.msgnum % 2); set $.testval = cnum($.msgnum % 4);
if $.testval == 0 then { if $.testval == 0 then {
# these should be successful # these should be successful
set $!msgnum = $.msgnum; set $!msgnum = $.msgnum;
set $.extrafield = "notmessage";
} else if $.testval == 1 then {
# these should cause "hard" errors
set $!msgnum = "x" & $.msgnum;
set $.extrafield = "notmessage";
} else if $.testval == 2 then {
# these should be successful
set $!msgnum = $.msgnum;
set $.extrafield = "message";
} else { } else {
# these should cause "hard" errors # these should cause "hard" errors
set $!msgnum = "x" & $.msgnum; set $!msgnum = "x" & $.msgnum;
set $.extrafield = "message";
} }
template(name="tpl" type="string" template(name="tpl" type="string"
string="{\"msgnum\":\"%$!msgnum%\"}") string="{\"msgnum\":\"%$!msgnum%\",\"%$.extrafield%\":\"extrafieldvalue\"}")
module(load="../plugins/omelasticsearch/.libs/omelasticsearch") module(load="../plugins/omelasticsearch/.libs/omelasticsearch")
@ -43,39 +54,46 @@ ruleset(name="error_es") {
} }
ruleset(name="try_es") { ruleset(name="try_es") {
set $.sendrec = 1;
if strlen($.omes!status) > 0 then { if strlen($.omes!status) > 0 then {
# retry case # retry case
if ($.omes!status == 200) or ($.omes!status == 201) or (($.omes!status == 409) and ($.omes!writeoperation == "create")) then { if ($.omes!status == 200) or ($.omes!status == 201) or (($.omes!status == 409) and ($.omes!writeoperation == "create")) then {
stop # successful reset $.sendrec = 0; # successful
} }
if ($.omes!writeoperation == "unknown") or (strlen($.omes!error!type) == 0) or (strlen($.omes!error!reason) == 0) then { if ($.omes!writeoperation == "unknown") or (strlen($.omes!error!type) == 0) or (strlen($.omes!error!reason) == 0) then {
call error_es call error_es
stop reset $.sendrec = 0;
} }
if ($.omes!status == 400) or ($.omes!status < 200) then { if ($.omes!status == 400) or ($.omes!status < 200) then {
call error_es call error_es
stop reset $.sendrec = 0;
}
if strlen($!notmessage) > 0 then {
set $.extrafield = "notmessage";
} else {
set $.extrafield = "message";
} }
# else fall through to retry operation
} }
if strlen($.omes!_id) > 0 then { if $.sendrec == 1 then {
set $.es_msg_id = $.omes!_id; if strlen($.omes!_id) > 0 then {
} else { set $.es_msg_id = $.omes!_id;
# NOTE: in production code, use $uuid - depends on rsyslog being compiled with --enable-uuid } else {
set $.es_msg_id = $.msgnum; # NOTE: in production code, use $uuid - depends on rsyslog being compiled with --enable-uuid
set $.es_msg_id = $.msgnum;
}
action(type="omelasticsearch"
server="127.0.0.1"
serverport="'${ES_PORT:-19200}'"
template="tpl"
writeoperation="create"
bulkid="id-template"
dynbulkid="on"
bulkmode="on"
retryfailures="on"
retryruleset="try_es"
searchType="test-type"
searchIndex="rsyslog_testbench")
} }
action(type="omelasticsearch"
server="127.0.0.1"
serverport="'${ES_PORT:-19200}'"
template="tpl"
writeoperation="create"
bulkid="id-template"
dynbulkid="on"
bulkmode="on"
retryfailures="on"
retryruleset="try_es"
searchType="test-type"
searchIndex="rsyslog_testbench")
} }
if $msg contains "msgnum:" then { if $msg contains "msgnum:" then {