rsyslog/rsyslog
1
0
Fork 0
mirror of https://github.com/rsyslog/rsyslog.git synced 2025-12-19 04:30:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2749 from jvymazal/wildcard_symlink

Browse Source 
Symlink support for imfile
This commit is contained in:
Rainer Gerhards 2018-08-03 13:32:25 +02:00 committed by GitHub
commit 3f9c1a423a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 258 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
plugins/imfile/imfile.c
View File

@ -152,6 +152,7 @@ struct act_obj_s {
fs_edge_t *edge; /* edge which this object belongs to */
char *name; /* full path name of active object */
char *basename; /* only basename */ //TODO: remove when refactoring rename support
char *source_name; /* if this object is target of a symlink, source_name is its name (else NULL) */
//char *statefile; /* base name of state file (for move operations) */
int wd;
#if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE)
@ -167,6 +168,7 @@ struct act_obj_s {
int nRecords; /**< How many records did we process before persisting the stream? */
ratelimit_t *ratelimiter;
multi_submit_t multiSub;
int is_symlink;
};
struct fs_edge_s {
fs_node_t *parent;
@ -189,7 +191,7 @@ static rsRetVal persistStrmState(act_obj_t *);
static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal);
static rsRetVal ATTR_NONNULL(1) pollFile(act_obj_t *act);
static int ATTR_NONNULL() getBasename(uchar *const __restrict__ basen, uchar *const __restrict__ path);
static void ATTR_NONNULL() act_obj_unlink(act_obj_t *const act);
static void ATTR_NONNULL() act_obj_unlink(act_obj_t *act);
static uchar * ATTR_NONNULL(1, 2) getStateFileName(const act_obj_t *, uchar *, const size_t);
static int ATTR_NONNULL() getFullStateFileName(const uchar *const, uchar *const pszout, const size_t ilenout);
@ -483,14 +485,17 @@ in_setupWatch(act_obj_t *const act, const int is_file)
goto done;
wd = inotify_add_watch(ino_fd, act->name,
(is_file) ? IN_MODIFY : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO);
(is_file) ? IN_MODIFY|IN_DONT_FOLLOW : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO);
if(wd < 0) {
LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'",
act->name);
if (errno == EACCES) { /* There is high probability of selinux denial on top-level paths */
DBGPRINTF("imfile: permission denied when adding watch for '%s'\n", act->name);
} else {
LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'", act->name);
}
goto done;
}
wdmapAdd(wd, act);
DBGPRINTF("in_setupDirWatch: watch %d added for dir %s(%p)\n", wd, act->name, act);
DBGPRINTF("in_setupWatch: watch %d added for %s(object %p)\n", wd, act->name, act);
done: return wd;
}
@ -605,7 +610,7 @@ done: return;
static void ATTR_NONNULL()
fen_setupWatch(act_obj_t *const act __attribute__((unused)))
{
DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?");
DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?\n");
}
#endif /* FEN */
@ -633,38 +638,48 @@ fs_node_print(const fs_node_t *const node, const int level)
}
}
/* add a new file system object if it not yet exists, ignore call
* if it already does.
*/
static rsRetVal ATTR_NONNULL()
static rsRetVal ATTR_NONNULL(1,2)
act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file,
const ino_t ino)
const ino_t ino, const int is_symlink, const char *const source)
{
act_obj_t *act;
char basename[MAXFNAME];
DEFiRet;
DBGPRINTF("act_obj_add: edge %p, name '%s'\n", edge, name);
DBGPRINTF("act_obj_add: edge %p, name '%s' (source '%s')\n", edge, name, source? source : "---");
for(act = edge->active ; act != NULL ; act = act->next) {
if(!strcmp(act->name, name)) {
DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n",
name, edge->path);
FINALIZE;
if (!source || !act->source_name || !strcmp(act->source_name, source)) {
DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n",
name, edge->path);
FINALIZE;
}
}
}
DBGPRINTF("add new active object '%s' in '%s'\n", name, edge->path);
CHKmalloc(act = calloc(sizeof(act_obj_t), 1));
CHKmalloc(act->name = strdup(name));
getBasename((uchar*)basename, (uchar*)name);
CHKmalloc(act->basename = strdup(basename));
if (-1 == getBasename((uchar*)basename, (uchar*)name)) {
CHKmalloc(act->basename = strdup(name)); /* assume basename is same as name */
} else {
CHKmalloc(act->basename = strdup(basename));
}
act->edge = edge;
act->ino = ino;
act->is_symlink = is_symlink;
if (source) { /* we are target of symlink */
CHKmalloc(act->source_name = strdup(source));
} else {
act->source_name = NULL;
}
#ifdef HAVE_INOTIFY_INIT
act->wd = in_setupWatch(act, is_file);
#endif
fen_setupWatch(act);
if(is_file) {
if(is_file && !is_symlink) {
const instanceConf_t *const inst = edge->instarr[0];// TODO: same file, multiple instances?
CHKiRet(ratelimitNew(&act->ratelimiter, "imfile", name));
CHKmalloc(act->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(smsg_t *)));
@ -702,27 +717,24 @@ detect_updates(fs_edge_t *const edge)
{
act_obj_t *act;
struct stat fileInfo;
int restart = 0;
for(act = edge->active ; act != NULL ; ) {
for(act = edge->active ; act != NULL ; act = act->next) {
DBGPRINTF("detect_updates checking active obj '%s'\n", act->name);
const int r = stat(act->name, &fileInfo);
const int r = lstat(act->name, &fileInfo);
if(r == -1) { /* object gone away? */
DBGPRINTF("object gone away, unlinking: '%s'\n", act->name);
act_obj_t *toDel = act;
act = act->next;
DBGPRINTF("new next act %p\n", act);
act_obj_unlink(toDel);
continue;
act_obj_unlink(act);
restart = 1;
break;
}
// TODO: add inode check for change notification!
/* Note: active nodes may get deleted, so we need to do the
* pointer advancement at the end of the for loop!
*/
act = act->next;
}
if (restart) {
detect_updates(edge);
}
}
@ -746,14 +758,39 @@ poll_active_files(fs_edge_t *const edge)
}
}
static rsRetVal ATTR_NONNULL()
process_symlink(fs_edge_t *const chld, const char *symlink)
{
DEFiRet;
char *target = NULL;
CHKmalloc(target = realpath(symlink, target));
struct stat fileInfo;
if(lstat(target, &fileInfo) != 0) {
LogError(errno, RS_RET_ERR, "imfile: process_symlink cannot stat file '%s' - ignored", target);
FINALIZE;
}
const int is_file = (S_ISREG(fileInfo.st_mode));
DBGPRINTF("process_symlink: found '%s', File: %d (config file: %d), symlink: %d\n",
target, is_file, chld->is_file, 0);
if(!is_file && S_ISREG(fileInfo.st_mode)) {
LogMsg(0, RS_RET_ERR, LOG_WARNING,
"imfile: '%s' is neither a regular file, symlink, nor a directory - ignored", target);
FINALIZE;
}
act_obj_add(chld, target, is_file, fileInfo.st_ino, 0, symlink);
finalize_it:
free(target);
RETiRet;
}
static void ATTR_NONNULL() poll_tree(fs_edge_t *const chld);
static void ATTR_NONNULL()
poll_tree(fs_edge_t *const chld)
{
struct stat fileInfo;
glob_t files;
int need_globfree = 0;
int issymlink;
DBGPRINTF("poll_tree: chld %p, name '%s', path: %s\n", chld, chld->name, chld->path);
detect_updates(chld);
const int ret = glob((char*)chld->path, runModConf->sortFiles|GLOB_BRACE, NULL, &files);
@ -766,18 +803,27 @@ poll_tree(fs_edge_t *const chld)
goto done;
}
char *const file = files.gl_pathv[i];
if(stat(file, &fileInfo) != 0) {
if(lstat(file, &fileInfo) != 0) {
LogError(errno, RS_RET_ERR,
"imfile: poll_tree cannot stat file '%s' - ignored", file);
continue;
}
const int is_file = S_ISREG(fileInfo.st_mode);
DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d)\n",
file, is_file, chld->is_file);
if (S_ISLNK(fileInfo.st_mode)) {
rsRetVal slink_ret = process_symlink(chld, file);
if (slink_ret != RS_RET_OK) {
continue;
}
issymlink = 1;
} else {
issymlink = 0;
}
const int is_file = (S_ISREG(fileInfo.st_mode) || issymlink);
DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d), symlink: %d\n",
file, is_file, chld->is_file, issymlink);
if(!is_file && S_ISREG(fileInfo.st_mode)) {
LogMsg(0, RS_RET_ERR, LOG_WARNING,
"imfile: '%s' is neither a regular file nor a "
"imfile: '%s' is neither a regular file, symlink, nor a "
"directory - ignored", file);
continue;
}
@ -788,7 +834,7 @@ poll_tree(fs_edge_t *const chld)
(chld->is_file) ? "FILE" : "DIRECTORY");
continue;
}
act_obj_add(chld, file, is_file, fileInfo.st_ino);
act_obj_add(chld, file, is_file, fileInfo.st_ino, issymlink, NULL);
}
}
@ -829,8 +875,20 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted)
if(act == NULL)
return;
DBGPRINTF("act_obj_destroy: act %p '%s', wd %d, pStrm %p, is_deleted %d, in_move %d\n",
act, act->name, act->wd, act->pStrm, is_deleted, act->in_move);
DBGPRINTF("act_obj_destroy: act %p '%s' (source '%s'), wd %d, pStrm %p, is_deleted %d, in_move %d\n",
act, act->name, act->source_name? act->source_name : "---", act->wd, act->pStrm, is_deleted,
act->in_move);
if(act->is_symlink && is_deleted) {
act_obj_t *target_act;
for(target_act = act->edge->active ; target_act != NULL ; target_act = target_act->next) {
if(target_act->source_name && !strcmp(target_act->source_name, act->name)) {
DBGPRINTF("act_obj_destroy: unlinking slink target %s of %s "
"symlink\n", target_act->name, act->name);
act_obj_unlink(target_act);
break;
}
}
}
if(act->ratelimiter != NULL) {
ratelimitDestruct(act->ratelimiter);
}
@ -862,6 +920,7 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted)
}
#endif
free(act->basename);
free(act->source_name);
//free(act->statefile);
free(act->multiSub.ppMsgs);
#if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE)
@ -909,7 +968,7 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted)
* destruct it.
*/
static void //ATTR_NONNULL()
act_obj_unlink(act_obj_t *const act)
act_obj_unlink(act_obj_t *act)
{
DBGPRINTF("act_obj_unlink %p: %s\n", act, act->name);
if(act->prev == NULL) {
@ -921,6 +980,7 @@ act_obj_unlink(act_obj_t *const act)
act->next->prev = act->prev;
}
act_obj_destroy(act, 1);
act = NULL;
//dbgprintf("printout of fs tree post unlink\n");
//fs_node_print(runModConf->conf_tree, 0);
//dbg_wdmapPrint("wdmap after");
@ -1162,7 +1222,11 @@ enqLine(act_obj_t *const act,
msgSetPRI(pMsg, inst->iFacility | inst->iSeverity);
MsgSetRuleset(pMsg, inst->pBindRuleset);
if(inst->addMetadata) {
metadata_values[0] = (const uchar*)act->name;
if (act->source_name) {
metadata_values[0] = (const uchar*)act->source_name;
} else {
metadata_values[0] = (const uchar*)act->name;
}
snprintf((char *)file_offset, MAX_OFFSET_REPRESENTATION_NUM_BYTES+1, "%lld", strtOffs);
metadata_values[1] = file_offset;
msgAddMultiMetadata(pMsg, metadata_names, metadata_values, 2);
@ -1389,13 +1453,16 @@ pollFile(act_obj_t *const act)
{
cstr_t *pCStr = NULL;
DEFiRet;
if (act->is_symlink) {
FINALIZE; /* no reason to poll symlink file */
}
/* Note: we must do pthread_cleanup_push() immediately, because the POSIX macros
* otherwise do not work if I include the _cleanup_pop() inside an if... -- rgerhards, 2008-08-14
*/
pthread_cleanup_push(pollFileCancelCleanup, &pCStr);
iRet = pollFileReal(act, &pCStr);
pthread_cleanup_pop(0);
RETiRet;
finalize_it: RETiRet;
}
@ -2057,7 +2124,7 @@ in_processEvent(struct inotify_event *ev)
}
if(ev->mask & (IN_MOVED_FROM | IN_MOVED_TO)) {
fs_node_walk(etry->act->edge->node, poll_tree);
} else if(etry->act->edge->is_file) {
} else if(etry->act->edge->is_file && !(etry->act->is_symlink)) {
in_handleFileEvent(ev, etry); // esentially poll_file()!
} else {
fs_node_walk(etry->act->edge->node, poll_tree);

4
tests/Makefile.am
View File

@ -877,6 +877,8 @@ TESTS += \
imfile-old-state-file.sh \
imfile-rename-while-stopped.sh \
imfile-rename.sh \
imfile-symlink.sh \
imfile-symlink-multi.sh \
glbl-oversizeMsg-truncate-imfile.sh
if HAVE_VALGRIND
@ -1594,6 +1596,8 @@ EXTRA_DIST= \
imfile-old-state-file.sh \
imfile-rename-while-stopped.sh \
imfile-rename.sh \
imfile-symlink.sh \
imfile-symlink-multi.sh \
glbl-oversizeMsg-truncate-imfile.sh \
testsuites/imfile-wildcards-simple.conf \
testsuites/imfile-wildcards-dirs.conf \

2
tests/diag.sh
View File

@ -416,6 +416,7 @@ function exit_test() {
rm -f rsyslog.random.data rsyslog.pipe
rm -f -r rsyslog.input.*
rm -f rsyslog.input rsyslog.conf.tlscert stat-file1 rsyslog.empty rsyslog.input.* imfile-state*
rm -rf rsyslog.input-symlink.log rsyslog-link.*.log targets
rm -f testconf.conf
rm -f rsyslog.errorfile tmp.qi nocert
rm -f HOSTNAME imfile-state:.-rsyslog.input
@ -514,6 +515,7 @@ case $1 in
rm -rf test-spool test-logdir stat-file1
rm -f rsyslog.pipe rsyslog.input.*
rm -f rsyslog.input rsyslog.empty rsyslog.input.* imfile-state* omkafka-failed.data
rm -rf rsyslog.input-symlink.log rsyslog-link.*.log targets
rm -f testconf*.conf HOSTNAME
rm -f rsyslog.errorfile tmp.qi nocert
rm -f core.* vgcore.* core*

68
tests/imfile-symlink-multi.sh Executable file
View File

@ -0,0 +1,68 @@
#!/bin/bash
# This test points multiple symlinks (all watched by rsyslog via wildcard)
# to single file and checks that message is reported once for each symlink
# with correct corresponding metadata.
# This is part of the rsyslog testbench, released under ASL 2.0
export IMFILEINPUTFILES="10"
echo [imfile-symlink.sh]
. $srcdir/diag.sh check-inotify
. $srcdir/diag.sh init
# generate input files first. Note that rsyslog processes it as
# soon as it start up (so the file should exist at that point).
imfilebefore="rsyslog.input-symlink.log"
./inputfilegen -m 1 > $imfilebefore
mkdir targets
generate_conf
add_conf '
# comment out if you need more debug info:
global( debug.whitelist="on"
debug.files=["imfile.c"])
module(load="../plugins/imfile/.libs/imfile"
mode="inotify" normalizePath="off")
input(type="imfile" File="./rsyslog.input-symlink.log" Tag="file:"
Severity="error" Facility="local7" addMetadata="on")
input(type="imfile" File="./rsyslog.input.*.log" Tag="file:"
Severity="error" Facility="local7" addMetadata="on")
template(name="outfmt" type="list") {
constant(value="HEADER ")
property(name="msg" format="json")
constant(value=", filename: ")
property(name="$!metadata!filename")
constant(value=", fileoffset: ")
property(name="$!metadata!fileoffset")
constant(value="\n")
}
if $msg contains "msgnum:" then
action( type="omfile" file="rsyslog.out.log" template="outfmt")
'
# Start rsyslog now before adding more files
startup
cp $imfilebefore targets/target.log
for i in `seq 2 $IMFILEINPUTFILES`;
do
ln -s targets/target.log rsyslog.input.$i.log
# Wait little for correct timing
./msleep 50
done
shutdown_when_empty # shut down rsyslogd when done processing messages
wait_shutdown # we need to wait until rsyslogd is finished!
sort rsyslog.out.log > rsyslog.out.sorted.log
{
echo HEADER msgnum:00000000:, filename: ./rsyslog.input-symlink.log, fileoffset: 0
for i in `seq 2 $IMFILEINPUTFILES` ; do
echo HEADER msgnum:00000000:, filename: ./rsyslog.input.${i}.log, fileoffset: 0
done
} | sort | cmp - rsyslog.out.sorted.log
if [ ! $? -eq 0 ]; then
echo "invalid output generated, rsyslog.out.log is:"
cat rsyslog.out.log
error_exit 1
fi;
exit_test

76
tests/imfile-symlink.sh Executable file
View File

@ -0,0 +1,76 @@
#!/bin/bash
# This test creates multiple symlinks (all watched by rsyslog via wildcard)
# chained to target files via additional symlinks and checks that all files
# are recorded with correct corresponding metadata (name of symlink
# matching configuration).
# This is part of the rsyslog testbench, released under ASL 2.0
export IMFILEINPUTFILES="10"
echo [imfile-symlink.sh]
. $srcdir/diag.sh check-inotify
. $srcdir/diag.sh init
# generate input files first. Note that rsyslog processes it as
# soon as it start up (so the file should exist at that point).
imfilebefore="rsyslog.input-symlink.log"
./inputfilegen -m 1 > $imfilebefore
mkdir targets
generate_conf
add_conf '
# comment out if you need more debug info:
global( debug.whitelist="on"
debug.files=["imfile.c"])
module(load="../plugins/imfile/.libs/imfile"
mode="inotify" normalizePath="off")
input(type="imfile" File="./rsyslog.input-symlink.log" Tag="file:"
Severity="error" Facility="local7" addMetadata="on")
input(type="imfile" File="./rsyslog.input.*.log" Tag="file:"
Severity="error" Facility="local7" addMetadata="on")
template(name="outfmt" type="list") {
constant(value="HEADER ")
property(name="msg" format="json")
constant(value=", filename: ")
property(name="$!metadata!filename")
constant(value=", fileoffset: ")
property(name="$!metadata!fileoffset")
constant(value="\n")
}
if $msg contains "msgnum:" then
action( type="omfile" file="rsyslog.out.log" template="outfmt")
'
# Start rsyslog now before adding more files
startup
for i in `seq 2 $IMFILEINPUTFILES`;
do
cp $imfilebefore targets/$i.log
ln -s targets/$i.log rsyslog-link.$i.log
ln -s rsyslog-link.$i.log rsyslog.input.$i.log
imfilebefore="targets/$i.log"
# Wait little for correct timing
./msleep 50
done
./inputfilegen -m 3 > rsyslog.input.$((IMFILEINPUTFILES + 1)).log
ls -l rsyslog.input.* rsyslog-link.* targets
shutdown_when_empty # shut down rsyslogd when done processing messages
wait_shutdown # we need to wait until rsyslogd is finished!
sort rsyslog.out.log > rsyslog.out.sorted.log
{
echo HEADER msgnum:00000000:, filename: ./rsyslog.input-symlink.log, fileoffset: 0
for i in `seq 2 $IMFILEINPUTFILES` ; do
echo HEADER msgnum:00000000:, filename: ./rsyslog.input.${i}.log, fileoffset: 0
done
echo HEADER msgnum:00000000:, filename: ./rsyslog.input.11.log, fileoffset: 0
echo HEADER msgnum:00000001:, filename: ./rsyslog.input.11.log, fileoffset: 17
echo HEADER msgnum:00000002:, filename: ./rsyslog.input.11.log, fileoffset: 34
} | sort | cmp - rsyslog.out.sorted.log
if [ ! $? -eq 0 ]; then
echo "invalid output generated, rsyslog.out.log is:"
cat rsyslog.out.log
error_exit 1
fi;
exit_test