logenc: milestone: rsyslog writes .encinfo side files

runtime/cryprov.h

@@ -33,7 +33,7 @@ BEGINinterface(cryprov) /* name must also be changed in ENDinterface macro! */
 	rsRetVal (*Destruct)(void *ppThis);
 	rsRetVal (*OnFileOpen)(void *pThis, uchar *fn, void *pFileInstData);
 	rsRetVal (*Encrypt)(void *pFileInstData, uchar *buf, size_t *lenBuf);
-	rsRetVal (*OnFileClose)(void *pFileInstData);
+	rsRetVal (*OnFileClose)(void *pFileInstData, off64_t offsLogfile);
 ENDinterface(cryprov)
 #define cryprovCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */
 rsRetVal initCrypt(int gcry_mode, char * iniVector);

runtime/libgcry.c

@@ -2,6 +2,22 @@
  *
  * Copyright 2013 Adiscon GmbH.
  *
+ * We need to store some additional information in support of encryption.
+ * For this, we create a side-file, which is named like the actual log
+ * file, but with the suffix ".encinfo" appended. It contains the following
+ * records:
+ * IV:<hex>   The initial vector used at block start. Also indicates start
+ *            start of block.
+ * END:<int>  The end offset of the block, as uint64_t in decimal notation.
+ *            This is used during encryption to know when the current
+ *            encryption block ends.
+ * For the current implementation, there must always be an IV record
+ * followed by an END record. Each records is LF-terminated. Record
+ * types can simply be extended in the future by specifying new 
+ * keywords (like "IV") before the colon.
+ * To identify a file as rsyslog encryption info file, it must start with
+ * the line "FILETYPE:rsyslog-enrcyption-info"
+ *
  * This file is part of rsyslog.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,21 +39,166 @@
 #endif
 #include <stdio.h>
 #include <gcrypt.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <fcntl.h>
+#include <errno.h>
 
 #include "rsyslog.h"
 #include "libgcry.h"
 
 #define GCRY_CIPHER GCRY_CIPHER_3DES  // TODO: make configurable
 
-static inline gcryfile
+
-gcryfileConstruct(gcryctx ctx)
+static rsRetVal
+eiWriteRec(gcryfile gf, char *recHdr, size_t lenRecHdr, char *buf, size_t lenBuf)
 {
-	gcryfile gf;
+	struct iovec iov[3];
-	if((gf = calloc(1, sizeof(struct gcryfile_s))) == NULL)
+	ssize_t nwritten, towrite;
-		goto done;
+	DEFiRet;
-	gf->ctx = ctx;
+
-done:	return gf;
+	iov[0].iov_base = recHdr;
+	iov[0].iov_len = lenRecHdr;
+	iov[1].iov_base = buf;
+	iov[1].iov_len = lenBuf;
+	iov[2].iov_base = "\n";
+	iov[2].iov_len = 1;
+	towrite = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len;
+	nwritten = writev(gf->fd, iov, sizeof(iov)/sizeof(struct iovec));
+	if(nwritten != towrite) {
+		DBGPRINTF("eiWrite%s: error writing file, towrite %d, "
+			"nwritten %d\n", recHdr, (int) towrite, (int) nwritten);
+		ABORT_FINALIZE(RS_RET_EI_WR_ERR);
+	}
+	DBGPRINTF("encryption info file %s: written %s, len %d\n",
+		  recHdr, gf->eiName, (int) nwritten);
+finalize_it:
+	RETiRet;
 }
+
+static rsRetVal
+eiOpenRead(gcryfile gf)
+{
+	DEFiRet;
+	gf->fd = open((char*)gf->eiName, O_RDONLY|O_NOCTTY|O_CLOEXEC);
+	if(gf->fd == -1) {
+		ABORT_FINALIZE(errno == ENOENT ? RS_RET_EI_NO_EXISTS : RS_RET_EI_OPN_ERR);
+	}
+finalize_it:
+	RETiRet;
+}
+
+
+static rsRetVal
+eiCheckFiletype(gcryfile gf)
+{
+	char hdrBuf[128];
+	size_t toRead, didRead;
+	DEFiRet;
+
+	CHKiRet(eiOpenRead(gf));
+	if(Debug) memset(hdrBuf, 0, sizeof(hdrBuf)); /* for dbgprintf below! */
+	toRead = sizeof("FILETYPE:")-1 + sizeof(RSGCRY_FILETYPE_NAME)-1 + 1;
+	didRead = read(gf->fd, hdrBuf, toRead);
+	close(gf->fd);
+	DBGPRINTF("eiCheckFiletype read %d bytes: '%s'\n", didRead, hdrBuf);
+	if(   didRead != toRead
+	   || strncmp(hdrBuf, "FILETYPE:" RSGCRY_FILETYPE_NAME "\n", toRead))
+		iRet = RS_RET_EI_INVLD_FILE;
+finalize_it:
+	RETiRet;
+}
+
+static rsRetVal
+eiOpenAppend(gcryfile gf)
+{
+	rsRetVal localRet;
+	DEFiRet;
+	localRet = eiCheckFiletype(gf);
+	if(localRet == RS_RET_OK) {
+		gf->fd = open((char*)gf->eiName,
+			       O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC, 0600);
+		if(gf->fd == -1) {
+			ABORT_FINALIZE(RS_RET_EI_OPN_ERR);
+		}
+	} else if(localRet == RS_RET_EI_NO_EXISTS) {
+		/* looks like we need to create a new file */
+		gf->fd = open((char*)gf->eiName,
+			       O_WRONLY|O_CREAT|O_NOCTTY|O_CLOEXEC, 0600);
+		if(gf->fd == -1) {
+			ABORT_FINALIZE(RS_RET_EI_OPN_ERR);
+		}
+		CHKiRet(eiWriteRec(gf, "FILETYPE:", 9, RSGCRY_FILETYPE_NAME,
+			    sizeof(RSGCRY_FILETYPE_NAME)-1));
+	} else {
+		gf->fd = -1;
+		ABORT_FINALIZE(localRet);
+	}
+	DBGPRINTF("encryption info file %s: opened as #%d\n",
+		gf->eiName, gf->fd);
+finalize_it:
+	RETiRet;
+}
+
+static rsRetVal
+eiWriteIV(gcryfile gf, uchar *iv)
+{
+	static const char hexchars[16] =
+	   {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
+	unsigned iSrc, iDst;
+	char hex[4096];
+	DEFiRet;
+
+	if(gf->blkLength > sizeof(hex)/2) {
+		DBGPRINTF("eiWriteIV: crypto block len way too large, aborting "
+			  "write");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	for(iSrc = iDst = 0 ; iSrc < gf->blkLength ; ++iSrc) {
+		hex[iDst++] = hexchars[iv[iSrc]>>4];
+		hex[iDst++] = hexchars[iv[iSrc]&0x0f];
+	}
+
+	iRet = eiWriteRec(gf, "IV:", 3, hex, gf->blkLength*2);
+finalize_it:
+	RETiRet;
+}
+
+/* we do not return an error state, as we MUST close the file,
+ * no matter what happens.
+ */
+static void
+eiClose(gcryfile gf, off64_t offsLogfile)
+{
+	char offs[21];
+	size_t len;
+	if(gf->fd == -1)
+		return;
+	/* 2^64 is 20 digits, so the snprintf buffer is large enough */
+	len = snprintf(offs, sizeof(offs), "%lld", offsLogfile);
+	eiWriteRec(gf, "END:", 4, offs, len);
+	close(gf->fd);
+	DBGPRINTF("encryption info file %s: closed\n", gf->eiName);
+}
+
+static rsRetVal
+gcryfileConstruct(gcryctx ctx, gcryfile *pgf, uchar *logfn)
+{
+	char fn[MAXFNAME+1];
+	gcryfile gf;
+	DEFiRet;
+
+	CHKmalloc(gf = calloc(1, sizeof(struct gcryfile_s)));
+	gf->ctx = ctx;
+	snprintf(fn, sizeof(fn), "%s.encinfo", logfn);
+	fn[MAXFNAME] = '\0'; /* be on save side */
+	gf->eiName = (uchar*) strdup(fn);
+	*pgf = gf;
+finalize_it:
+	RETiRet;
+}
+
 gcryctx
 gcryCtxNew(void)
 {
@@ -47,12 +208,14 @@ gcryCtxNew(void)
 }
 
 int
-gcryfileDestruct(gcryfile gf)
+gcryfileDestruct(gcryfile gf, off64_t offsLogfile)
 {
 	int r = 0;
 	if(gf == NULL)
 		goto done;
 
+	eiClose(gf, offsLogfile);
+	free(gf->eiName);
 	free(gf);
 done:	return r;
 }
@@ -119,14 +282,40 @@ rsgcrySetKey(gcryctx ctx, unsigned char *key, uint16_t keyLen)
 done:	return r;
 }
 
+/* As of some Linux and security expert I spoke to, /dev/urandom
+ * provides very strong random numbers, even if it runs out of
+ * entropy. As far as he knew, this is save for all applications
+ * (and he had good proof that I currently am not permitted to
+ * reproduce). -- rgerhards, 2013-03-04
+ */
+void
+seedIV(gcryfile gf, uchar **iv)
+{
+	int fd;
+
+	*iv = malloc(gf->blkLength); /* do NOT zero-out! */
+	/* if we cannot obtain data from /dev/urandom, we use whatever
+	 * is present at the current memory location as random data. Of
+	 * course, this is very weak and we should consider a different
+	 * option, especially when not running under Linux (for Linux,
+	 * unavailability of /dev/urandom is just a theoretic thing, it
+	 * will always work...).  -- TODO -- rgerhards, 2013-03-06
+	 */
+	if((fd = open("/dev/urandom", O_RDONLY)) > 0) {
+		if(read(fd, *iv, gf->blkLength)) {}; /* keep compiler happy */
+		close(fd);
+	}
+}
+
 rsRetVal
-rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, int gcry_mode, char *iniVector)
+rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, int gcry_mode, uchar *fname)
 {
 	gcry_error_t gcryError;
 	gcryfile gf = NULL;
+	uchar *iv = NULL;
 	DEFiRet;
 
-	CHKmalloc(gf = gcryfileConstruct(ctx));
+	CHKiRet(gcryfileConstruct(ctx, &gf, fname));
 
 	gf->blkLength = gcry_cipher_get_algo_blklen(GCRY_CIPHER);
 
@@ -150,17 +339,21 @@ rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, int gcry_mode, char *iniVector)
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
-	gcryError = gcry_cipher_setiv(gf->chd, iniVector, gf->blkLength);
+	seedIV(gf, &iv);
+	gcryError = gcry_cipher_setiv(gf->chd, iv, gf->blkLength);
 	if (gcryError) {
 		dbgprintf("gcry_cipher_setiv failed:  %s/%s\n",
 			gcry_strsource(gcryError),
 			gcry_strerror(gcryError));
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
+	CHKiRet(eiOpenAppend(gf));
+	CHKiRet(eiWriteIV(gf, iv));
 	*pgf = gf;
 finalize_it:
+	free(iv);
 	if(iRet != RS_RET_OK && gf != NULL)
-		gcryfileDestruct(gf);
+		gcryfileDestruct(gf, -1);
 	RETiRet;
 }
 

runtime/libgcry.h

@@ -34,6 +34,8 @@ typedef struct gcryfile_s *gcryfile;
 struct gcryfile_s {
 	gcry_cipher_hd_t chd; /* cypher handle */
 	size_t blkLength; /* size of low-level crypto block */
+	uchar *eiName; /* name of .encinfo file */
+	int fd; /* descriptor of .encinfo file (-1 if not open) */
 	gcryctx ctx;
 };
 
@@ -42,8 +44,14 @@ void rsgcryExit(void);
 int rsgcrySetKey(gcryctx ctx, unsigned char *key, uint16_t keyLen);
 gcryctx gcryCtxNew(void);
 void rsgcryCtxDel(gcryctx ctx);
-int gcryfileDestruct(gcryfile gf);
+int gcryfileDestruct(gcryfile gf, off64_t offsLogfile);
-rsRetVal rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, int gcry_mode, char * iniVector);
+rsRetVal rsgcryInitCrypt(gcryctx ctx, gcryfile *pgf, int gcry_mode, uchar *fname);
 int rsgcryEncrypt(gcryfile pF, uchar *buf, size_t *len);
 
+/* error states */
+#define RSGCRYE_EI_OPEN 1 	/* error opening .encinfo file */
+#define RSGCRYE_OOM 4	/* ran out of memory */
+
+#define RSGCRY_FILETYPE_NAME "rsyslog-enrcyption-info"
+
 #endif  /* #ifndef INCLUDED_LIBGCRY_H */

runtime/lmcry_gcry.c

@@ -150,11 +150,13 @@ OnFileOpen(void *pT, uchar *fn, void *pGF)
 	gcryfile *pgf = (gcryfile*) pGF;
 	DEFiRet;
 dbgprintf("DDDD: cry: onFileOpen: %s\n", fn);
-	/* note: if *pgf is set to NULL, this auto-disables GT functions */
-	//*pgf = gcryCtxOpenFile(pThis->ctx, fn);
 
-	CHKiRet(rsgcryInitCrypt(pThis->ctx, pgf, GCRY_CIPHER_MODE_CBC, "TODO: init value"));
+	CHKiRet(rsgcryInitCrypt(pThis->ctx, pgf, GCRY_CIPHER_MODE_CBC, fn));
 finalize_it:
+	/* TODO: enable this error message (need to cleanup loop first ;))
+	errmsg.LogError(0, iRet, "Encryption Provider"
+		"Error: cannot open .encinfo file - disabling log file");
+	*/
 	RETiRet;
 }
 
@@ -169,11 +171,11 @@ dbgprintf("DDDD: Encrypt (%u): %s\n", *lenRec-1, rec);
 }
 
 static rsRetVal
-OnFileClose(void *pF)
+OnFileClose(void *pF, off64_t offsLogfile)
 {
 	DEFiRet;
 dbgprintf("DDDD: onFileClose\n");
-	gcryfileDestruct(pF);
+	gcryfileDestruct(pF, offsLogfile);
 
 	RETiRet;
 }

runtime/rsyslog.h

@@ -404,6 +404,10 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_REPLCHAR_IGNORED = -2313,/**< mmanon: replacementChar parameter is ignored */
 	RS_RET_SIGPROV_ERR = -2320,/**< error in signature provider */
 	RS_RET_CRYPROV_ERR = -2321,/**< error in cryptography encryption provider */
+	RS_RET_EI_OPN_ERR = -2322,/**< error opening an .encinfo file */
+	RS_RET_EI_NO_EXISTS = -2323,/**< .encinfo file does not exist (status, not necessarily error!)*/
+	RS_RET_EI_WR_ERR = -2324,/**< error writing an .encinfo file */
+	RS_RET_EI_INVLD_FILE = -2325,/**< header indicates the file is no .encinfo file */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */

runtime/stream.c

@@ -45,6 +45,7 @@
 #include <pthread.h>
 #include <fcntl.h>
 #include <unistd.h>
+#include <sys/types.h>
 #include <sys/stat.h>	 /* required for HP UX */
 #include <errno.h>
 #include <pthread.h>
@@ -388,6 +389,7 @@ strmWaitAsyncWriterDone(strm_t *pThis)
  */
 static rsRetVal strmCloseFile(strm_t *pThis)
 {
+	off64_t currOffs;
 	DEFiRet;
 
 	ASSERT(pThis != NULL);
@@ -408,11 +410,12 @@ static rsRetVal strmCloseFile(strm_t *pThis)
 	 * against this. -- rgerhards, 2010-03-19
 	 */
 	if(pThis->fd != -1) {
+		currOffs = lseek64(pThis->fd, 0, SEEK_CUR);
 		close(pThis->fd);
 		pThis->fd = -1;
 		pThis->inode = 0;
 		if(pThis->cryprov != NULL) {
-			pThis->cryprov->OnFileClose(pThis->cryprovFileData);
+			pThis->cryprov->OnFileClose(pThis->cryprovFileData, currOffs);
 			pThis->cryprovFileData = NULL;
 		}
 	}