From 66bcd91196b51422656548f6b14efd0d05d73e02 Mon Sep 17 00:00:00 2001 From: alakatos Date: Wed, 15 Dec 2021 09:50:45 +0100 Subject: [PATCH 001/134] Clarify meaning of loadConf and RunConf --- action.c | 14 ++++----- grammar/rainerscript.c | 2 +- outchannel.c | 6 ++-- outchannel.h | 2 +- runtime/dynstats.c | 14 ++++----- runtime/lookup.c | 6 ++-- runtime/parser.c | 8 ++--- runtime/perctile_stats.c | 4 +-- runtime/rsconf.c | 64 +++++++++++++++++++++------------------- runtime/ruleset.c | 6 ++-- template.c | 2 +- tools/rsyslogd.c | 8 ++--- 12 files changed, 67 insertions(+), 69 deletions(-) diff --git a/action.c b/action.c index 17a74a74c7..c1b48de388 100644 --- a/action.c +++ b/action.c @@ -393,7 +393,7 @@ rsRetVal actionConstruct(action_t **ppThis) action_t *pThis; assert(ppThis != NULL); - + CHKmalloc(pThis = (action_t*) calloc(1, sizeof(action_t))); pThis->iResumeInterval = 30; pThis->iResumeIntervalMax = 1800; /* max interval default is half an hour */ @@ -584,7 +584,7 @@ actionConstructFinalize(action_t *__restrict__ const pThis, struct nvlst *lst) "that they will have no effect - " "see https://www.rsyslog.com/mm-no-queue/", (char*)modGetName(pThis->pMod)); } - + /* and now reset the queue params (see comment in its function header!) */ actionResetQueueParams(); @@ -1979,7 +1979,7 @@ rsRetVal activateActions(void) { DEFiRet; - iRet = ruleset.IterateAllActions(ourConf, doActivateActions, NULL); + iRet = ruleset.IterateAllActions(runConf, doActivateActions, NULL); RETiRet; } @@ -2038,7 +2038,7 @@ static rsRetVal actionApplyCnfParam(action_t * const pAction, struct cnfparamvals * const pvals) { int i; - + for(i = 0 ; i < pblk.nParams ; ++i) { if(!pvals[i].bUsed) continue; @@ -2139,7 +2139,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, CHKmalloc(pAction->peParamPassing = (paramPassing_t*)calloc(pAction->iNumTpls, sizeof(paramPassing_t))); } - + pAction->bUsesMsgPassingMode = 0; pAction->bNeedReleaseBatch = 0; for(i = 0 ; i < pAction->iNumTpls ; ++i) { @@ -2149,7 +2149,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, */ if(!(iTplOpts & OMSR_TPL_AS_MSG)) { if((pAction->ppTpl[i] = - tplFind(ourConf, (char*)pTplName, strlen((char*)pTplName))) == NULL) { + tplFind(loadConf, (char*)pTplName, strlen((char*)pTplName))) == NULL) { snprintf(errMsg, sizeof(errMsg), " Could not find template %d '%s' - action disabled", i, pTplName); @@ -2188,7 +2188,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, pAction->pModData = pModData; CHKiRet(actionConstructFinalize(pAction, lst)); - + *ppAction = pAction; /* finally store the action pointer */ finalize_it: diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 0ec18a6d7e..67c159bb1d 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -3738,7 +3738,7 @@ initFunc_exec_template(struct cnffunc *func) } tplName = es_str2cstr(((struct cnfstringval*) func->expr[0])->estr, NULL); - func->funcdata = tplFind(ourConf, tplName, strlen(tplName)); + func->funcdata = tplFind(loadConf, tplName, strlen(tplName)); if(func->funcdata == NULL) { parser_errmsg("exec_template(): template '%s' could not be found", tplName); FINALIZE; diff --git a/outchannel.c b/outchannel.c index ba5809f4af..2f456b53b0 100644 --- a/outchannel.c +++ b/outchannel.c @@ -265,7 +265,7 @@ void ochDeleteAll(void) { struct outchannel *pOch, *pOchDel; - pOch = loadConf->och.ochRoot; + pOch = runConf->och.ochRoot; while(pOch != NULL) { dbgprintf("Delete Outchannel: Name='%s'\n ", pOch->pszName == NULL? "NULL" : pOch->pszName); pOchDel = pOch; @@ -284,11 +284,11 @@ void ochDeleteAll(void) /* Print the outchannel structure. This is more or less a * debug or test aid, but anyhow I think it's worth it... */ -void ochPrintList(void) +void ochPrintList(rsconf_t *cnf) { struct outchannel *pOch; - pOch = loadConf->och.ochRoot; + pOch = cnf->och.ochRoot; while(pOch != NULL) { dbgprintf("Outchannel: Name='%s'\n", pOch->pszName == NULL? "NULL" : pOch->pszName); dbgprintf("\tFile Template: '%s'\n", pOch->pszFileTemplate == NULL ? "NULL" : diff --git a/outchannel.h b/outchannel.h index a8fe863402..23699f7bfd 100644 --- a/outchannel.h +++ b/outchannel.h @@ -34,5 +34,5 @@ struct outchannel* ochConstruct(void); struct outchannel *ochAddLine(char* pName, unsigned char** pRestOfConfLine); struct outchannel *ochFind(char *pName, int iLenName); void ochDeleteAll(void); -void ochPrintList(void); +void ochPrintList(rsconf_t *cnf); #endif /* #ifdef OUTCHANNEL_H */ diff --git a/runtime/dynstats.c b/runtime/dynstats.c index 75b947b523..1bdba57df2 100644 --- a/runtime/dynstats.c +++ b/runtime/dynstats.c @@ -97,11 +97,7 @@ dynstats_destroyCounters(dynstats_bucket_t *b) { } static void -dynstats_destroyBucket(dynstats_bucket_t* b) { - dynstats_buckets_t *bkts; - - bkts = &loadConf->dynstats_buckets; - +dynstats_destroyBucket(dynstats_buckets_t *bkts, dynstats_bucket_t* b) { pthread_rwlock_wrlock(&b->lock); dynstats_destroyCounters(b); dynstats_destroyCountersIn(b, b->survivor_table, b->survivor_ctrs); @@ -282,7 +278,7 @@ dynstats_resetIfExpired(dynstats_bucket_t *b) { static void dynstats_readCallback(statsobj_t __attribute__((unused)) *ignore, void *b) { dynstats_buckets_t *bkts; - bkts = &loadConf->dynstats_buckets; + bkts = &runConf->dynstats_buckets; pthread_rwlock_rdlock(&bkts->lock); dynstats_resetIfExpired((dynstats_bucket_t *) b); @@ -362,7 +358,7 @@ dynstats_newBucket(const uchar* name, uint8_t resettable, uint32_t maxCardinalit pthread_rwlock_destroy(&b->lock); } if (b != NULL) { - dynstats_destroyBucket(b); + dynstats_destroyBucket(bkts, b); } } RETiRet; @@ -436,7 +432,7 @@ void dynstats_destroyAllBuckets(void) { dynstats_buckets_t *bkts; dynstats_bucket_t *b; - bkts = &loadConf->dynstats_buckets; + bkts = &runConf->dynstats_buckets; if (bkts->initialized) { pthread_rwlock_wrlock(&bkts->lock); while(1) { @@ -445,7 +441,7 @@ dynstats_destroyAllBuckets(void) { break; } else { bkts->list = b->next; - dynstats_destroyBucket(b); + dynstats_destroyBucket(bkts, b); } } statsobj.Destruct(&bkts->global_stats); diff --git a/runtime/lookup.c b/runtime/lookup.c index 352e7360fb..dba7d63e5a 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -229,7 +229,7 @@ void lookupDestroyCnf(void) { lookup_ref_t *luref, *luref_next; - for(luref = loadConf->lu_tabs.root ; luref != NULL ; ) { + for(luref = runConf->lu_tabs.root ; luref != NULL ; ) { luref_next = luref->next; lookupRefDestruct(luref); luref = luref_next; @@ -861,7 +861,7 @@ void lookupDoHUP(void) { lookup_ref_t *luref; - for(luref = loadConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { + for(luref = runConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { if (luref->reload_on_hup) { lookupReload(luref, NULL); } @@ -873,7 +873,7 @@ lookupPendingReloadCount(void) { uint pending_reload_count = 0; lookup_ref_t *luref; - for(luref = loadConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { + for(luref = runConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { if (lookupIsReloadPending(luref)) { pending_reload_count++; } diff --git a/runtime/parser.c b/runtime/parser.c index 0dc90533ab..8fcddfb475 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -152,7 +152,7 @@ FindParser(parser_t **ppParser, uchar *pName) { parserList_t *pThis; DEFiRet; - + for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { if(ustrcmp(pThis->pParser->pName, pName) == 0) { *ppParser = pThis->pParser; @@ -183,7 +183,7 @@ AddDfltParser(uchar *pName) CHKiRet(FindParser(&pParser, pName)); CHKiRet(AddParserToList(&pDfltParsLst, pParser)); DBGPRINTF("Parser '%s' added to default parser set.\n", pName); - + finalize_it: RETiRet; } @@ -316,7 +316,7 @@ static rsRetVal uncompressMessage(smsg_t *pMsg) uLongf iLenDefBuf; uchar *pszMsg; size_t lenMsg; - + assert(pMsg != NULL); pszMsg = pMsg->pszRawMsg; lenMsg = pMsg->iLenRawMsg; @@ -642,7 +642,7 @@ ParseMsg(smsg_t *pMsg) * will cause it to happen. After that, access to the unsanitized message is no * loger possible. */ - pParserList = ruleset.GetParserList(ourConf, pMsg); + pParserList = ruleset.GetParserList(runConf, pMsg); if(pParserList == NULL) { pParserList = pDfltParsLst; } diff --git a/runtime/perctile_stats.c b/runtime/perctile_stats.c index 88eaad4bbc..55a62aa232 100644 --- a/runtime/perctile_stats.c +++ b/runtime/perctile_stats.c @@ -150,7 +150,7 @@ static void perctileBucketDestruct(perctile_bucket_t *bkt) { } void perctileBucketsDestruct(void) { - perctile_buckets_t *bkts = &loadConf->perctile_buckets; + perctile_buckets_t *bkts = &runConf->perctile_buckets; if (bkts->initialized) { perctile_bucket_t *head = bkts->listBuckets; @@ -441,7 +441,7 @@ static rsRetVal report_perctile_stats(perctile_bucket_t* pbkt) { static void perctile_readCallback(statsobj_t __attribute__((unused)) *ignore, void __attribute__((unused)) *b) { - perctile_buckets_t *bkts = &loadConf->perctile_buckets; + perctile_buckets_t *bkts = &runConf->perctile_buckets; pthread_rwlock_rdlock(&bkts->lock); for (perctile_bucket_t *pbkt = bkts->listBuckets; pbkt != NULL; pbkt = pbkt->next) { diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 6eb7042526..42195e2c5c 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -298,7 +298,7 @@ BEGINobjDebugPrint(rsconf) /* be sure to specify the object type also in END and setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); - ochPrintList(); + ochPrintList(pThis); dbgprintf("Modules used in this configuration:\n"); for(modNode = pThis->modules.root ; modNode != NULL ; modNode = modNode->next) { dbgprintf(" %s\n", module.GetName(modNode->pMod)); @@ -549,13 +549,13 @@ void cnfDoBSDHost(char *ln) * if something goes wrong, the function never returns */ static -rsRetVal doDropPrivGid(void) +rsRetVal doDropPrivGid(rsconf_t *cnf) { int res; uchar szBuf[1024]; DEFiRet; - if(!ourConf->globals.gidDropPrivKeepSupplemental) { + if(!cnf->globals.gidDropPrivKeepSupplemental) { res = setgroups(0, NULL); /* remove all supplemental group IDs */ if(res) { LogError(errno, RS_RET_ERR_DROP_PRIV, @@ -564,15 +564,15 @@ rsRetVal doDropPrivGid(void) } DBGPRINTF("setgroups(0, NULL): %d\n", res); } - res = setgid(ourConf->globals.gidDropPriv); + res = setgid(cnf->globals.gidDropPriv); if(res) { LogError(errno, RS_RET_ERR_DROP_PRIV, - "could not set requested group id %d", ourConf->globals.gidDropPriv); + "could not set requested group id %d", cnf->globals.gidDropPriv); ABORT_FINALIZE(RS_RET_ERR_DROP_PRIV); } - DBGPRINTF("setgid(%d): %d\n", ourConf->globals.gidDropPriv, res); + DBGPRINTF("setgid(%d): %d\n", cnf->globals.gidDropPriv, res); snprintf((char*)szBuf, sizeof(szBuf), "rsyslogd's groupid changed to %d", - ourConf->globals.gidDropPriv); + cnf->globals.gidDropPriv); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); finalize_it: RETiRet; @@ -584,7 +584,7 @@ rsRetVal doDropPrivGid(void) * Note that such an abort can cause damage to on-disk structures, so we should * re-design the "interface" in the long term. -- rgerhards, 2008-11-19 */ -static void doDropPrivUid(const int iUid) +static void doDropPrivUid(rsconf_t *cnf) { int res; uchar szBuf[1024]; @@ -594,23 +594,24 @@ static void doDropPrivUid(const int iUid) /* Try to set appropriate supplementary groups for this user. * Failure is not fatal. */ - pw = getpwuid(iUid); + pw = getpwuid(cnf->globals.uidDropPriv); if (pw) { gid = getgid(); res = initgroups(pw->pw_name, gid); DBGPRINTF("initgroups(%s, %ld): %d\n", pw->pw_name, (long) gid, res); } else { - LogError(errno, NO_ERRCODE, "could not get username for userid '%d'", iUid); + LogError(errno, NO_ERRCODE, "could not get username for userid '%d'", + cnf->globals.uidDropPriv); } - res = setuid(iUid); + res = setuid(cnf->globals.uidDropPriv); if(res) { /* if we can not set the userid, this is fatal, so let's unconditionally abort */ perror("could not set requested userid"); exit(1); } - DBGPRINTF("setuid(%d): %d\n", iUid, res); - snprintf((char*)szBuf, sizeof(szBuf), "rsyslogd's userid changed to %d", iUid); + DBGPRINTF("setuid(%d): %d\n", cnf->globals.uidDropPriv, res); + snprintf((char*)szBuf, sizeof(szBuf), "rsyslogd's userid changed to %d", cnf->globals.uidDropPriv); logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); } @@ -626,15 +627,15 @@ dropPrivileges(rsconf_t *cnf) DEFiRet; if(cnf->globals.gidDropPriv != 0) { - CHKiRet(doDropPrivGid()); + CHKiRet(doDropPrivGid(cnf)); DBGPRINTF("group privileges have been dropped to gid %u\n", (unsigned) - ourConf->globals.gidDropPriv); + cnf->globals.gidDropPriv); } if(cnf->globals.uidDropPriv != 0) { - doDropPrivUid(ourConf->globals.uidDropPriv); + doDropPrivUid(cnf); DBGPRINTF("user privileges have been dropped to uid %u\n", (unsigned) - ourConf->globals.uidDropPriv); + cnf->globals.uidDropPriv); } finalize_it: @@ -667,7 +668,7 @@ tellModulesConfigLoadDone(void) DBGPRINTF("calling endCnfLoad() for module '%s'\n", node->pMod->pszName); node->pMod->endCnfLoad(node->modCnf); } - node = module.GetNxtCnfType(runConf, node, eMOD_ANY); + node = module.GetNxtCnfType(loadConf, node, eMOD_ANY); // loadConf -> runConf } return RS_RET_OK; /* intentional: we do not care about module errors */ @@ -694,7 +695,7 @@ tellModulesCheckConfig(void) node->canActivate = 0; } } - node = module.GetNxtCnfType(runConf, node, eMOD_ANY); + node = module.GetNxtCnfType(loadConf, node, eMOD_ANY); // runConf -> loadConf } return RS_RET_OK; /* intentional: we do not care about module errors */ @@ -830,7 +831,7 @@ activateMainQueue(void) FINALIZE; } - if(ourConf->globals.mainQ.MainMsgQueType == QUEUETYPE_DIRECT) { + if(runConf->globals.mainQ.MainMsgQueType == QUEUETYPE_DIRECT) { // ourConf -> runConf PREFER_STORE_0_TO_INT(&bHaveMainQueue); } else { PREFER_STORE_1_TO_INT(&bHaveMainQueue); @@ -868,6 +869,7 @@ activate(rsconf_t *cnf) /* at this point, we "switch" over to the running conf */ runConf = cnf; + loadConf = NULL; # if 0 /* currently the DAG is not supported -- code missing! */ /* TODO: re-enable this functionality some time later! */ /* check if we need to generate a config DAG and, if so, do that */ @@ -1281,31 +1283,31 @@ initLegacyConf(void) } -/* validate the current configuration, generate error messages, do +/* validate the configuration pointed by conf, generate error messages, do * optimizations, etc, etc,... */ static rsRetVal -validateConf(void) +validateConf(rsconf_t *cnf) { DEFiRet; /* some checks */ - if(ourConf->globals.mainQ.iMainMsgQueueNumWorkers < 1) { + if(cnf->globals.mainQ.iMainMsgQueueNumWorkers < 1) { LogError(0, NO_ERRCODE, "$MainMsgQueueNumWorkers must be at least 1! Set to 1.\n"); - ourConf->globals.mainQ.iMainMsgQueueNumWorkers = 1; + cnf->globals.mainQ.iMainMsgQueueNumWorkers = 1; } - if(ourConf->globals.mainQ.MainMsgQueType == QUEUETYPE_DISK) { + if(cnf->globals.mainQ.MainMsgQueType == QUEUETYPE_DISK) { errno = 0; /* for logerror! */ if(glbl.GetWorkDir() == NULL) { LogError(0, NO_ERRCODE, "No $WorkDirectory specified - can not run main " "message queue in 'disk' mode. Using 'FixedArray' instead.\n"); - ourConf->globals.mainQ.MainMsgQueType = QUEUETYPE_FIXED_ARRAY; + cnf->globals.mainQ.MainMsgQueType = QUEUETYPE_FIXED_ARRAY; } - if(ourConf->globals.mainQ.pszMainMsgQFName == NULL) { + if(cnf->globals.mainQ.pszMainMsgQFName == NULL) { LogError(0, NO_ERRCODE, "No $MainMsgQueueFileName specified - can not run main " "message queue in 'disk' mode. Using 'FixedArray' instead.\n"); - ourConf->globals.mainQ.MainMsgQueType = QUEUETYPE_FIXED_ARRAY; + cnf->globals.mainQ.MainMsgQueType = QUEUETYPE_FIXED_ARRAY; } } RETiRet; @@ -1328,7 +1330,7 @@ load(rsconf_t **cnf, uchar *confFile) DEFiRet; CHKiRet(rsconfConstruct(&loadConf)); -ourConf = loadConf; // TODO: remove, once ourConf is gone! + ourConf = loadConf; // TODO: remove, once ourConf is gone! CHKiRet(loadBuildInModules()); CHKiRet(initLegacyConf()); @@ -1367,7 +1369,7 @@ ourConf = loadConf; // TODO: remove, once ourConf is gone! tellModulesConfigLoadDone(); tellModulesCheckConfig(); - CHKiRet(validateConf()); + CHKiRet(validateConf(loadConf)); /* we are done checking the config - now validate if we should actually run or not. * If not, terminate. -- rgerhards, 2008-07-25 @@ -1381,7 +1383,7 @@ ourConf = loadConf; // TODO: remove, once ourConf is gone! /* all OK, pass loaded conf to caller */ *cnf = loadConf; -// TODO: enable this once all config code is moved to here! loadConf = NULL; + // TODO: enable this once all config code is moved to here! loadConf = NULL; dbgprintf("rsyslog finished loading master config %p\n", loadConf); rsconfDebugPrint(loadConf); diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 42a1ee3f99..64d75f0a48 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -252,7 +252,7 @@ execCallIndirect(struct cnfstmt *const __restrict__ stmt, cnfexprEval(stmt->d.s_call_ind.expr, &result, pMsg, pWti); uchar *const rsName = (uchar*) var2CString(&result, &bMustFree); - const rsRetVal localRet = rulesetGetRuleset(loadConf, &pRuleset, rsName); + const rsRetVal localRet = rulesetGetRuleset(runConf, &pRuleset, rsName); if(localRet != RS_RET_OK) { /* in that case, we accept that a NOP will "survive" */ LogError(0, RS_RET_RULESET_NOT_FOUND, "error: CALL_INDIRECT: " @@ -656,7 +656,7 @@ processBatch(batch_t *pBatch, wti_t *pWti) for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pWti->pbShutdownImmediate) ; ++i) { pMsg = pBatch->pElem[i].pMsg; DBGPRINTF("processBATCH: next msg %d: %.128s\n", i, pMsg->pszRawMsg); - pRuleset = (pMsg->pRuleset == NULL) ? ourConf->rulesets.pDflt : pMsg->pRuleset; + pRuleset = (pMsg->pRuleset == NULL) ? runConf->rulesets.pDflt : pMsg->pRuleset; localRet = scriptExec(pRuleset->root, pMsg, pWti); /* the most important case here is that processing may be aborted * due to pbShutdownImmediate, in which case we MUST NOT flag this @@ -1050,7 +1050,7 @@ doRulesetAddParser(ruleset_t *pRuleset, uchar *pName) static rsRetVal rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) { - return doRulesetAddParser(ourConf->rulesets.pCurr, pName); + return doRulesetAddParser(loadConf->rulesets.pCurr, pName); } diff --git a/template.c b/template.c index 21d8b8dd15..b8655adad5 100644 --- a/template.c +++ b/template.c @@ -2235,7 +2235,7 @@ void tplDeleteNew(rsconf_t *conf) } } -/* Store the pointer to the last hardcoded teplate */ +/* Store the pointer to the last hardcoded template */ void tplLastStaticInit(rsconf_t *conf, struct template *tpl) { conf->templates.lastStatic = tpl; diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index f1eea071da..01e1bf4b51 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -1626,14 +1626,14 @@ initAll(int argc, char **argv) hdlr_enable(SIGCHLD, hdlr_sigchld); hdlr_enable(SIGHUP, hdlr_sighup); - if(rsconfNeedDropPriv(ourConf)) { + if(rsconfNeedDropPriv(loadConf)) { /* need to write pid file early as we may loose permissions */ CHKiRet(writePidFile()); } CHKiRet(rsconf.Activate(ourConf)); - if(ourConf->globals.bLogStatusMsgs) { + if(runConf->globals.bLogStatusMsgs) { char bufStartUpMsg[512]; snprintf(bufStartUpMsg, sizeof(bufStartUpMsg), "[origin software=\"rsyslogd\" " "swVersion=\"" VERSION \ @@ -1642,7 +1642,7 @@ initAll(int argc, char **argv) logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)bufStartUpMsg, 0); } - if(!rsconfNeedDropPriv(ourConf)) { + if(!rsconfNeedDropPriv(runConf)) { CHKiRet(writePidFile()); } @@ -1654,7 +1654,7 @@ initAll(int argc, char **argv) stddbg = -1; /* turn off writing to fd 1 */ close(1); close(2); - ourConf->globals.bErrMsgToStderr = 0; + runConf->globals.bErrMsgToStderr = 0; } finalize_it: From 96ad69b26df16bab00bae02b46fb3209f06fd37a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 18 Dec 2021 16:17:07 +0100 Subject: [PATCH 002/134] prepare for new scheduled stable release cycle --- ChangeLog | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 347a29f85a..ce26bc8998 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ ---------------------------------------------------------------------------------------- +Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +---------------------------------------------------------------------------------------- Scheduled Release 8.2112.0 (aka 2021.12) 2021-12-16 - 2021-12-14: refactor:Deallocate outchannel resources in rsconf destructor Thanks to Attila Lakatos for the patch. diff --git a/configure.ac b/configure.ac index e5ec696848..f19532fa23 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2112.0],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2202.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 21, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 12, [month part of real rsyslog version]) # UPDATE on release From a11ad1a45129b2c5634879cd3224d1219b3cea5c Mon Sep 17 00:00:00 2001 From: Mattia Barbon Date: Mon, 27 Dec 2021 23:49:56 +0100 Subject: [PATCH 003/134] add property options to support ISO week/year number --- runtime/datetime.c | 49 ++++++++++++++++++++++++++++++++++++++ runtime/datetime.h | 1 + runtime/msg.c | 29 ++++++++++++++++++++++ template.c | 4 ++++ template.h | 3 ++- tests/Makefile.am | 2 ++ tests/timestamp-isoweek.sh | 24 +++++++++++++++++++ 7 files changed, 111 insertions(+), 1 deletion(-) create mode 100755 tests/timestamp-isoweek.sh diff --git a/runtime/datetime.c b/runtime/datetime.c index cc7b3c90eb..795f556ec5 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -86,6 +86,21 @@ static const long long yearInSecs[] = { 3944678399, 3976214399, 4007836799, 4039372799, 4070908799, 4102444799}; +/* note ramge is 1969 -> 2100 because it needs to access previous/next year */ +/* for x in $(seq 1969 2100) ; do + * printf %s', ' $(date --date="Dec 28 ${x} UTC 12:00:00" +%V) + * done | fold -w 70 -s */ +static const int weeksInYear[] = { + 52, 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, + 52, 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, + 52, 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, + 53, 52, 52, 52, 52, 53, 52, 52, 52, 52, 52, 53, 52, +}; + static const char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; @@ -1265,6 +1280,40 @@ int getWeek(struct syslogTime *ts) return weekNum; } +/* getISOWeek - 1-53 week of the year */ +int getISOWeek(struct syslogTime *ts, int *year) +{ + int weekNum; + int curDow; + int curYearDay; + + /* get current day in year, current day of week + * and the day of week of 1/1 */ + curYearDay = getOrdinal(ts); + curDow = getWeekdayNbr(ts); + + /* map from 0 - Sunday, 1, Monday to 1, Monday, 7 - Sunday */ + if (curDow == 0) { + curDow = 7; + } + /* make ordinal in range 1-366 */ + curYearDay++; + + weekNum = (10 + curYearDay - curDow) / 7; + *year = ts->year; + if (weekNum == 0) { + /* this is actually W52 or W53 of previous year */ + weekNum = weeksInYear[ts->year - 1 - 1969]; + *year = ts->year - 1; + } else if (weekNum > weeksInYear[ts->year - 1969]) { + /* this is actually W01 of next year */ + weekNum = 1; + *year = ts->year + 1; + } + + return weekNum; +} + void timeConvertToUTC(const struct syslogTime *const __restrict__ local, struct syslogTime *const __restrict__ utc) diff --git a/runtime/datetime.h b/runtime/datetime.h index 81731e2a54..ee13e2b5df 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -92,6 +92,7 @@ void applyDfltTZ(struct syslogTime *pTime, char *tz); int getWeekdayNbr(struct syslogTime *ts); int getOrdinal(struct syslogTime *ts); int getWeek(struct syslogTime *ts); +int getISOWeek(struct syslogTime *ts, int *year); void timeConvertToUTC(const struct syslogTime *const __restrict__ local, struct syslogTime *const __restrict__ utc); time_t getTime(time_t *ttSeconds); dateTimeFormat_t getDateTimeFormatFromStr(const char * const __restrict__ s); diff --git a/runtime/msg.c b/runtime/msg.c index ee0d757352..b0b355ee83 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1784,6 +1784,25 @@ getPRI(smsg_t * const pM) } +static const char * +formatISOWeekOrYear(enum tplFormatTypes eFmt, struct syslogTime *pTm) +{ + if(pTm->year >= 1970 && pTm->year <= 2099) { + int isoWeekYear; + int isoWeek; + + isoWeek = getISOWeek(pTm, &isoWeekYear); + + if (eFmt == tplFmtISOWeek) { + return two_digits[isoWeek]; + } else { + return years[isoWeekYear - 1967]; + } + } else { + return "YEAR OUT OF RANGE(1970-2099)"; + } +} + const char * getTimeReported(smsg_t * const pM, enum tplFormatTypes eFmt) { @@ -1878,6 +1897,9 @@ getTimeReported(smsg_t * const pM, enum tplFormatTypes eFmt) return daysInYear[getOrdinal(&pM->tTIMESTAMP)]; case tplFmtWeek: return two_digits[getWeek(&pM->tTIMESTAMP)]; + case tplFmtISOWeek: + case tplFmtISOWeekYear: + return formatISOWeekOrYear(eFmt, &pM->tTIMESTAMP); } return "INVALID eFmt OPTION!"; } @@ -1973,6 +1995,10 @@ static const char *getTimeUTC(struct syslogTime *const __restrict__ pTmIn, case tplFmtWeek: retbuf = strdup(two_digits[getWeek(pTm)]); break; + case tplFmtISOWeek: + case tplFmtISOWeekYear: + retbuf = strdup(formatISOWeekOrYear(eFmt, pTm)); + break; } if(retbuf == NULL) { @@ -2096,6 +2122,9 @@ getTimeGenerated(smsg_t *const __restrict__ pM, return daysInYear[getOrdinal(pTm)]; case tplFmtWeek: return two_digits[getWeek(pTm)]; + case tplFmtISOWeek: + case tplFmtISOWeekYear: + return formatISOWeekOrYear(eFmt, pTm); } return "INVALID eFmt OPTION!"; } diff --git a/template.c b/template.c index b8655adad5..93e2880edb 100644 --- a/template.c +++ b/template.c @@ -724,6 +724,10 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) pTpe->data.field.eDateFormat = tplFmtOrdinal; } else if (!strcmp((char*)Buf, "date-week")) { pTpe->data.field.eDateFormat = tplFmtWeek; + } else if (!strcmp((char*)Buf, "date-iso-week")) { + pTpe->data.field.eDateFormat = tplFmtISOWeek; + } else if (!strcmp((char*)Buf, "date-iso-week-year")) { + pTpe->data.field.eDateFormat = tplFmtISOWeekYear; } else if(!strcmp((char*)Buf, "date-utc")) { pTpe->data.field.options.bDateInUTC = 1; } else if(!strcmp((char*)Buf, "lowercase")) { diff --git a/template.h b/template.h index b387b16f53..9afbaeea66 100644 --- a/template.h +++ b/template.h @@ -65,7 +65,8 @@ enum tplFormatTypes { tplFmtDefault = 0, tplFmtMySQLDate = 1, tplFmtWDayName = 8, tplFmtYear = 9, tplFmtMonth = 10, tplFmtDay = 11, tplFmtHour = 12, tplFmtMinute = 13, tplFmtSecond = 14, tplFmtTZOffsHour = 15, tplFmtTZOffsMin = 16, tplFmtTZOffsDirection = 17, - tplFmtWDay = 18, tplFmtOrdinal = 19, tplFmtWeek = 20}; + tplFmtWDay = 18, tplFmtOrdinal = 19, tplFmtWeek = 20, + tplFmtISOWeek = 21, tplFmtISOWeekYear = 22}; enum tplFormatCaseConvTypes { tplCaseConvNo = 0, tplCaseConvUpper = 1, tplCaseConvLower = 2 }; enum tplRegexType { TPL_REGEX_BRE = 0, /* posix BRE */ TPL_REGEX_ERE = 1 /* posix ERE */ diff --git a/tests/Makefile.am b/tests/Makefile.am index c995304739..2220e300df 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -172,6 +172,7 @@ TESTS += \ empty-hostname.sh \ timestamp-3164.sh \ timestamp-3339.sh \ + timestamp-isoweek.sh \ timestamp-mysql.sh \ timestamp-pgsql.sh \ timestamp-subseconds.sh \ @@ -1800,6 +1801,7 @@ EXTRA_DIST= \ proprepltest-rfctag.sh \ timestamp-3164.sh \ timestamp-3339.sh \ + timestamp-isoweek.sh \ timestamp-mysql.sh \ timestamp-pgsql.sh \ timestamp-subseconds.sh \ diff --git a/tests/timestamp-isoweek.sh b/tests/timestamp-isoweek.sh new file mode 100755 index 0000000000..43c8fd9322 --- /dev/null +++ b/tests/timestamp-isoweek.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# add 2021-12-27 by Mattia Barbon, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +template(name="outfmt" type="string" string="%timestamp:::date-iso-week-year%/%timestamp:::date-iso-week%\n") + +:syslogtag, contains, "su" action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` + template="outfmt") + + +' +startup +injectmsg_literal "<34>1 1971-01-01T12:34:56.003Z mymachine.example.com su - ID47 - MSG" +injectmsg_literal "<34>1 2021-12-02T12:34:56.123456Z mymachine.example.com su - ID47 - MSG" +injectmsg_literal "<34>1 2099-12-31T12:34:56Z mymachine.example.com su - ID47 - MSG" +shutdown_when_empty +wait_shutdown + +export EXPECTED='1970/53 +2021/48 +2099/53' +cmp_exact +exit_test From d0b88c748c041f2bb8648930c7a9625d98ab53f7 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Wed, 5 Jan 2022 21:58:57 +0100 Subject: [PATCH 004/134] omhttp: Fix memory leak in lokirest batchmode A JSON object was created (valueObj) but not used and also not released causing a memory leak. This was properly caused by the initial copy&paste from serializeBatchKafkaRest. - Also added a valgrind test for lokirest batchmode Uncomment "--keep-debuginfo" in testcaseif stack traces are incomplete closes: https://github.com/rsyslog/rsyslog/issues/4766 --- contrib/omhttp/omhttp.c | 7 ------- tests/Makefile.am | 4 +++- tests/omhttp-batch-lokirest-vg.sh | 5 +++++ 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100755 tests/omhttp-batch-lokirest-vg.sh diff --git a/contrib/omhttp/omhttp.c b/contrib/omhttp/omhttp.c index 15f5338d69..a2a8111237 100644 --- a/contrib/omhttp/omhttp.c +++ b/contrib/omhttp/omhttp.c @@ -1244,7 +1244,6 @@ serializeBatchLokiRest(wrkrInstanceData_t *pWrkrData, char **batchBuf) { fjson_object *batchArray = NULL; fjson_object *recordObj = NULL; - fjson_object *valueObj = NULL; fjson_object *msgObj = NULL; size_t numMessages = pWrkrData->batch.nmemb; @@ -1260,12 +1259,6 @@ serializeBatchLokiRest(wrkrInstanceData_t *pWrkrData, char **batchBuf) } for (size_t i = 0; i < numMessages; i++) { - valueObj = fjson_object_new_object(); - if (valueObj == NULL) { - fjson_object_put(batchArray); // cleanup - LogError(0, RS_RET_ERR, "omhttp: serializeBatchLokiRest failed to create value object"); - ABORT_FINALIZE(RS_RET_ERR); - } DBGPRINTF("omhttp: serializeBatchLokiRest parsing message [%s]\n",(char *) pWrkrData->batch.data[i]); msgObj = fjson_tokener_parse((char *) pWrkrData->batch.data[i]); if (msgObj == NULL) { diff --git a/tests/Makefile.am b/tests/Makefile.am index c995304739..3caf70b6f0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -895,7 +895,8 @@ TESTS += \ omhttp-batch-jsonarray-vg.sh \ omhttp-batch-kafkarest-retry-vg.sh \ omhttp-batch-lokirest-retry-vg.sh \ - omhttp-retry-vg.sh + omhttp-retry-vg.sh \ + omhttp-batch-lokirest-vg.sh endif endif @@ -2367,6 +2368,7 @@ EXTRA_DIST= \ omhttp-batch-kafkarest.sh \ omhttp-batch-lokirest-retry.sh \ omhttp-batch-lokirest.sh \ + omhttp-batch-lokirest-vg.sh \ omhttp-batch-newline.sh \ omhttp-retry.sh \ omhttp-httpheaderkey.sh \ diff --git a/tests/omhttp-batch-lokirest-vg.sh b/tests/omhttp-batch-lokirest-vg.sh new file mode 100755 index 0000000000..e1eeca8bfc --- /dev/null +++ b/tests/omhttp-batch-lokirest-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +# export RS_TEST_VALGRIND_EXTRA_OPTS="--keep-debuginfo=yes --leak-check=full" + +source ${srcdir:=.}/omhttp-batch-lokirest.sh From 4a072d6c93015a63716c49a6c7756df22750086a Mon Sep 17 00:00:00 2001 From: r0b0t <1168808+secretnonempty@users.noreply.github.com> Date: Wed, 5 Jan 2022 21:14:53 -0800 Subject: [PATCH 005/134] Update omlibdbi.c Fix a use-after-free bug in omlibdbi.c --- plugins/omlibdbi/omlibdbi.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 3190c053ff..60fd172ed9 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -290,15 +290,9 @@ static rsRetVal initConn(instanceData *pData, int bSilent) int is_sqlite2 = !strcmp((const char *)pData->drvrName, "sqlite"); int is_sqlite3 = !strcmp((const char *)pData->drvrName, "sqlite3"); if(is_sqlite2 || is_sqlite3) { - char *const dn_org = strdup((char*)pData->dbName); - char *const dn = dirname(dn_org); - dbi_conn_set_option(pData->conn, is_sqlite3 ? "sqlite3_dbdir" : "sqlite_dbdir",dn); - free(dn_org); /* Free original buffer - dirname may return different pointer */ - - char *tmp = strdup((char*)pData->dbName); - char *bn = basename(tmp); - free(tmp); - dbi_conn_set_option(pData->conn, "dbname", bn); + dbi_conn_set_option(pData->conn, is_sqlite3 ? "sqlite3_dbdir" : "sqlite_dbdir", + dirname((char *)pData->dbName)); + dbi_conn_set_option(pData->conn, "dbname", basename((char *)pData->dbName )); } else { dbi_conn_set_option(pData->conn, "dbname", (char*) pData->dbName); } From b8d2c321f9de0bbf8617021f7d7f1ec341ae9f6b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 6 Jan 2022 18:44:47 +0100 Subject: [PATCH 006/134] maintain ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index ce26bc8998..2d697a2e90 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-01-06: omhttp bugfix: memory leak in lokirest batchmode + A JSON object was created (valueObj) but not used and also not released causing a + memory leak. Over time, this could lead to memory overcomittent. + closes: https://github.com/rsyslog/rsyslog/issues/4766 ---------------------------------------------------------------------------------------- Scheduled Release 8.2112.0 (aka 2021.12) 2021-12-16 - 2021-12-14: refactor:Deallocate outchannel resources in rsconf destructor From d384cb61b5ec55a017e48e822260be7b1bd63209 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 6 Jan 2022 18:46:58 +0100 Subject: [PATCH 007/134] CI: remove unnecessary dependency Especially as this dependency currently causes the check run to fail. No point in trying to fix it, as it is not needed. So removing is the most robust option. --- .github/workflows/run_journal.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run_journal.yml b/.github/workflows/run_journal.yml index 2c9597dd68..ba7aec7626 100644 --- a/.github/workflows/run_journal.yml +++ b/.github/workflows/run_journal.yml @@ -56,7 +56,6 @@ jobs: liblognorm-dev \ liblz4-dev \ libmongoc-dev \ - libmysqlclient-dev \ libnet1-dev \ libpcap-dev \ librelp-dev \ @@ -69,7 +68,6 @@ jobs: logrotate \ lsof \ make \ - mysql-server \ net-tools \ pkg-config \ postgresql-client libpq-dev \ From 3dde61a06241d86a22b37b547f67c544ca4c8492 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 7 Jan 2022 08:59:04 +0100 Subject: [PATCH 008/134] CI: cleanup journal test environment remove unnecessary components - gives slightly better runtime --- .github/workflows/run_journal.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/run_journal.yml b/.github/workflows/run_journal.yml index ba7aec7626..0c17df139f 100644 --- a/.github/workflows/run_journal.yml +++ b/.github/workflows/run_journal.yml @@ -33,9 +33,6 @@ jobs: timeout-minutes: 30 steps: - - name: git checkout project - uses: actions/checkout@v1 - - name: add extra dependencies run: | echo 'deb http://download.opensuse.org/repositories/home:/rgerhards/xUbuntu_20.04/ /' \ @@ -44,43 +41,29 @@ jobs: | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/home:rgerhards.gpg > /dev/null sudo apt-get update sudo apt-get install -y \ - libcurl4-gnutls-dev \ - libdbi-dev \ libestr-dev \ libgcrypt20-dev \ libglib2.0-dev \ libgnutls28-dev \ - libgrok1 libgrok-dev \ - libhiredis-dev \ - libkrb5-dev \ liblognorm-dev \ liblz4-dev \ - libmongoc-dev \ libnet1-dev \ - libpcap-dev \ librelp-dev \ - libsasl2-dev \ - libsnmp-dev \ libssl-dev \ libsystemd-dev \ libtool \ libtool-bin \ - logrotate \ lsof \ make \ net-tools \ pkg-config \ - postgresql-client libpq-dev \ - python3-pip \ - python3-pysnmp4 \ python-docutils \ software-properties-common \ - uuid-dev \ valgrind \ - vim \ - wget \ - librdkafka-dev \ - zlib1g-dev + wget + + - name: git checkout project + uses: actions/checkout@v1 - name: build dependencies which require that run: | From c116d2bcef6673c8f752c268ee7db83c9ae59cff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 7 Jan 2022 13:09:51 +0100 Subject: [PATCH 009/134] maintain ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2d697a2e90..d15c4b793f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-01-07: omlibdbi bugfix: use-after-free bug + This occurred in when sqllite driver was used. Depending on circumstances, this had + no visible issues (often) up to rsyslog segfault. The busier rsyslog is, the more + likely a bad outcome. - 2022-01-06: omhttp bugfix: memory leak in lokirest batchmode A JSON object was created (valueObj) but not used and also not released causing a memory leak. Over time, this could lead to memory overcomittent. From 7d2e532d3460f4d2a1071096a73ba71674bd662a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 12 Jan 2022 15:22:08 +0100 Subject: [PATCH 010/134] cleanup: remove unused variable Especially as this caused confusion when doing other work inside the code. see also https://github.com/rsyslog/rsyslog/pull/4760#discussion_r783118550 --- runtime/strms_sess.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c index eb87472ed0..95b499c6fe 100644 --- a/runtime/strms_sess.c +++ b/runtime/strms_sess.c @@ -3,7 +3,7 @@ * This implements a session of the strmsrv object. For general * comments, see header of strmsrv.c. * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2022 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -49,8 +49,6 @@ DEFobjCurrIf(prop) DEFobjCurrIf(netstrm) DEFobjCurrIf(datetime) -static int iMaxLine; /* maximum size of a single message */ - /* forward definitions */ static rsRetVal Close(strms_sess_t *pThis); @@ -288,7 +286,6 @@ BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); - iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ objRelease(glbl, CORE_COMPONENT); /* set our own handlers */ From 321fc76f0f52523e085e68f6e0320a1577088c17 Mon Sep 17 00:00:00 2001 From: alakatos Date: Wed, 12 Jan 2022 15:50:17 +0100 Subject: [PATCH 011/134] Move rsyslog global parameters to rsconf_t struct --- action.c | 6 +- action.h | 3 - contrib/imbatchreport/imbatchreport.c | 2 +- contrib/imdocker/imdocker.c | 2 +- contrib/imkmsg/imkmsg.c | 2 +- contrib/improg/improg.c | 2 +- contrib/imtuxedoulog/imtuxedoulog.c | 10 +- contrib/omfile-hardened/omfile-hardened.c | 8 +- contrib/omrabbitmq/omrabbitmq.c | 3 +- .../pmaixforwardedfrom/pmaixforwardedfrom.c | 3 +- contrib/pmcisconames/pmcisconames.c | 3 +- contrib/pmpanngfw/pmpanngfw.c | 3 +- contrib/pmsnare/pmsnare.c | 23 +- plugins/imfile/imfile.c | 4 +- plugins/imgssapi/imgssapi.c | 8 +- plugins/imjournal/imjournal.c | 23 +- plugins/imklog/imklog.c | 2 +- plugins/imptcp/imptcp.c | 8 +- plugins/imrelp/imrelp.c | 8 +- plugins/imsolaris/imsolaris.c | 3 +- plugins/imudp/imudp.c | 6 +- plugins/imuxsock/imuxsock.c | 20 +- plugins/mmexternal/mmexternal.c | 7 +- plugins/omgssapi/omgssapi.c | 11 +- plugins/omprog/omprog.c | 5 +- plugins/omrelp/omrelp.c | 7 +- plugins/omudpspoof/omudpspoof.c | 4 +- plugins/omuxsock/omuxsock.c | 4 +- plugins/pmlastmsg/pmlastmsg.c | 3 +- runtime/dnscache.c | 20 +- runtime/dnscache.h | 2 - runtime/errmsg.c | 25 +- runtime/glbl.c | 666 +++++++++--------- runtime/glbl.h | 82 +-- runtime/msg.c | 14 +- runtime/net.c | 40 +- runtime/net.h | 6 +- runtime/netstrms.c | 3 +- runtime/nsd_gtls.c | 15 +- runtime/nsd_ossl.c | 8 +- runtime/nsd_ptcp.c | 3 +- runtime/nspoll.c | 3 +- runtime/nssel.c | 5 +- runtime/operatingstate.c | 5 +- runtime/parser.c | 33 +- runtime/parser.h | 2 - runtime/queue.c | 34 +- runtime/queue.h | 11 - runtime/ratelimit.c | 2 +- runtime/rsconf.c | 84 ++- runtime/rsconf.h | 77 ++ runtime/rsyslog.c | 1 - runtime/srutils.c | 7 +- runtime/statsobj.c | 11 +- runtime/stream.c | 21 +- runtime/tcps_sess.c | 11 +- runtime/tcpsrv.c | 2 +- runtime/typedefs.h | 1 + threads.c | 7 +- tools/omfile.c | 9 +- tools/omfwd.c | 6 +- tools/pmrfc3164.c | 5 +- tools/rsyslogd.c | 85 +-- 63 files changed, 799 insertions(+), 700 deletions(-) diff --git a/action.c b/action.c index c1b48de388..90bfa53287 100644 --- a/action.c +++ b/action.c @@ -190,8 +190,6 @@ static configSettings_t cs; /* our current config settings */ * is no better name available. */ int iActionNbr = 0; -int bActionReportSuspension = 1; -int bActionReportSuspensionCont = 0; /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfparamdescr[] = { @@ -752,9 +750,9 @@ static void setSuspendMessageConfVars(action_t *__restrict__ const pThis) { if(pThis->bReportSuspension == -1) - pThis->bReportSuspension = bActionReportSuspension; + pThis->bReportSuspension = runConf->globals.bActionReportSuspension; if(pThis->bReportSuspensionCont == -1) { - pThis->bReportSuspensionCont = bActionReportSuspensionCont; + pThis->bReportSuspensionCont = runConf->globals.bActionReportSuspensionCont; if(pThis->bReportSuspensionCont == -1) pThis->bReportSuspensionCont = 1; } diff --git a/action.h b/action.h index 3f3d4c7b1f..403226bc76 100644 --- a/action.h +++ b/action.h @@ -30,9 +30,6 @@ /* external data */ extern int glbliActionResumeRetryCount; -extern int bActionReportSuspension; -extern int bActionReportSuspensionCont; - /* the following struct defines the action object data structure */ diff --git a/contrib/imbatchreport/imbatchreport.c b/contrib/imbatchreport/imbatchreport.c index ab17f0d0f7..e26375ef12 100644 --- a/contrib/imbatchreport/imbatchreport.c +++ b/contrib/imbatchreport/imbatchreport.c @@ -930,7 +930,7 @@ CODESTARTwillRun sizeof("imbatchreport") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); - fixedModConf.max_msg_size = glbl.GetMaxLine(); + fixedModConf.max_msg_size = glbl.GetMaxLine(runConf); DBGPRINTF("Max message len %zu\n", fixedModConf.max_msg_size); CHKmalloc(fixedModConf.msg_buffer = (char*)malloc(fixedModConf.max_msg_size + 1)); finalize_it: diff --git a/contrib/imdocker/imdocker.c b/contrib/imdocker/imdocker.c index f6722f64d3..d7445ea3a2 100644 --- a/contrib/imdocker/imdocker.c +++ b/contrib/imdocker/imdocker.c @@ -1059,7 +1059,7 @@ enqMsg(docker_cont_logs_inst_t *pInst, uchar *msg, size_t len, const uchar *pszT lenMsg--; } - if(glbl.GetParserDropTrailingLFOnReception() + if(glbl.GetParserDropTrailingLFOnReception(loadModConf->pConf) && lenMsg > 0 && pszMsg[lenMsg-1] == '\n') { DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); lenMsg--; diff --git a/contrib/imkmsg/imkmsg.c b/contrib/imkmsg/imkmsg.c index 24aaa1b984..8091240c39 100644 --- a/contrib/imkmsg/imkmsg.c +++ b/contrib/imkmsg/imkmsg.c @@ -155,7 +155,7 @@ rsRetVal Syslog(syslog_pri_t priority, uchar *pMsg, struct timeval *tp, struct j */ int klog_getMaxLine(void) { - return glbl.GetMaxLine(); + return glbl.GetMaxLine(runModConf->pConf); } diff --git a/contrib/improg/improg.c b/contrib/improg/improg.c index 77916759fd..d525ddd779 100644 --- a/contrib/improg/improg.c +++ b/contrib/improg/improg.c @@ -330,7 +330,7 @@ static void waitForChild(instanceConf_t *pInst) /* waitpid will fail with errno == ECHILD if the child process has already been reaped by the rsyslogd main loop (see rsyslogd.c) */ if(ret == pInst->pid) { - glblReportChildProcessExit(pInst->pszBinary, pInst->pid, status); + glblReportChildProcessExit(runConf, pInst->pszBinary, pInst->pid, status); } } diff --git a/contrib/imtuxedoulog/imtuxedoulog.c b/contrib/imtuxedoulog/imtuxedoulog.c index 98fbc7a120..ffb863c2c0 100644 --- a/contrib/imtuxedoulog/imtuxedoulog.c +++ b/contrib/imtuxedoulog/imtuxedoulog.c @@ -89,6 +89,8 @@ struct modConfData_s { }; static instanceConf_t *confRoot = NULL; +static modConfData_t *loadModConf = NULL; /* modConf ptr to use for the current load process */ +static modConfData_t *runModConf = NULL; /* modConf ptr to use for run process */ MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -160,7 +162,7 @@ static int getFullStateFileName(uchar* pszstatefile, uchar* pszout, int ilenout) const uchar* pszworkdir; /* Get Raw Workdir, if it is NULL we need to propper handle it */ - pszworkdir = glblGetWorkDirRaw(); + pszworkdir = glblGetWorkDirRaw(runModConf->pConf); /* Construct file name */ lenout = snprintf((char*)pszout, ilenout, "%s/%s", @@ -472,9 +474,9 @@ static void persistStrmState(instanceConf_t *pInst) DBGPRINTF("persisting state for '%s' to file '%s'\n", pInst->pszUlogBaseName, statefn); CHKiRet(strm.Construct(&psSF)); - lenDir = ustrlen(glbl.GetWorkDir()); + lenDir = ustrlen(glbl.GetWorkDir(runModConf->pConf)); if(lenDir > 0) - CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), lenDir)); + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(runModConf->pConf), lenDir)); CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psSF, statefn, strlen((char*) statefn))); @@ -764,6 +766,7 @@ ENDnewInpInst BEGINbeginCnfLoad CODESTARTbeginCnfLoad + loadModConf = pModConf; pModConf->pConf = pConf; ENDbeginCnfLoad @@ -781,6 +784,7 @@ ENDcheckCnf BEGINactivateCnf CODESTARTactivateCnf + runModConf = pModConf; ENDactivateCnf BEGINfreeCnf diff --git a/contrib/omfile-hardened/omfile-hardened.c b/contrib/omfile-hardened/omfile-hardened.c index dc376e2324..8a93225cb7 100644 --- a/contrib/omfile-hardened/omfile-hardened.c +++ b/contrib/omfile-hardened/omfile-hardened.c @@ -650,7 +650,7 @@ prepareFile(instanceData *__restrict__ const pData, const uchar *__restrict__ co if(pData->useSigprov) sigprovPrepare(pData, szNameBuf); - + finalize_it: if(iRet != RS_RET_OK) { if(pData->pStrm != NULL) { @@ -682,7 +682,7 @@ fsCheck(instanceData *__restrict__ const pData, const uchar *__restrict__ const /* check if we have space available for all buffers to be flushed and for * a maximum lenght message, perhaps current msg size would be enough */ if (stat.f_bsize * stat.f_bavail < - pData->iIOBufSize * pData->iDynaFileCacheSize + (uint)(glbl.GetMaxLine())) + pData->iIOBufSize * pData->iDynaFileCacheSize + (uint)(glbl.GetMaxLine(runModConf->pConf))) { iRet = RS_RET_FS_ERR; LogError(0, iRet, "too few available blocks in %s", path); @@ -1002,7 +1002,7 @@ janitorChkDynaFiles(instanceData *__restrict__ const pData) pData->iCurrElt = -1; /* no longer available! */ } } else { - pCache[i]->nInactive += janitorInterval; + pCache[i]->nInactive += runModConf->pConf->globals.janitorInterval; } } } @@ -1023,7 +1023,7 @@ janitorCB(void *pUsr) STATSCOUNTER_INC(pData->ctrCloseTimeouts, pData->mutCtrCloseTimeouts); closeFile(pData); } else { - pData->nInactive += janitorInterval; + pData->nInactive += runModConf->pConf->globals.janitorInterval; } } } diff --git a/contrib/omrabbitmq/omrabbitmq.c b/contrib/omrabbitmq/omrabbitmq.c index 4d76964599..6514884e64 100644 --- a/contrib/omrabbitmq/omrabbitmq.c +++ b/contrib/omrabbitmq/omrabbitmq.c @@ -44,6 +44,7 @@ #include "cfsysline.h" #include "debug.h" #include "datetime.h" +#include "rsconf.h" #include @@ -253,7 +254,7 @@ static int amqp_authenticate(wrkrInstanceData_t *self, amqp_connection_state_t a amqp_rpc_reply_t ret; /* define the frame size */ - int frame_size = (glbl.GetMaxLine()<130000) ? 131072 : (glbl.GetMaxLine()+1072); + int frame_size = (glbl.GetMaxLine(runConf)<130000) ? 131072 : (glbl.GetMaxLine(runConf)+1072); /* authenticate */ ret = amqp_login(a_conn, (char const *)self->pData->vhost, 1, frame_size, 0, diff --git a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c index ebf12ebbef..dc28611adf 100644 --- a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c +++ b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -39,6 +39,7 @@ #include "parser.h" #include "datetime.h" #include "unicode-helper.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP @@ -174,7 +175,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); DBGPRINTF("aixforwardedfrom parser init called, compiled with version %s\n", VERSION); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ diff --git a/contrib/pmcisconames/pmcisconames.c b/contrib/pmcisconames/pmcisconames.c index 39506ce592..1220bfdd0e 100644 --- a/contrib/pmcisconames/pmcisconames.c +++ b/contrib/pmcisconames/pmcisconames.c @@ -40,6 +40,7 @@ #include "parser.h" #include "datetime.h" #include "unicode-helper.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP @@ -174,7 +175,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); DBGPRINTF("cisconames parser init called, compiled with version %s\n", VERSION); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ diff --git a/contrib/pmpanngfw/pmpanngfw.c b/contrib/pmpanngfw/pmpanngfw.c index eb3d715a92..c65cee7e56 100644 --- a/contrib/pmpanngfw/pmpanngfw.c +++ b/contrib/pmpanngfw/pmpanngfw.c @@ -38,6 +38,7 @@ #include "datetime.h" #include "unicode-helper.h" #include "typedefs.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP @@ -285,7 +286,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); DBGPRINTF("panngfw parser init called, compiled with version %s\n", VERSION); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ diff --git a/contrib/pmsnare/pmsnare.c b/contrib/pmsnare/pmsnare.c index a74517bddb..22f17f27f7 100644 --- a/contrib/pmsnare/pmsnare.c +++ b/contrib/pmsnare/pmsnare.c @@ -59,6 +59,7 @@ #include "parser.h" #include "datetime.h" #include "unicode-helper.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP @@ -117,7 +118,7 @@ static rsRetVal createInstance(instanceConf_t **pinst) { CHKmalloc(inst = malloc(sizeof(instanceConf_t))); inst->next = NULL; *pinst = inst; - + /* Add to list of instances. */ if(modInstances == NULL) { CHKmalloc(modInstances = malloc(sizeof(modInstances_t))); @@ -151,7 +152,7 @@ CODESTARTnewParserInst /* If using the old config, just use global settings for each instance. */ if (lst == NULL) FINALIZE; - + /* If using the new config, process module settings for this instance. */ if((pvals = nvlstGetParams(lst, &parserpblk, NULL)) == NULL) { ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); @@ -177,7 +178,7 @@ CODESTARTnewParserInst dbgprintf("pmsnare: program error, non-handled param '%s'\n", parserpblk.descr[i].name); } } - + finalize_it: CODE_STD_FINALIZERnewParserInst if(lst != NULL) @@ -220,13 +221,13 @@ CODESTARTendCnfLoad * This can't be done any earlier because the config wasn't fully loaded until now. */ for(inst = modInstances->root; inst != NULL; inst = inst->next) { if(inst->bEscapeCCOnRcv == -1) - inst->bEscapeCCOnRcv = glbl.GetParserEscapeControlCharactersOnReceive(); + inst->bEscapeCCOnRcv = glbl.GetParserEscapeControlCharactersOnReceive(modConf->pConf); if(inst->bEscapeTab == -1) - inst->bEscapeTab = glbl.GetParserEscapeControlCharacterTab(); + inst->bEscapeTab = glbl.GetParserEscapeControlCharacterTab(modConf->pConf); if(inst->bParserEscapeCCCStyle == -1) - inst->bParserEscapeCCCStyle = glbl.GetParserEscapeControlCharactersCStyle(); + inst->bParserEscapeCCCStyle = glbl.GetParserEscapeControlCharactersCStyle(modConf->pConf); if(inst->cCCEscapeChar == '\0') - inst->cCCEscapeChar = glbl.GetParserControlCharacterEscapePrefix(); + inst->cCCEscapeChar = glbl.GetParserControlCharacterEscapePrefix(modConf->pConf); /* Determine tab representation. Possible options: * "#011" escape on, escapetabs on, no change to prefix (default) @@ -275,7 +276,7 @@ BEGINparse2 uchar *p2parse; int lenMsg; int snaremessage; /* 0 means not a snare message, otherwise it's the index of the tab after the tag */ - + CODESTARTparse2 dbgprintf("Message will now be parsed by fix Snare parser.\n"); assert(pMsg != NULL); @@ -383,7 +384,7 @@ CODESTARTparse2 snaremessage = p2parse - pMsg->pszRawMsg + 11; } } - + if(snaremessage) { /* Skip to the end of the tag. */ p2parse = pMsg->pszRawMsg + snaremessage; @@ -403,7 +404,7 @@ CODESTARTparse2 DBGPRINTF("pmsnare: new message: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); } - + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: @@ -435,7 +436,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); DBGPRINTF("snare parser init called, compiled with version %s\n", VERSION); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ ENDmodInit diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index e064de49af..29fe8155e3 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -389,7 +389,7 @@ getStateFileDir(void) const uchar *wrkdir; assert(currModConf != NULL); if(currModConf->stateFileDirectory == NULL) { - wrkdir = glblGetWorkDirRaw(); + wrkdir = glblGetWorkDirRaw(currModConf->pConf); } else { wrkdir = currModConf->stateFileDirectory; } @@ -733,7 +733,7 @@ act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file, char basename[MAXFNAME]; DEFiRet; int fd = -1; - + DBGPRINTF("act_obj_add: edge %p, name '%s' (source '%s')\n", edge, name, source? source : "---"); if (isIgnoreOlderFile(edge->instarr[0], name)) { diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index b700b5c3b4..a58d599b0c 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -59,7 +59,7 @@ #include "glbl.h" #include "debug.h" #include "unlimited_select.h" - +#include "rsconf.h" MODULE_TYPE_INPUT MODULE_TYPE_NOKEEP @@ -203,7 +203,7 @@ onSessAccept(tcpsrv_t *pThis, tcps_sess_t *pSess) { DEFiRet; gsssrv_t *pGSrv; - + pGSrv = (gsssrv_t*) pThis->pUsr; if(pGSrv->allowedMethods & ALLOWEDMETHOD_GSS) { @@ -438,7 +438,7 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) allowedMethods = pGSrv->allowedMethods; if(allowedMethods & ALLOWEDMETHOD_GSS) { int ret = 0; - const size_t bufsize = glbl.GetMaxLine(); + const size_t bufsize = glbl.GetMaxLine(runConf); CHKmalloc(buf = (char*) malloc(bufsize + 1)); prop.GetString(pSess->fromHostIP, &pszPeer, &lenPeer); @@ -586,7 +586,7 @@ OnSessAcceptGSS(tcpsrv_t *pThis, tcps_sess_t *pSess) gssutil.display_ctx_flags(*sess_flags); pGSess->allowedMethods = ALLOWEDMETHOD_GSS; } - + finalize_it: free(buf); RETiRet; diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c index 62a5ab2064..9b227a1793 100644 --- a/plugins/imjournal/imjournal.c +++ b/plugins/imjournal/imjournal.c @@ -67,6 +67,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(statsobj) struct modConfData_s { + rsconf_t *pConf; int bIgnPrevMsg; }; @@ -141,6 +142,8 @@ struct journalContext_s { /* structure encapsulating all the journald_API-relate char *cursor; /* should point to last valid journald entry we processed */ }; static struct journalContext_s journalContext = {NULL, 0, 1, NULL}; +static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ +static modConfData_t *runModConf = NULL;/* modConf ptr to use for run process */ #define J_PROCESS_PERIOD 1024 /* Call sd_journal_process() every 1,024 records */ @@ -158,7 +161,7 @@ static rsRetVal openJournal(void) { LogError(-r, RS_RET_IO_ERROR, "imjournal: sd_journal_open() failed"); iRet = RS_RET_IO_ERROR; } - if ((r = sd_journal_set_data_threshold(journalContext.j, glbl.GetMaxLine())) < 0) { + if ((r = sd_journal_set_data_threshold(journalContext.j, glbl.GetMaxLine(runModConf->pConf))) < 0) { LogError(-r, RS_RET_IO_ERROR, "imjournal: sd_journal_set_data_threshold() failed"); iRet = RS_RET_IO_ERROR; } @@ -577,12 +580,14 @@ persistJournalState(void) } /* In order to guarantee physical write we need to force parent sync as well */ DIR *wd; - if (!(wd = opendir((char *)glbl.GetWorkDir()))) { - LogError(errno, RS_RET_IO_ERROR, "imjournal: failed to open '%s' directory", glbl.GetWorkDir()); + if (!(wd = opendir((char *)glbl.GetWorkDir(runModConf->pConf)))) { + LogError(errno, RS_RET_IO_ERROR, "imjournal: failed to open '%s' directory", + glbl.GetWorkDir(runModConf->pConf)); ABORT_FINALIZE(RS_RET_IO_ERROR); } if (fsync(dirfd(wd)) != 0) { - LogError(errno, RS_RET_IO_ERROR, "imjournal: fsync on '%s' failed", glbl.GetWorkDir()); + LogError(errno, RS_RET_IO_ERROR, "imjournal: fsync on '%s' failed", + glbl.GetWorkDir(runModConf->pConf)); ABORT_FINALIZE(RS_RET_IO_ERROR); } @@ -884,6 +889,8 @@ ENDrunInput BEGINbeginCnfLoad CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; bLegacyCnfModGlobalsPermitted = 1; cs.bIgnoreNonValidStatefile = 1; @@ -906,9 +913,10 @@ CODESTARTendCnfLoad /* bad trick to handle old and new style config all in old-style var */ if(cs.stateFile != NULL && cs.stateFile[0] != '/') { char *new_stateFile; - if (-1 == asprintf(&new_stateFile, "%s/%s", (char *)glbl.GetWorkDir(), cs.stateFile)) { - LogError(0, RS_RET_OUT_OF_MEMORY, "imjournal: asprintf failed\n"); - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + if (-1 == asprintf(&new_stateFile, "%s/%s", + (char *)glbl.GetWorkDir(loadModConf->pConf), cs.stateFile)) { + LogError(0, RS_RET_OUT_OF_MEMORY, "imjournal: asprintf failed\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } free (cs.stateFile); cs.stateFile = new_stateFile; @@ -924,6 +932,7 @@ ENDcheckCnf BEGINactivateCnf CODESTARTactivateCnf + runModConf = pModConf; /* support statistic gathering */ CHKiRet(statsobj.Construct(&(statsCounter.stats))); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 0a1bc44d84..38730c3b3c 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -309,7 +309,7 @@ rsRetVal Syslog(modConfData_t *pModConf, syslog_pri_t priority, uchar *pMsg, str */ int klog_getMaxLine(void) { - return glbl.GetMaxLine(); + return glbl.GetMaxLine(runConf); } diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 6b806a1de5..33a575c3d4 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -549,7 +549,7 @@ startupSrv(ptcpsrv_t *pSrv) memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_PASSIVE; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runModConf->pConf); hints.ai_socktype = SOCK_STREAM; error = getaddrinfo((char*)pSrv->lstnIP, (char*) pSrv->port, &hints, &res); @@ -762,7 +762,7 @@ getPeerNames(prop_t **peerName, prop_t **peerIP, struct sockaddr *pAddr, sbool b ABORT_FINALIZE(RS_RET_INVALID_HNAME); } - if (!glbl.GetDisableDNS()) { + if (!glbl.GetDisableDNS(runConf)) { error = getnameinfo(pAddr, SALEN(pAddr), (char *) szHname, NI_MAXHOST, NULL, 0, NI_NAMEREQD); if (error == 0) { memset(&hints, 0, sizeof(struct addrinfo)); @@ -1282,7 +1282,7 @@ DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, struct syslogT iRet = multiSubmitFlush(&multiSub); - if(glblSenderKeepTrack) + if(runConf->globals.senderKeepTrack) statsRecordSender(propGetSzStr(pThis->peerName), nMsgs, ttGenTime); finalize_it: @@ -2386,7 +2386,7 @@ ENDcheckCnf BEGINactivateCnfPrePrivDrop instanceConf_t *inst; CODESTARTactivateCnfPrePrivDrop - iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + iMaxLine = glbl.GetMaxLine(runConf); /* get maximum size we currently support */ DBGPRINTF("imptcp: config params iMaxLine %d\n", iMaxLine); runModConf = pModConf; diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 90009bb60a..5d7d16a483 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -379,13 +379,13 @@ addListner(modConfData_t __attribute__((unused)) *modConf, instanceConf_t *inst) if(pRelpEngine == NULL) { CHKiRet(relpEngineConstruct(&pRelpEngine)); CHKiRet(relpEngineSetDbgprint(pRelpEngine, (void (*)(char *, ...))imrelp_dbgprintf)); - CHKiRet(relpEngineSetFamily(pRelpEngine, glbl.GetDefPFFamily())); + CHKiRet(relpEngineSetFamily(pRelpEngine, glbl.GetDefPFFamily(runModConf->pConf))); CHKiRet(relpEngineSetEnableCmd(pRelpEngine, (uchar*) "syslog", eRelpCmdState_Required)); CHKiRet(relpEngineSetSyslogRcv2(pRelpEngine, onSyslogRcv)); CHKiRet(relpEngineSetOnErr(pRelpEngine, onErr)); CHKiRet(relpEngineSetOnGenericErr(pRelpEngine, onGenericErr)); CHKiRet(relpEngineSetOnAuthErr(pRelpEngine, onAuthErr)); - if (!glbl.GetDisableDNS()) { + if (!glbl.GetDisableDNS(runModConf->pConf)) { CHKiRet(relpEngineSetDnsLookupMode(pRelpEngine, 1)); } #if defined(HAVE_RELPENGINESETTLSLIBBYNAME) @@ -768,9 +768,9 @@ CODESTARTcheckCnf /* We set default value for maxDataSize here because * otherwise the maxMessageSize isn't set. */ - inst->maxDataSize = glbl.GetMaxLine(); + inst->maxDataSize = glbl.GetMaxLine(loadConf); } - maxMessageSize = (size_t)glbl.GetMaxLine(); + maxMessageSize = (size_t)glbl.GetMaxLine(loadConf); if(inst->maxDataSize < maxMessageSize) { LogError(0, RS_RET_INVALID_PARAMS, "error: " "maxDataSize (%zu) is smaller than global parameter " diff --git a/plugins/imsolaris/imsolaris.c b/plugins/imsolaris/imsolaris.c index 732104e188..eff8ffb044 100644 --- a/plugins/imsolaris/imsolaris.c +++ b/plugins/imsolaris/imsolaris.c @@ -84,6 +84,7 @@ #include "prop.h" #include "sun_cddl.h" #include "datetime.h" +#include "rsconf.h" MODULE_TYPE_INPUT MODULE_TYPE_NOKEEP @@ -244,7 +245,7 @@ getMsgs(thrdInfo_t *pThrd, int timeout) uchar bufRcv[4096+1]; char errStr[1024]; - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runConf); /* we optimize performance: if iMaxLine is below 4K (which it is in almost all * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 8ddac9c101..1b6c5d24ea 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -443,11 +443,11 @@ processPacket(struct lstn_s *lstn, struct sockaddr_storage *frominetPrev, int *p */ *pbIsPermitted = net.isAllowedSender2((uchar*)"UDP", (struct sockaddr *)frominet, "", 0); - + if(*pbIsPermitted == 0) { DBGPRINTF("msg is not from an allowed sender\n"); STATSCOUNTER_INC(lstn->ctrDisallowed, lstn->mutCtrDisallowed); - if(glbl.GetOption_DisallowWarning) { + if(glbl.GetOptionDisallowWarning(runModConf->pConf)) { LogError(0, NO_ERRCODE, "imudp: UDP message from disallowed sender discarded"); } @@ -1187,7 +1187,7 @@ BEGINactivateCnf int lenRcvBuf; CODESTARTactivateCnf /* caching various settings */ - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runConf); lenRcvBuf = iMaxLine + 1; # ifdef HAVE_RECVMMSG lenRcvBuf *= runModConf->batchSize; diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 7d90166060..7129535f19 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -336,7 +336,7 @@ createInstance(instanceConf_t **pinst) inst->bWritePid = 0; inst->bAnnotate = 0; inst->bParseTrusted = 0; - inst->bDiscardOwnMsgs = bProcessInternalMessages; + inst->bDiscardOwnMsgs = loadModConf->pConf->globals.bProcessInternalMessages; inst->bUnlink = 1; inst->next = NULL; @@ -700,7 +700,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) if(cred == NULL) return; - + lenPID = snprintf(bufPID, sizeof(bufPID), "[%lu]:", (unsigned long) cred->pid); for(i = *lenTag ; i >= 0 && bufTAG[i] != '[' ; --i) @@ -708,7 +708,7 @@ fixPID(uchar *bufTAG, int *lenTag, struct ucred *cred) if(i < 0) i = *lenTag - 1; /* go right at end of TAG, pid was not present (-1 for ':') */ - + if(i + lenPID > CONF_TAG_MAXSIZE) return; /* do not touch, as things would break */ @@ -744,7 +744,7 @@ getTrustedProp(struct ucred *cred, const char *propName, uchar *buf, size_t lenB close(fd); ABORT_FINALIZE(RS_RET_ERR); } - + /* we strip after the first \n */ for(i = 0 ; i < lenRead ; ++i) { if(buf[i] == '\n') @@ -780,7 +780,7 @@ getTrustedExe(struct ucred *cred, uchar *buf, size_t lenBuf, int* lenProp) DBGPRINTF("error reading link '%s'\n", namebuf); ABORT_FINALIZE(RS_RET_ERR); } - + buf[lenRead] = '\0'; *lenProp = lenRead; @@ -843,7 +843,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim parse = pRcv; lenMsg = lenRcv; offs = 1; /* '<' */ - + parse++; pri = 0; while(offs < lenMsg && isdigit(*parse)) { @@ -930,7 +930,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim (long unsigned) cred->gid); memcpy(pmsgbuf+toffs, propBuf, lenProp); toffs = toffs + lenProp; - + if(getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { memcpy(pmsgbuf+toffs, " _COMM=", 7); memcpy(pmsgbuf+toffs+7, propBuf, lenProp); @@ -1071,7 +1071,7 @@ static rsRetVal readSocket(lstn_t *pLstn) assert(pLstn->fd >= 0); - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runConf); /* we optimize performance: if iMaxLine is below 4K (which it is in almost all * cases, we use a fixed buffer on the stack. Only if it is higher, heap memory @@ -1252,7 +1252,7 @@ CODESTARTbeginCnfLoad /* if we do not process internal messages, we will see messages * from ourselves, and so we need to permit this. */ - pModConf->bDiscardOwnMsgs = bProcessInternalMessages; + pModConf->bDiscardOwnMsgs = pConf->globals.bProcessInternalMessages; pModConf->bUnlink = 1; pModConf->ratelimitIntervalSysSock = DFLT_ratelimitInterval; pModConf->ratelimitBurstSysSock = DFLT_ratelimitBurst; @@ -1748,7 +1748,7 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &cs.ratelimitBurstSysSock, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); CHKiRet(regCfSysLineHdlr2((uchar *)"systemlogratelimitseverity", 0, eCmdHdlrInt, NULL, &cs.ratelimitSeveritySysSock, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); - + /* support statistics gathering */ CHKiRet(statsobj.Construct(&modStats)); CHKiRet(statsobj.SetName(modStats, UCHAR_CONSTANT("imuxsock"))); diff --git a/plugins/mmexternal/mmexternal.c b/plugins/mmexternal/mmexternal.c index b27cedb087..317420afdc 100644 --- a/plugins/mmexternal/mmexternal.c +++ b/plugins/mmexternal/mmexternal.c @@ -41,6 +41,7 @@ #include "errmsg.h" #include "cfsysline.h" #include "glbl.h" +#include "rsconf.h" MODULE_TYPE_OUTPUT @@ -393,7 +394,7 @@ cleanup(wrkrInstanceData_t *pWrkrData) /* waitpid will fail with errno == ECHILD if the child process has already been reaped by the rsyslogd main loop (see rsyslogd.c) */ if(ret == pWrkrData->pid) { - glblReportChildProcessExit(pWrkrData->pData->szBinary, pWrkrData->pid, status); + glblReportChildProcessExit(runConf, pWrkrData->pData->szBinary, pWrkrData->pid, status); } if(pWrkrData->fdOutput != -1) { @@ -442,7 +443,7 @@ callExtProg(wrkrInstanceData_t *__restrict__ const pWrkrData, smsg_t *__restrict int bFreeInputstr = 1; /* we must only free if it does not point to msg-obj mem! */ const uchar *inputstr = NULL; /* string to be processed by external program */ DEFiRet; - + if(pWrkrData->pData->inputProp == INPUT_MSG) { inputstr = getMSG(pMsg); lenWrite = getMSGLen(pMsg); @@ -512,7 +513,7 @@ CODESTARTdoAction if(pWrkrData->bIsRunning == 0) { openPipe(pWrkrData); } - + iRet = callExtProg(pWrkrData, pMsg); if(iRet != RS_RET_OK) diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 26ac21f22a..8f3edcc9a5 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -54,6 +54,7 @@ #include "tcpclt.h" #include "glbl.h" #include "errmsg.h" +#include "rsconf.h" MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP @@ -323,7 +324,7 @@ static rsRetVal TCPSendGSSSend(void *pvData, char *msg, size_t len) gssutil.display_status((char*)"wrapping message", maj_stat, min_stat); goto fail; } - + if (gssutil.send_token(s, &out_buf) < 0) { goto fail; } @@ -366,7 +367,7 @@ static rsRetVal doTryResume(instanceData *pData) * a common function. */ hints.ai_flags = AI_NUMERICSERV; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runConf); hints.ai_socktype = SOCK_STREAM; if(getaddrinfo(pData->f_hname, getFwdSyslogPt(pData), &hints, &res) == 0) { dbgprintf("%s found, resuming.\n", pData->f_hname); @@ -413,7 +414,7 @@ CODESTARTdoAction case eDestFORW: dbgprintf(" %s:%s/%s\n", pData->f_hname, getFwdSyslogPt(pData), "tcp-gssapi"); - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runConf); psz = (char*) ppString[0]; l = strlen((char*) psz); if((int) l > iMaxLine) @@ -586,7 +587,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) *(pData->port + i) = '\0'; } } - + /* now skip to template */ bErr = 0; @@ -620,7 +621,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) memset(&hints, 0, sizeof(hints)); /* port must be numeric, because config file syntax requests this */ hints.ai_flags = AI_NUMERICSERV; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(loadConf); hints.ai_socktype = SOCK_STREAM; if(getaddrinfo(pData->f_hname, getFwdSyslogPt(pData), &hints, &res) != 0) { pData->eDestState = eDestFORW_UNKN; diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 4f46f92404..11dc31afc2 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -48,6 +48,7 @@ #include "errmsg.h" #include "cfsysline.h" #include "glbl.h" +#include "rsconf.h" MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP @@ -347,7 +348,7 @@ waitForChild(instanceData *pData, childProcessCtx_t *pChildCtx) /* waitpid will fail with errno == ECHILD if the child process has already been reaped by the rsyslogd main loop (see rsyslogd.c) */ if(ret == pChildCtx->pid) { - glblReportChildProcessExit(pData->szBinary, pChildCtx->pid, status); + glblReportChildProcessExit(runConf, pData->szBinary, pChildCtx->pid, status); } } @@ -913,7 +914,7 @@ postInitInstance(instanceData *pData) BEGINcreateWrkrInstance CODESTARTcreateWrkrInstance pWrkrData->pChildCtx = NULL; - + if(pWrkrData->pData->pOutputCaptureCtx != NULL) { CHKiRet(startOutputCaptureOnce(pWrkrData->pData->pOutputCaptureCtx)); } diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index bb0d13e551..5fc1dccdbe 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -390,6 +390,7 @@ setInstParamDefaults(instanceData *pData) BEGINbeginCnfLoad CODESTARTbeginCnfLoad loadModConf = pModConf; + pModConf->pConf = pConf; pModConf->tlslib = NULL; /* create our relp engine */ CHKiRet(relpEngineConstruct(&pRelpEngine)); @@ -583,7 +584,7 @@ doConnect(wrkrInstanceData_t *const pWrkrData) DEFiRet; if(pWrkrData->bInitialConnect) { - iRet = relpCltConnect(pWrkrData->pRelpClt, glbl.GetDefPFFamily(), + iRet = relpCltConnect(pWrkrData->pRelpClt, glbl.GetDefPFFamily(runModConf->pConf), getRelpPt(pWrkrData->pData), pWrkrData->pData->target); if(iRet == RELP_RET_OK) pWrkrData->bInitialConnect = 0; @@ -671,8 +672,8 @@ CODESTARTdoAction lenMsg = strlen((char*) pMsg); /* TODO: don't we get this? */ /* we need to truncate oversize msgs - no way around that... */ - if((int) lenMsg > glbl.GetMaxLine()) - lenMsg = glbl.GetMaxLine(); + if((int) lenMsg > glbl.GetMaxLine(runModConf->pConf)) + lenMsg = glbl.GetMaxLine(runModConf->pConf); /* forward */ ret = relpCltSendSyslog(pWrkrData->pRelpClt, (uchar*) pMsg, lenMsg); diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c index 2848cba7a8..4c4f9d8294 100644 --- a/plugins/omudpspoof/omudpspoof.c +++ b/plugins/omudpspoof/omudpspoof.c @@ -582,7 +582,7 @@ static rsRetVal doTryResume(wrkrInstanceData_t *pWrkrData) memset(&hints, 0, sizeof(hints)); /* port must be numeric, because config file syntax requires this */ hints.ai_flags = AI_NUMERICSERV; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runModConf->pConf); hints.ai_socktype = SOCK_DGRAM; if((iErr = (getaddrinfo((char*)pData->host, (char*)getFwdPt(pData), &hints, &res))) != 0) { DBGPRINTF("could not get addrinfo for hostname '%s':'%s': %d%s\n", @@ -620,7 +620,7 @@ CODESTARTdoAction DBGPRINTF(" %s:%s/omudpspoof, src '%s', msg strt '%.256s'\n", pWrkrData->pData->host, getFwdPt(pWrkrData->pData), ppString[1], ppString[0]); - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runModConf->pConf); psz = (char*) ppString[0]; l = strlen((char*) psz); if((int) l > iMaxLine) diff --git a/plugins/omuxsock/omuxsock.c b/plugins/omuxsock/omuxsock.c index d64c3851f0..99fbc22c7e 100644 --- a/plugins/omuxsock/omuxsock.c +++ b/plugins/omuxsock/omuxsock.c @@ -349,7 +349,7 @@ CODESTARTdoAction pthread_mutex_lock(&mutDoAct); CHKiRet(doTryResume(pWrkrData->pData)); - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runModConf->pConf); DBGPRINTF(" omuxsock:%s\n", pWrkrData->pData->sockName); @@ -382,7 +382,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) if(*(p-1) == ';') --p; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, getDfltTpl())); - + if(cs.sockName == NULL) { LogError(0, RS_RET_NO_SOCK_CONFIGURED, "No output socket configured for omuxsock\n"); ABORT_FINALIZE(RS_RET_NO_SOCK_CONFIGURED); diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c index 21d6a2b43d..7916515ee8 100644 --- a/plugins/pmlastmsg/pmlastmsg.c +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -44,6 +44,7 @@ #include "parser.h" #include "datetime.h" #include "unicode-helper.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP @@ -158,7 +159,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ diff --git a/runtime/dnscache.c b/runtime/dnscache.c index d41c6f226d..7dd0aabbc6 100644 --- a/runtime/dnscache.c +++ b/runtime/dnscache.c @@ -44,6 +44,7 @@ #include "hashtable.h" #include "prop.h" #include "dnscache.h" +#include "rsconf.h" /* module data structures */ struct dnscache_entry_s { @@ -64,9 +65,6 @@ struct dnscache_s { }; typedef struct dnscache_s dnscache_t; -unsigned dnscacheDefaultTTL = 24 * 60 * 60; /* 24 hrs default TTL */ -int dnscacheEnableTTL = 0; /* expire entries or not (0) ? */ - /* static data */ DEFobjStaticHelpers @@ -259,7 +257,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) char fqdnBuf[NI_MAXHOST]; rs_size_t fqdnLen; rs_size_t i; - + error = mygetnameinfo((struct sockaddr *)addr, SALEN((struct sockaddr *)addr), (char*) szIP, sizeof(szIP), NULL, 0, NI_NUMERICHOST); if(error) { @@ -267,7 +265,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) ABORT_FINALIZE(RS_RET_INVALID_SOURCE); } - if(!glbl.GetDisableDNS()) { + if(!glbl.GetDisableDNS(runConf)) { sigemptyset(&nmask); sigaddset(&nmask, SIGHUP); pthread_sigmask(SIG_BLOCK, &nmask, &omask); @@ -294,7 +292,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) * time being, we simply drop the name we obtained and use the IP - that one * is OK in any way. We do also log the error message. rgerhards, 2007-07-16 */ - if(glbl.GetDropMalPTRMsgs() == 1) { + if(glbl.GetDropMalPTRMsgs(runConf) == 1) { LogError(0, RS_RET_MALICIOUS_ENTITY, "Malicious PTR record, message dropped " "IP = \"%s\" HOST = \"%s\"", @@ -335,7 +333,7 @@ resolveAddr(struct sockaddr_storage *addr, dnscache_entry_t *etry) prop.CreateStringProp(&etry->ip, (uchar*)szIP, strlen(szIP)); - if(error || glbl.GetDisableDNS()) { + if(error || glbl.GetDisableDNS(runConf)) { dbgprintf("Host name for your address (%s) unknown\n", szIP); prop.AddRef(etry->ip); etry->fqdn = etry->ip; @@ -364,8 +362,8 @@ addEntry(struct sockaddr_storage *const addr, dnscache_entry_t **const pEtry) assert(etry != NULL); memcpy(&etry->addr, addr, SALEN((struct sockaddr*) addr)); etry->nUsed = 0; - if(dnscacheEnableTTL) { - etry->validUntil = time(NULL) + dnscacheDefaultTTL; + if(runConf->globals.dnscacheEnableTTL) { + etry->validUntil = time(NULL) + runConf->globals.dnscacheDefaultTTL; } memcpy(keybuf, addr, sizeof(struct sockaddr_storage)); @@ -395,12 +393,12 @@ findEntry(struct sockaddr_storage *const addr, dnscache_entry_t * etry = hashtable_search(dnsCache.ht, addr); DBGPRINTF("findEntry: 1st lookup found %p\n", etry); - if(etry == NULL || (dnscacheEnableTTL && (etry->validUntil <= time(NULL)))) { + if(etry == NULL || (runConf->globals.dnscacheEnableTTL && (etry->validUntil <= time(NULL)))) { pthread_rwlock_unlock(&dnsCache.rwlock); pthread_rwlock_wrlock(&dnsCache.rwlock); etry = hashtable_search(dnsCache.ht, addr); /* re-query, might have changed */ DBGPRINTF("findEntry: 2nd lookup found %p\n", etry); - if(etry == NULL || (dnscacheEnableTTL && (etry->validUntil <= time(NULL)))) { + if(etry == NULL || (runConf->globals.dnscacheEnableTTL && (etry->validUntil <= time(NULL)))) { if(etry != NULL) { DBGPRINTF("hashtable: entry timed out, discarding it; " "valid until %lld, now %lld\n", diff --git a/runtime/dnscache.h b/runtime/dnscache.h index 9f0be2008e..0ec1dbccca 100644 --- a/runtime/dnscache.h +++ b/runtime/dnscache.h @@ -28,6 +28,4 @@ rsRetVal ATTR_NONNULL(1, 5) dnscacheLookup(struct sockaddr_storage *const addr, prop_t **const fqdn, prop_t **const fqdnLowerCase, prop_t **const localName, prop_t **const ip); -extern unsigned dnscacheDefaultTTL; -extern int dnscacheEnableTTL; #endif /* #ifndef INCLUDED_DNSCACHE_H */ diff --git a/runtime/errmsg.c b/runtime/errmsg.c index cd52ddbf4d..ecc0452ce9 100644 --- a/runtime/errmsg.c +++ b/runtime/errmsg.c @@ -45,6 +45,7 @@ #include "operatingstate.h" #include "srUtils.h" #include "stringbuf.h" +#include "rsconf.h" /* static data */ #ifndef O_LARGEFILE @@ -90,7 +91,7 @@ doLogMsg(const int iErrno, const int iErrCode, const int severity, const char * { char buf[2048]; char errStr[1024]; - + dbgprintf("Called LogMsg, msg: %s\n", msg); osf_write(OSF_TAG_MSG, msg); @@ -112,9 +113,9 @@ doLogMsg(const int iErrno, const int iErrCode, const int severity, const char * } buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ errno = 0; - + const int msglen = (int) strlen(buf); - if(msglen > glblGetMaxLine()) { + if(msglen > glblGetMaxLine(ourConf)) { /* in extreme cases, our error messages may be longer than the configured * max message size. If so, we just truncate without further indication, as * anything else would probably lead to a death loop on error messages. @@ -122,7 +123,7 @@ doLogMsg(const int iErrno, const int iErrCode, const int severity, const char * * much value in supporting extremely short max message sizes - we assume * it's just a testbench thing. -- rgerhards, 2018-05-11 */ - buf[glblGetMaxLine()] = '\0'; /* space must be available! */ + buf[glblGetMaxLine(ourConf)] = '\0'; /* space must be available! */ } glblErrLogger(severity, iErrCode, (uchar*)buf); @@ -147,7 +148,7 @@ LogError(const int iErrno, const int iErrCode, const char *fmt, ... ) va_list ap; char buf[2048]; int lenBuf; - + va_start(ap, fmt); lenBuf = vsnprintf(buf, sizeof(buf), fmt, ap); if(lenBuf < 0) { @@ -155,7 +156,7 @@ LogError(const int iErrno, const int iErrCode, const char *fmt, ... ) } va_end(ap); buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ - + doLogMsg(iErrno, iErrCode, LOG_ERR, buf); } @@ -175,7 +176,7 @@ LogMsg(const int iErrno, const int iErrCode, const int severity, const char *fmt va_list ap; char buf[2048]; int lenBuf; - + va_start(ap, fmt); lenBuf = vsnprintf(buf, sizeof(buf), fmt, ap); if(lenBuf < 0) { @@ -183,7 +184,7 @@ LogMsg(const int iErrno, const int iErrCode, const int severity, const char *fmt } va_end(ap); buf[sizeof(buf) - 1] = '\0'; /* just to be on the safe side... */ - + doLogMsg(iErrno, iErrCode, severity, buf); } @@ -208,7 +209,7 @@ writeOversizeMessageLog(const smsg_t *const pMsg) DEFiRet; ISOBJ_TYPE_assert(pMsg, msg); - if(glblGetOversizeMsgErrorFile() == NULL) { + if(glblGetOversizeMsgErrorFile(runConf) == NULL) { FINALIZE; } @@ -216,12 +217,12 @@ writeOversizeMessageLog(const smsg_t *const pMsg) mutexLocked = 1; if(fdOversizeMsgLog == -1) { - fdOversizeMsgLog = open((char*)glblGetOversizeMsgErrorFile(), + fdOversizeMsgLog = open((char*)glblGetOversizeMsgErrorFile(runConf), O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE|O_CLOEXEC, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); if(fdOversizeMsgLog == -1) { LogError(errno, RS_RET_ERR, "error opening oversize message log file %s", - glblGetOversizeMsgErrorFile()); + glblGetOversizeMsgErrorFile(runConf)); FINALIZE; } } @@ -251,7 +252,7 @@ writeOversizeMessageLog(const smsg_t *const pMsg) if(wrRet != (ssize_t) toWrite) { LogError(errno, RS_RET_IO_ERROR, "error writing oversize message log file %s, write returned %lld", - glblGetOversizeMsgErrorFile(), (long long) wrRet); + glblGetOversizeMsgErrorFile(runConf), (long long) wrRet); } finalize_it: diff --git a/runtime/glbl.c b/runtime/glbl.c index 52a598f8ff..39f5fa7c1e 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -59,10 +59,6 @@ #include "dnscache.h" #include "parser.h" -#define REPORT_CHILD_PROCESS_EXITS_NONE 0 -#define REPORT_CHILD_PROCESS_EXITS_ERRORS 1 -#define REPORT_CHILD_PROCESS_EXITS_ALL 2 - /* some defaults */ #ifndef DFLT_NETSTRM_DRVR # define DFLT_NETSTRM_DRVR ((uchar*)"ptcp") @@ -77,36 +73,9 @@ DEFobjCurrIf(net) * For this object, these variables are obviously what makes the "meat" of the * class... */ -int glblDebugOnShutdown = 0; /* start debug log when we are shut down */ -#ifdef ENABLE_LIBLOGGING_STDLOG -stdlog_channel_t stdlog_hdl = NULL; /* handle to be used for stdlog */ -#endif static struct cnfobj *mainqCnfObj = NULL;/* main queue object, to be used later in startup sequence */ -#ifndef DFLT_INT_MSGS_SEV_FILTER - #define DFLT_INT_MSGS_SEV_FILTER 6 /* Warning level and more important */ -#endif -int glblIntMsgsSeverityFilter = DFLT_INT_MSGS_SEV_FILTER;/* filter for logging internal messages by syslog sev. */ -int bProcessInternalMessages = 0; /* Should rsyslog itself process internal messages? - * 1 - yes - * 0 - send them to libstdlog (e.g. to push to journal) or syslog() - */ -static uchar *pszWorkDir = NULL; -#ifdef ENABLE_LIBLOGGING_STDLOG -static uchar *stdlog_chanspec = NULL; -#endif -static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */ static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ -static int iMaxLine = 8096; /* maximum length of a syslog message */ -static uchar * oversizeMsgErrorFile = NULL; /* File where oversize messages are written to */ -static int oversizeMsgInputMode = 0; /* Mode which oversize messages will be forwarded */ -static int reportOversizeMsg = 1; /* shall error messages be generated for oversize messages? */ -static int reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; -static int iGnuTLSLoglevel = 0; /* Sets GNUTLS Debug Level */ -static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ -static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ -static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ -static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ static prop_t *propLocalIPIF = NULL;/* IP address to report for the local host (default is 127.0.0.1) */ static int propLocalIPIF_set = 0; /* is propLocalIPIF already set? */ static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ @@ -115,40 +84,12 @@ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup, e static uchar *LocalHostNameOverride = NULL;/* user-overridden hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup, except HUP */ static uchar *LocalDomain = NULL;/* our local domain name - read-only after startup, except HUP */ -static char **StripDomains = NULL; -/* these domains may be stripped before writing logs - r/o after s.u., never touched by init */ -static char **LocalHosts = NULL; -/* these hosts are logged with their hostname - read-only after startup, never touched by init */ -static uchar *pszDfltNetstrmDrvr = NULL; /* module name of default netstream driver */ -static uchar *pszDfltNetstrmDrvrCAF = NULL; /* default CA file for the netstrm driver */ -static uchar *pszDfltNetstrmDrvrKeyFile = NULL; /* default key file for the netstrm driver (server) */ -static uchar *pszDfltNetstrmDrvrCertFile = NULL; /* default cert file for the netstrm driver (server) */ int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ -static uchar cCCEscapeChar = '#'; /* character to be used to start an escape sequence for control chars */ -static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ -static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ -static int bSpaceLFOnRcv = 0; /* replace newlines with spaces on reception: 0 - no, 1 - yes */ -static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */ -static int bEscapeTab = 1; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ -static int bParserEscapeCCCStyle = 0; /* escape control characters in c style: 0 - no, 1 - yes */ -short janitorInterval = 10; /* interval (in minutes) at which the janitor runs */ -int glblReportNewSenders = 0; -int glblReportGoneAwaySenders = 0; -int glblSenderStatsTimeout = 12 * 60 * 60; /* 12 hr timeout for senders */ -int glblSenderKeepTrack = 0; /* keep track of known senders? */ int glblUnloadModules = 1; -int bPermitSlashInProgramname = 0; -int glblIntMsgRateLimitItv = 5; -int glblIntMsgRateLimitBurst = 500; char** glblDbgFiles = NULL; size_t glblDbgFilesNum = 0; int glblDbgWhitelist = 1; int glblPermitCtlC = 0; -int glblInputTimeoutShutdown = 1000; /* input shutdown timeout in ms */ -int glblShutdownQueueDoubleSize = 0; -static const uchar * operatingStateFile = NULL; - -uint64_t glblDevOptions = 0; /* to be used by developers only */ pid_t glbl_ourpid; #ifndef HAVE_ATOMIC_BUILTINS @@ -260,16 +201,17 @@ static struct cnfparamvals *cnfparamvals = NULL; */ int -glblGetMaxLine(void) +glblGetMaxLine(rsconf_t *cnf) { - return(iMaxLine); + assert(cnf != NULL); + return(cnf->globals.iMaxLine); } int -GetGnuTLSLoglevel(void) +GetGnuTLSLoglevel(rsconf_t *cnf) { - return(iGnuTLSLoglevel); + return(cnf->globals.iGnuTLSLoglevel); } /* define a macro for the simple properties' set and get functions @@ -293,32 +235,52 @@ static dataType Get##nameFunc(void) \ SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) SIMP_PROP(mainqCnfObj, mainqCnfObj, struct cnfobj *) -SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) -SIMP_PROP(StripDomains, StripDomains, char**) -SIMP_PROP(LocalHosts, LocalHosts, char**) -SIMP_PROP(ParserControlCharacterEscapePrefix, cCCEscapeChar, uchar) -SIMP_PROP(ParserDropTrailingLFOnReception, bDropTrailingLF, int) -SIMP_PROP(ParserEscapeControlCharactersOnReceive, bEscapeCCOnRcv, int) -SIMP_PROP(ParserSpaceLFOnReceive, bSpaceLFOnRcv, int) -SIMP_PROP(ParserEscape8BitCharactersOnReceive, bEscape8BitChars, int) -SIMP_PROP(ParserEscapeControlCharacterTab, bEscapeTab, int) -SIMP_PROP(ParserEscapeControlCharactersCStyle, bParserEscapeCCCStyle, int) #ifdef USE_UNLIMITED_SELECT SIMP_PROP(FdSetSize, iFdSetSize, int) #endif -SIMP_PROP_SET(DfltNetstrmDrvr, pszDfltNetstrmDrvr, uchar*) /* TODO: use custom function which frees existing value */ -SIMP_PROP_SET(DfltNetstrmDrvrCAF, pszDfltNetstrmDrvrCAF, uchar*) -/* TODO: use custom function which frees existing value */ -SIMP_PROP_SET(DfltNetstrmDrvrKeyFile, pszDfltNetstrmDrvrKeyFile, uchar*) -/* TODO: use custom function which frees existing value */ -SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) -/* TODO: use custom function which frees existing value */ - #undef SIMP_PROP #undef SIMP_PROP_SET #undef SIMP_PROP_GET +/* This is based on the previous SIMP_PROP but as a getter it uses + * additional parameter specifying the configuration it belongs to. + * The setter uses loadConf + */ +#define SIMP_PROP(nameFunc, nameVar, dataType) \ + SIMP_PROP_GET(nameFunc, nameVar, dataType) \ + SIMP_PROP_SET(nameFunc, nameVar, dataType) +#define SIMP_PROP_SET(nameFunc, nameVar, dataType) \ +static rsRetVal Set##nameFunc(dataType newVal) \ +{ \ + loadConf->globals.nameVar = newVal; \ + return RS_RET_OK; \ +} +#define SIMP_PROP_GET(nameFunc, nameVar, dataType) \ +static dataType Get##nameFunc(rsconf_t *cnf) \ +{ \ + return(cnf->globals.nameVar); \ +} + +SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) +SIMP_PROP(DisableDNS, bDisableDNS, int) +SIMP_PROP(ParserEscapeControlCharactersCStyle, parser.bParserEscapeCCCStyle, int) +SIMP_PROP(ParseHOSTNAMEandTAG, parser.bParseHOSTNAMEandTAG, int) +SIMP_PROP(OptionDisallowWarning, optionDisallowWarning, int) +/* We omit setter on purpose, because we want to customize it */ +SIMP_PROP_GET(DfltNetstrmDrvrCAF, pszDfltNetstrmDrvrCAF, uchar*) +SIMP_PROP_GET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) +SIMP_PROP_GET(DfltNetstrmDrvrKeyFile, pszDfltNetstrmDrvrKeyFile, uchar*) +SIMP_PROP_GET(ParserControlCharacterEscapePrefix, parser.cCCEscapeChar, uchar) +SIMP_PROP_GET(ParserDropTrailingLFOnReception, parser.bDropTrailingLF, int) +SIMP_PROP_GET(ParserEscapeControlCharactersOnReceive, parser.bEscapeCCOnRcv, int) +SIMP_PROP_GET(ParserSpaceLFOnReceive, parser.bSpaceLFOnRcv, int) +SIMP_PROP_GET(ParserEscape8BitCharactersOnReceive, parser.bEscape8BitChars, int) +SIMP_PROP_GET(ParserEscapeControlCharacterTab, parser.bEscapeTab, int) + +#undef SIMP_PROP +#undef SIMP_PROP_SET +#undef SIMP_PROP_GET /* return global input termination status * rgerhards, 2009-07-20 @@ -438,14 +400,120 @@ static rsRetVal setWorkDir(void __attribute__((unused)) *pVal, uchar *pNewVal) ABORT_FINALIZE(RS_RET_ERR_WRKDIR); } - free(pszWorkDir); - pszWorkDir = pNewVal; + free(loadConf->globals.pszWorkDir); + loadConf->globals.pszWorkDir = pNewVal; finalize_it: RETiRet; } +static rsRetVal +setDfltNetstrmDrvrCAF(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + FILE *fp; + free(loadConf->globals.pszDfltNetstrmDrvrCAF); + fp = fopen((const char*)pNewVal, "r"); + if(fp == NULL) { + LogError(errno, RS_RET_NO_FILE_ACCESS, + "error: defaultnetstreamdrivercafile file '%s' " + "could not be accessed", pNewVal); + } else { + fclose(fp); + loadConf->globals.pszDfltNetstrmDrvrCAF = pNewVal; + } + + RETiRet; +} + +static rsRetVal +setDfltNetstrmDrvrCertFile(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + FILE *fp; + + free(loadConf->globals.pszDfltNetstrmDrvrCertFile); + fp = fopen((const char*)pNewVal, "r"); + if(fp == NULL) { + LogError(errno, RS_RET_NO_FILE_ACCESS, + "error: defaultnetstreamdrivercertfile '%s' " + "could not be accessed", pNewVal); + } else { + fclose(fp); + loadConf->globals.pszDfltNetstrmDrvrCertFile = pNewVal; + } + + RETiRet; +} + +static rsRetVal +setDfltNetstrmDrvrKeyFile(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + FILE *fp; + + free(loadConf->globals.pszDfltNetstrmDrvrKeyFile); + fp = fopen((const char*)pNewVal, "r"); + if(fp == NULL) { + LogError(errno, RS_RET_NO_FILE_ACCESS, + "error: defaultnetstreamdriverkeyfile '%s' " + "could not be accessed", pNewVal); + } else { + fclose(fp); + loadConf->globals.pszDfltNetstrmDrvrKeyFile = pNewVal; + } + + RETiRet; +} + +static rsRetVal +setDfltNetstrmDrvr(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + free(loadConf->globals.pszDfltNetstrmDrvr); + loadConf->globals.pszDfltNetstrmDrvr = pNewVal; + RETiRet; +} + +static rsRetVal +setParserControlCharacterEscapePrefix(void __attribute__((unused)) *pVal, uchar *pNewVal) { + DEFiRet; + loadConf->globals.parser.cCCEscapeChar = *pNewVal; + RETiRet; +} + +static rsRetVal +setParserDropTrailingLFOnReception(void __attribute__((unused)) *pVal, int pNewVal) { + DEFiRet; + loadConf->globals.parser.bDropTrailingLF = pNewVal; + RETiRet; +} + +static rsRetVal +setParserEscapeControlCharactersOnReceive(void __attribute__((unused)) *pVal, int pNewVal) { + DEFiRet; + loadConf->globals.parser.bEscapeCCOnRcv = pNewVal; + RETiRet; +} + +static rsRetVal +setParserSpaceLFOnReceive(void __attribute__((unused)) *pVal, int pNewVal) { + DEFiRet; + loadConf->globals.parser.bSpaceLFOnRcv = pNewVal; + RETiRet; +} + +static rsRetVal +setParserEscape8BitCharactersOnReceive(void __attribute__((unused)) *pVal, int pNewVal) { + DEFiRet; + loadConf->globals.parser.bEscape8BitChars = pNewVal; + RETiRet; +} + +static rsRetVal +setParserEscapeControlCharacterTab(void __attribute__((unused)) *pVal, int pNewVal) { + DEFiRet; + loadConf->globals.parser.bEscapeTab = pNewVal; + RETiRet; +} + /* This function is used both by legacy and RainerScript conf. It is a real setter. */ static void setMaxLine(const int64_t iNew) @@ -454,13 +522,13 @@ setMaxLine(const int64_t iNew) LogError(0, RS_RET_INVALID_VALUE, "maxMessageSize tried to set " "to %lld, but cannot be less than 128 - set to 128 " "instead", (long long) iNew); - iMaxLine = 128; + loadConf->globals.iMaxLine = 128; } else if(iNew > (int64_t) INT_MAX) { LogError(0, RS_RET_INVALID_VALUE, "maxMessageSize larger than " "INT_MAX (%d) - reduced to INT_MAX", INT_MAX); - iMaxLine = INT_MAX; + loadConf->globals.iMaxLine = INT_MAX; } else { - iMaxLine = (int) iNew; + loadConf->globals.iMaxLine = (int) iNew; } } @@ -497,13 +565,13 @@ setOversizeMsgInputMode(const uchar *const mode) { DEFiRet; if(!strcmp((char*)mode, "truncate")) { - oversizeMsgInputMode = glblOversizeMsgInputMode_Truncate; + loadConf->globals.oversizeMsgInputMode = glblOversizeMsgInputMode_Truncate; } else if(!strcmp((char*)mode, "split")) { - oversizeMsgInputMode = glblOversizeMsgInputMode_Split; + loadConf->globals.oversizeMsgInputMode = glblOversizeMsgInputMode_Split; } else if(!strcmp((char*)mode, "accept")) { - oversizeMsgInputMode = glblOversizeMsgInputMode_Accept; + loadConf->globals.oversizeMsgInputMode = glblOversizeMsgInputMode_Accept; } else { - oversizeMsgInputMode = glblOversizeMsgInputMode_Truncate; + loadConf->globals.oversizeMsgInputMode = glblOversizeMsgInputMode_Truncate; } RETiRet; } @@ -513,11 +581,11 @@ setReportChildProcessExits(const uchar *const mode) { DEFiRet; if(!strcmp((char*)mode, "none")) { - reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_NONE; + loadConf->globals.reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_NONE; } else if(!strcmp((char*)mode, "errors")) { - reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; + loadConf->globals.reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; } else if(!strcmp((char*)mode, "all")) { - reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ALL; + loadConf->globals.reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ALL; } else { LogError(0, RS_RET_CONF_PARAM_INVLD, "invalid value '%s' for global parameter reportChildProcessExits -- ignored", @@ -527,57 +595,10 @@ setReportChildProcessExits(const uchar *const mode) RETiRet; } -static rsRetVal -setDisableDNS(int val) -{ - bDisableDNS = val; - return RS_RET_OK; -} - static int -getDisableDNS(void) +getDefPFFamily(rsconf_t *cnf) { - return bDisableDNS; -} - -static rsRetVal -setOption_DisallowWarning(int val) -{ - option_DisallowWarning = val; - return RS_RET_OK; -} - -static int -getOption_DisallowWarning(void) -{ - return option_DisallowWarning; -} - -static rsRetVal -setParseHOSTNAMEandTAG(int val) -{ - bParseHOSTNAMEandTAG = val; - return RS_RET_OK; -} - -static int -getParseHOSTNAMEandTAG(void) -{ - return bParseHOSTNAMEandTAG; -} - -static rsRetVal -setDefPFFamily(int level) -{ - DEFiRet; - iDefPFFamily = level; - RETiRet; -} - -static int -getDefPFFamily(void) -{ - return iDefPFFamily; + return cnf->globals.iDefPFFamily; } /* return our local IP. @@ -645,29 +666,29 @@ GetLocalHostName(void) /* return the name of the file where oversize messages are written to */ uchar* -glblGetOversizeMsgErrorFile(void) +glblGetOversizeMsgErrorFile(rsconf_t *cnf) { - return oversizeMsgErrorFile; + return cnf->globals.oversizeMsgErrorFile; } const uchar* -glblGetOperatingStateFile(void) +glblGetOperatingStateFile(rsconf_t *cnf) { - return operatingStateFile; + return cnf->globals.operatingStateFile; } /* return the mode with which oversize messages will be put forward */ int -glblGetOversizeMsgInputMode(void) +glblGetOversizeMsgInputMode(rsconf_t *cnf) { - return oversizeMsgInputMode; + return cnf->globals.oversizeMsgInputMode; } int -glblReportOversizeMessage(void) +glblReportOversizeMessage(rsconf_t *cnf) { - return reportOversizeMsg; + return cnf->globals.reportOversizeMsg; } @@ -675,12 +696,12 @@ glblReportOversizeMessage(void) * If name != NULL, prints it as the program name. */ void -glblReportChildProcessExit(const uchar *name, pid_t pid, int status) +glblReportChildProcessExit(rsconf_t *cnf, const uchar *name, pid_t pid, int status) { DBGPRINTF("waitpid for child %ld returned status: %2.2x\n", (long) pid, status); - if(reportChildProcessExits == REPORT_CHILD_PROCESS_EXITS_NONE - || (reportChildProcessExits == REPORT_CHILD_PROCESS_EXITS_ERRORS + if(cnf->globals.reportChildProcessExits == REPORT_CHILD_PROCESS_EXITS_NONE + || (cnf->globals.reportChildProcessExits == REPORT_CHILD_PROCESS_EXITS_ERRORS && WIFEXITED(status) && WEXITSTATUS(status) == 0)) { return; } @@ -816,52 +837,27 @@ GetLocalFQDNName(void) /* return the current working directory */ static uchar* -GetWorkDir(void) +GetWorkDir(rsconf_t *cnf) { - return(pszWorkDir == NULL ? (uchar*) "" : pszWorkDir); + return(cnf->globals.pszWorkDir == NULL ? (uchar*) "" : cnf->globals.pszWorkDir); } /* return the "raw" working directory, which means * NULL if unset. */ const uchar * -glblGetWorkDirRaw(void) +glblGetWorkDirRaw(rsconf_t *cnf) { - return pszWorkDir; + return cnf->globals.pszWorkDir; } /* return the current default netstream driver */ static uchar* -GetDfltNetstrmDrvr(void) -{ - return(pszDfltNetstrmDrvr == NULL ? DFLT_NETSTRM_DRVR : pszDfltNetstrmDrvr); -} - - -/* return the current default netstream driver CA File */ -static uchar* -GetDfltNetstrmDrvrCAF(void) -{ - return(pszDfltNetstrmDrvrCAF); -} - - -/* return the current default netstream driver key File */ -static uchar* -GetDfltNetstrmDrvrKeyFile(void) +GetDfltNetstrmDrvr(rsconf_t *cnf) { - return(pszDfltNetstrmDrvrKeyFile); + return(cnf->globals.pszDfltNetstrmDrvr == NULL ? DFLT_NETSTRM_DRVR : cnf->globals.pszDfltNetstrmDrvr); } - -/* return the current default netstream driver certificate File */ -static uchar* -GetDfltNetstrmDrvrCertFile(void) -{ - return(pszDfltNetstrmDrvrCertFile); -} - - /* [ar] Source IP for local client to be used on multihomed host */ static rsRetVal SetSourceIPofLocalClient(uchar *newname) @@ -901,15 +897,20 @@ CODESTARTobjQueryInterface(glbl) pIf->GetGlobalInputTermState = GetGlobalInputTermState; pIf->GetSourceIPofLocalClient = GetSourceIPofLocalClient; /* [ar] */ pIf->SetSourceIPofLocalClient = SetSourceIPofLocalClient; /* [ar] */ - pIf->SetDefPFFamily = setDefPFFamily; pIf->GetDefPFFamily = getDefPFFamily; - pIf->SetDisableDNS = setDisableDNS; - pIf->GetDisableDNS = getDisableDNS; + pIf->GetDisableDNS = GetDisableDNS; pIf->GetMaxLine = glblGetMaxLine; - pIf->SetOption_DisallowWarning = setOption_DisallowWarning; - pIf->GetOption_DisallowWarning = getOption_DisallowWarning; - pIf->SetParseHOSTNAMEandTAG = setParseHOSTNAMEandTAG; - pIf->GetParseHOSTNAMEandTAG = getParseHOSTNAMEandTAG; + pIf->GetOptionDisallowWarning = GetOptionDisallowWarning; + pIf->GetDfltNetstrmDrvrCAF = GetDfltNetstrmDrvrCAF; + pIf->GetDfltNetstrmDrvrCertFile = GetDfltNetstrmDrvrCertFile; + pIf->GetDfltNetstrmDrvrKeyFile = GetDfltNetstrmDrvrKeyFile; + pIf->GetDfltNetstrmDrvr = GetDfltNetstrmDrvr; + pIf->GetParserControlCharacterEscapePrefix = GetParserControlCharacterEscapePrefix; + pIf->GetParserDropTrailingLFOnReception = GetParserDropTrailingLFOnReception; + pIf->GetParserEscapeControlCharactersOnReceive = GetParserEscapeControlCharactersOnReceive; + pIf->GetParserSpaceLFOnReceive = GetParserSpaceLFOnReceive; + pIf->GetParserEscape8BitCharactersOnReceive = GetParserEscape8BitCharactersOnReceive; + pIf->GetParserEscapeControlCharacterTab = GetParserEscapeControlCharacterTab; #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; @@ -919,19 +920,8 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(LocalFQDNName) SIMP_PROP(LocalHostName) SIMP_PROP(LocalDomain) - SIMP_PROP(StripDomains) - SIMP_PROP(LocalHosts) - SIMP_PROP(ParserControlCharacterEscapePrefix) - SIMP_PROP(ParserDropTrailingLFOnReception) - SIMP_PROP(ParserEscapeControlCharactersOnReceive) - SIMP_PROP(ParserSpaceLFOnReceive) - SIMP_PROP(ParserEscape8BitCharactersOnReceive) - SIMP_PROP(ParserEscapeControlCharacterTab) SIMP_PROP(ParserEscapeControlCharactersCStyle) - SIMP_PROP(DfltNetstrmDrvr) - SIMP_PROP(DfltNetstrmDrvrCAF) - SIMP_PROP(DfltNetstrmDrvrKeyFile) - SIMP_PROP(DfltNetstrmDrvrCertFile) + SIMP_PROP(ParseHOSTNAMEandTAG) #ifdef USE_UNLIMITED_SELECT SIMP_PROP(FdSetSize) #endif @@ -944,35 +934,35 @@ ENDobjQueryInterface(glbl) */ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { - free(pszDfltNetstrmDrvr); - pszDfltNetstrmDrvr = NULL; - free(pszDfltNetstrmDrvrCAF); - pszDfltNetstrmDrvrCAF = NULL; - free(pszDfltNetstrmDrvrKeyFile); - pszDfltNetstrmDrvrKeyFile = NULL; - free(pszDfltNetstrmDrvrCertFile); - pszDfltNetstrmDrvrCertFile = NULL; + free(loadConf->globals.pszDfltNetstrmDrvr); + loadConf->globals.pszDfltNetstrmDrvr = NULL; + free(loadConf->globals.pszDfltNetstrmDrvrCAF); + loadConf->globals.pszDfltNetstrmDrvrCAF = NULL; + free(loadConf->globals.pszDfltNetstrmDrvrKeyFile); + loadConf->globals.pszDfltNetstrmDrvrKeyFile = NULL; + free(loadConf->globals.pszDfltNetstrmDrvrCertFile); + loadConf->globals.pszDfltNetstrmDrvrCertFile = NULL; free(LocalHostNameOverride); LocalHostNameOverride = NULL; - free(oversizeMsgErrorFile); - oversizeMsgErrorFile = NULL; - oversizeMsgInputMode = glblOversizeMsgInputMode_Accept; - reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; - free(pszWorkDir); - pszWorkDir = NULL; - free((void*)operatingStateFile); - operatingStateFile = NULL; - bDropMalPTRMsgs = 0; + free(loadConf->globals.oversizeMsgErrorFile); + loadConf->globals.oversizeMsgErrorFile = NULL; + loadConf->globals.oversizeMsgInputMode = glblOversizeMsgInputMode_Accept; + loadConf->globals.reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; + free(loadConf->globals.pszWorkDir); + loadConf->globals.pszWorkDir = NULL; + free((void*)loadConf->globals.operatingStateFile); + loadConf->globals.operatingStateFile = NULL; + loadConf->globals.bDropMalPTRMsgs = 0; bPreserveFQDN = 0; - iMaxLine = 8192; - cCCEscapeChar = '#'; - bDropTrailingLF = 1; - reportOversizeMsg = 1; - bEscapeCCOnRcv = 1; /* default is to escape control characters */ - bSpaceLFOnRcv = 0; - bEscape8BitChars = 0; /* default is not to escape control characters */ - bEscapeTab = 1; /* default is to escape tab characters */ - bParserEscapeCCCStyle = 0; + loadConf->globals.iMaxLine = 8192; + loadConf->globals.reportOversizeMsg = 1; + loadConf->globals.parser.cCCEscapeChar = '#'; + loadConf->globals.parser.bDropTrailingLF = 1; + loadConf->globals.parser.bEscapeCCOnRcv = 1; /* default is to escape control characters */ + loadConf->globals.parser.bSpaceLFOnRcv = 0; + loadConf->globals.parser.bEscape8BitChars = 0; /* default is not to escape control characters */ + loadConf->globals.parser.bEscapeTab = 1; /* default is to escape tab characters */ + loadConf->globals.parser.bParserEscapeCCCStyle = 0; #ifdef USE_UNLIMITED_SELECT iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); #endif @@ -1166,10 +1156,10 @@ glblProcessCnf(struct cnfobj *o) if(!cnfparamvals[i].bUsed) continue; if(!strcmp(paramblk.descr[i].name, "processinternalmessages")) { - bProcessInternalMessages = (int) cnfparamvals[i].val.d.n; + loadConf->globals.bProcessInternalMessages = (int) cnfparamvals[i].val.d.n; cnfparamvals[i].bUsed = TRUE; } else if(!strcmp(paramblk.descr[i].name, "internal.developeronly.options")) { - glblDevOptions = (uint64_t) cnfparamvals[i].val.d.n; + loadConf->globals.glblDevOptions = (uint64_t) cnfparamvals[i].val.d.n; cnfparamvals[i].bUsed = TRUE; } else if(!strcmp(paramblk.descr[i].name, "stdlog.channelspec")) { #ifndef ENABLE_LIBLOGGING_STDLOG @@ -1178,20 +1168,21 @@ glblProcessCnf(struct cnfobj *o) "The 'stdlog.channelspec' parameter " "is ignored. Note: the syslog API is used instead.\n"); #else - stdlog_chanspec = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + loadConf->globals.stdlog_chanspec = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); /* we need to re-open with the new channel */ - stdlog_close(stdlog_hdl); - stdlog_hdl = stdlog_open("rsyslogd", 0, STDLOG_SYSLOG, - (char*) stdlog_chanspec); + stdlog_close(loadConf->globals.stdlog_hdl); + loadConf->globals.stdlog_hdl = stdlog_open("rsyslogd", 0, STDLOG_SYSLOG, + (char*) loadConf->globals.stdlog_chanspec); cnfparamvals[i].bUsed = TRUE; #endif } else if(!strcmp(paramblk.descr[i].name, "operatingstatefile")) { - if(operatingStateFile != NULL) { + if(loadConf->globals.operatingStateFile != NULL) { LogError(errno, RS_RET_PARAM_ERROR, "error: operatingStateFile already set to '%s' - " - "new value ignored", operatingStateFile); + "new value ignored", loadConf->globals.operatingStateFile); } else { - operatingStateFile = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + loadConf->globals.operatingStateFile = + (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); osf_open(); } } else if(!strcmp(paramblk.descr[i].name, "security.abortonidresolutionfail")) { @@ -1294,7 +1285,6 @@ glblDoneLoadCnf(void) { int i; unsigned char *cstr; - FILE *fp; DEFiRet; CHKiRet(objUse(net, CORE_COMPONENT)); @@ -1323,61 +1313,33 @@ glblDoneLoadCnf(void) LocalHostNameOverride = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); } else if(!strcmp(paramblk.descr[i].name, "defaultnetstreamdriverkeyfile")) { - free(pszDfltNetstrmDrvrKeyFile); - uchar *const fn = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); - fp = fopen((const char*)fn, "r"); - if(fp == NULL) { - LogError(errno, RS_RET_NO_FILE_ACCESS, - "error: defaultnetstreamdriverkeyfile '%s' " - "could not be accessed", fn); - } else { - fclose(fp); - pszDfltNetstrmDrvrKeyFile = fn; - } + cstr = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + setDfltNetstrmDrvrKeyFile(NULL, cstr); } else if(!strcmp(paramblk.descr[i].name, "defaultnetstreamdrivercertfile")) { - free(pszDfltNetstrmDrvrCertFile); - uchar *const fn = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); - fp = fopen((const char*)fn, "r"); - if(fp == NULL) { - LogError(errno, RS_RET_NO_FILE_ACCESS, - "error: defaultnetstreamdrivercertfile '%s' " - "could not be accessed", fn); - } else { - fclose(fp); - pszDfltNetstrmDrvrCertFile = fn; - } + cstr = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + setDfltNetstrmDrvrCertFile(NULL, cstr); } else if(!strcmp(paramblk.descr[i].name, "defaultnetstreamdrivercafile")) { - free(pszDfltNetstrmDrvrCAF); - uchar *const fn = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); - fp = fopen((const char*)fn, "r"); - if(fp == NULL) { - LogError(errno, RS_RET_NO_FILE_ACCESS, - "error: defaultnetstreamdrivercafile file '%s' " - "could not be accessed", fn); - } else { - fclose(fp); - pszDfltNetstrmDrvrCAF = fn; - } + cstr = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + setDfltNetstrmDrvrCAF(NULL, cstr); } else if(!strcmp(paramblk.descr[i].name, "defaultnetstreamdriver")) { - free(pszDfltNetstrmDrvr); - pszDfltNetstrmDrvr = (uchar*) - es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + cstr = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + setDfltNetstrmDrvr(NULL, cstr); } else if(!strcmp(paramblk.descr[i].name, "preservefqdn")) { bPreserveFQDN = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "dropmsgswithmaliciousdnsptrrecords")) { - bDropMalPTRMsgs = (int) cnfparamvals[i].val.d.n; + loadConf->globals.bDropMalPTRMsgs = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "action.reportsuspension")) { - bActionReportSuspension = (int) cnfparamvals[i].val.d.n; + loadConf->globals.bActionReportSuspension = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "action.reportsuspensioncontinuation")) { - bActionReportSuspensionCont = (int) cnfparamvals[i].val.d.n; + loadConf->globals.bActionReportSuspensionCont = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "maxmessagesize")) { setMaxLine(cnfparamvals[i].val.d.n); } else if(!strcmp(paramblk.descr[i].name, "oversizemsg.errorfile")) { - free(oversizeMsgErrorFile); - oversizeMsgErrorFile = (uchar*)es_str2cstr(cnfparamvals[i].val.d.estr, NULL); + free(loadConf->globals.oversizeMsgErrorFile); + loadConf->globals.oversizeMsgErrorFile = (uchar*)es_str2cstr(cnfparamvals[i].val.d.estr, NULL); } else if(!strcmp(paramblk.descr[i].name, "oversizemsg.report")) { - reportOversizeMsg = (int) cnfparamvals[i].val.d.n; + loadConf->globals.reportOversizeMsg = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "oversizemsg.input.mode")) { const char *const tmp = es_str2cstr(cnfparamvals[i].val.d.estr, NULL); setOversizeMsgInputMode((uchar*) tmp); @@ -1387,32 +1349,39 @@ glblDoneLoadCnf(void) setReportChildProcessExits((uchar*) tmp); free((void*)tmp); } else if(!strcmp(paramblk.descr[i].name, "debug.onshutdown")) { - glblDebugOnShutdown = (int) cnfparamvals[i].val.d.n; - LogError(0, RS_RET_OK, "debug: onShutdown set to %d", glblDebugOnShutdown); + loadConf->globals.debugOnShutdown = (int) cnfparamvals[i].val.d.n; + LogError(0, RS_RET_OK, "debug: onShutdown set to %d", loadConf->globals.debugOnShutdown); } else if(!strcmp(paramblk.descr[i].name, "debug.gnutls")) { - iGnuTLSLoglevel = (int) cnfparamvals[i].val.d.n; + loadConf->globals.iGnuTLSLoglevel = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "debug.unloadmodules")) { glblUnloadModules = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "parser.controlcharacterescapeprefix")) { uchar* tmp = (uchar*) es_str2cstr(cnfparamvals[i].val.d.estr, NULL); - cCCEscapeChar = tmp[0]; + setParserControlCharacterEscapePrefix(NULL, tmp); free(tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.droptrailinglfonreception")) { - bDropTrailingLF = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + setParserDropTrailingLFOnReception(NULL, tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.escapecontrolcharactersonreceive")) { - bEscapeCCOnRcv = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + setParserEscapeControlCharactersOnReceive(NULL, tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.spacelfonreceive")) { - bSpaceLFOnRcv = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + setParserSpaceLFOnReceive(NULL, tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.escape8bitcharactersonreceive")) { - bEscape8BitChars = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + setParserEscape8BitCharactersOnReceive(NULL, tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.escapecontrolcharactertab")) { - bEscapeTab = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + setParserEscapeControlCharacterTab(NULL, tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.escapecontrolcharacterscstyle")) { - bParserEscapeCCCStyle = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + SetParserEscapeControlCharactersCStyle(tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.parsehostnameandtag")) { - bParseHOSTNAMEandTAG = (int) cnfparamvals[i].val.d.n; + const int tmp = (int) cnfparamvals[i].val.d.n; + SetParseHOSTNAMEandTAG(tmp); } else if(!strcmp(paramblk.descr[i].name, "parser.permitslashinprogramname")) { - bPermitSlashInProgramname = (int) cnfparamvals[i].val.d.n; + loadConf->globals.parser.bPermitSlashInProgramname = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "debug.logfile")) { if(pszAltDbgFileName == NULL) { pszAltDbgFileName = es_str2cstr(cnfparamvals[i].val.d.estr, NULL); @@ -1428,30 +1397,30 @@ glblDoneLoadCnf(void) } LogError(0, RS_RET_OK, "debug log file is '%s', fd %d", pszAltDbgFileName, altdbg); } else if(!strcmp(paramblk.descr[i].name, "janitor.interval")) { - janitorInterval = (int) cnfparamvals[i].val.d.n; + loadConf->globals.janitorInterval = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "net.ipprotocol")) { char *proto = es_str2cstr(cnfparamvals[i].val.d.estr, NULL); if(!strcmp(proto, "unspecified")) { - iDefPFFamily = PF_UNSPEC; + loadConf->globals.iDefPFFamily = PF_UNSPEC; } else if(!strcmp(proto, "ipv4-only")) { - iDefPFFamily = PF_INET; + loadConf->globals.iDefPFFamily = PF_INET; } else if(!strcmp(proto, "ipv6-only")) { - iDefPFFamily = PF_INET6; + loadConf->globals.iDefPFFamily = PF_INET6; } else{ LogError(0, RS_RET_ERR, "invalid net.ipprotocol " "parameter '%s' -- ignored", proto); } free(proto); } else if(!strcmp(paramblk.descr[i].name, "senders.reportnew")) { - glblReportNewSenders = (int) cnfparamvals[i].val.d.n; + loadConf->globals.reportNewSenders = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "senders.reportgoneaway")) { - glblReportGoneAwaySenders = (int) cnfparamvals[i].val.d.n; + loadConf->globals.reportGoneAwaySenders = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "senders.timeoutafter")) { - glblSenderStatsTimeout = (int) cnfparamvals[i].val.d.n; + loadConf->globals.senderStatsTimeout = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "senders.keeptrack")) { - glblSenderKeepTrack = (int) cnfparamvals[i].val.d.n; + loadConf->globals.senderKeepTrack = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "inputs.timeout.shutdown")) { - glblInputTimeoutShutdown = (int) cnfparamvals[i].val.d.n; + loadConf->globals.inputTimeoutShutdown = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "privdrop.group.keepsupplemental")) { loadConf->globals.gidDropPrivKeepSupplemental = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "privdrop.group.id")) { @@ -1465,24 +1434,25 @@ glblDoneLoadCnf(void) } else if(!strcmp(paramblk.descr[i].name, "security.abortonidresolutionfail")) { loadConf->globals.abortOnIDResolutionFail = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "net.acladdhostnameonfail")) { - *(net.pACLAddHostnameOnFail) = (int) cnfparamvals[i].val.d.n; + loadConf->globals.ACLAddHostnameOnFail = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "net.aclresolvehostname")) { - *(net.pACLDontResolve) = !((int) cnfparamvals[i].val.d.n); + loadConf->globals.ACLDontResolve = !((int) cnfparamvals[i].val.d.n); } else if(!strcmp(paramblk.descr[i].name, "net.enabledns")) { - setDisableDNS(!((int) cnfparamvals[i].val.d.n)); + SetDisableDNS(!((int) cnfparamvals[i].val.d.n)); } else if(!strcmp(paramblk.descr[i].name, "net.permitwarning")) { - setOption_DisallowWarning(!((int) cnfparamvals[i].val.d.n)); + SetOptionDisallowWarning(!((int) cnfparamvals[i].val.d.n)); } else if(!strcmp(paramblk.descr[i].name, "abortonuncleanconfig")) { loadConf->globals.bAbortOnUncleanConfig = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "internalmsg.ratelimit.burst")) { - glblIntMsgRateLimitBurst = (int) cnfparamvals[i].val.d.n; + loadConf->globals.intMsgRateLimitBurst = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "internalmsg.ratelimit.interval")) { - glblIntMsgRateLimitItv = (int) cnfparamvals[i].val.d.n; + loadConf->globals.intMsgRateLimitItv = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "internalmsg.severity")) { - glblIntMsgsSeverityFilter = (int) cnfparamvals[i].val.d.n; - if((glblIntMsgsSeverityFilter < 0) || (glblIntMsgsSeverityFilter > 7)) { + loadConf->globals.intMsgsSeverityFilter = (int) cnfparamvals[i].val.d.n; + if((loadConf->globals.intMsgsSeverityFilter < 0) || + (loadConf->globals.intMsgsSeverityFilter > 7)) { parser_errmsg("invalid internalmsg.severity value"); - glblIntMsgsSeverityFilter = DFLT_INT_MSGS_SEV_FILTER; + loadConf->globals.intMsgsSeverityFilter = DFLT_INT_MSGS_SEV_FILTER; } } else if(!strcmp(paramblk.descr[i].name, "environment")) { for(int j = 0 ; j < cnfparamvals[i].val.d.ar->nmemb ; ++j) { @@ -1503,40 +1473,40 @@ glblDoneLoadCnf(void) } else if(!strcmp(paramblk.descr[i].name, "debug.whitelist")) { glblDbgWhitelist = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "shutdown.queue.doublesize")) { - glblShutdownQueueDoubleSize = (int) cnfparamvals[i].val.d.n; + loadConf->globals.shutdownQueueDoubleSize = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "umask")) { loadConf->globals.umask = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "shutdown.enable.ctlc")) { - glblPermitCtlC = (int) cnfparamvals[i].val.d.n; + loadConf->globals.permitCtlC = (int) cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.action.queue.timeoutshutdown")) { - actq_dflt_toQShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.actq_dflt_toQShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.action.queue.timeoutactioncompletion")) { - actq_dflt_toActShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.actq_dflt_toActShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.action.queue.timeoutenqueue")) { - actq_dflt_toEnq = cnfparamvals[i].val.d.n; + loadConf->globals.actq_dflt_toEnq = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.action.queue.timeoutworkerthreadshutdown")) { - actq_dflt_toWrkShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.actq_dflt_toWrkShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.ruleset.queue.timeoutshutdown")) { - ruleset_dflt_toQShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.ruleset_dflt_toQShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.ruleset.queue.timeoutactioncompletion")) { - ruleset_dflt_toActShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.ruleset_dflt_toActShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.ruleset.queue.timeoutenqueue")) { - ruleset_dflt_toEnq = cnfparamvals[i].val.d.n; + loadConf->globals.ruleset_dflt_toEnq = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "default.ruleset.queue.timeoutworkerthreadshutdown")) { - ruleset_dflt_toWrkShutdown = cnfparamvals[i].val.d.n; + loadConf->globals.ruleset_dflt_toWrkShutdown = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "reverselookup.cache.ttl.default")) { - dnscacheDefaultTTL = cnfparamvals[i].val.d.n; + loadConf->globals.dnscacheDefaultTTL = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "reverselookup.cache.ttl.enable")) { - dnscacheEnableTTL = cnfparamvals[i].val.d.n; + loadConf->globals.dnscacheEnableTTL = cnfparamvals[i].val.d.n; } else if(!strcmp(paramblk.descr[i].name, "parser.supportcompressionextension")) { - bSupportCompressionExtension = cnfparamvals[i].val.d.n; + loadConf->globals.bSupportCompressionExtension = cnfparamvals[i].val.d.n; } else { dbgprintf("glblDoneLoadCnf: program error, non-handled " "param '%s'\n", paramblk.descr[i].name); } } - if(glblDebugOnShutdown && Debug != DEBUG_FULL) { + if(loadConf->globals.debugOnShutdown && Debug != DEBUG_FULL) { Debug = DEBUG_ONDEMAND; stddbg = -1; } @@ -1560,16 +1530,16 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"debugfile", 0, eCmdHdlrGetWord, setDebugFile, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debuglevel", 0, eCmdHdlrInt, setDebugLevel, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, setWorkDir, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, NULL, - &bDropMalPTRMsgs, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvr, + CHKiRet(regCfSysLineHdlr((uchar *)"dropmsgswithmaliciousdnsptrrecords", 0, eCmdHdlrBinary, SetDropMalPTRMsgs, + NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriver", 0, eCmdHdlrGetWord, setDfltNetstrmDrvr, NULL, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, - &pszDfltNetstrmDrvrCAF, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, - &pszDfltNetstrmDrvrKeyFile, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, - &pszDfltNetstrmDrvrCertFile, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, + setDfltNetstrmDrvrCAF, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, + setDfltNetstrmDrvrKeyFile, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, + setDfltNetstrmDrvrCertFile, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"localhostname", 0, eCmdHdlrGetWord, NULL, &LocalHostNameOverride, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"localhostipif", 0, eCmdHdlrGetWord, setLocalHostIPIF, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrGoneAway, NULL, NULL, NULL)); @@ -1577,16 +1547,18 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, legacySetMaxMessageSize, NULL, NULL)); /* Deprecated parser config options */ - CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, - &cCCEscapeChar, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, - &bDropTrailingLF, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, - &bEscapeCCOnRcv, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"spacelfonreceive", 0, eCmdHdlrBinary, NULL, &bSpaceLFOnRcv, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, - &bEscape8BitChars, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, + setParserControlCharacterEscapePrefix, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, + setParserDropTrailingLFOnReception, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, + setParserEscapeControlCharactersOnReceive, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"spacelfonreceive", 0, eCmdHdlrBinary, + setParserSpaceLFOnReceive, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, + setParserEscape8BitCharactersOnReceive, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, + setParserEscapeControlCharacterTab, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); @@ -1599,15 +1571,9 @@ ENDObjClassInit(glbl) * rgerhards, 2008-04-17 */ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ - free(pszDfltNetstrmDrvr); - free(pszDfltNetstrmDrvrCAF); - free(pszDfltNetstrmDrvrKeyFile); - free(pszDfltNetstrmDrvrCertFile); - free(pszWorkDir); free(LocalDomain); free(LocalHostName); free(LocalHostNameOverride); - free(oversizeMsgErrorFile); free(LocalFQDNName); freeTimezoneInfo(); objRelease(prop, CORE_COMPONENT); diff --git a/runtime/glbl.h b/runtime/glbl.h index 9ccf7b6467..f90b426fb3 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -43,42 +43,20 @@ #define glblOversizeMsgInputMode_Accept 2 extern pid_t glbl_ourpid; -extern int bProcessInternalMessages; -extern int bPermitSlashInProgramname; -#ifdef ENABLE_LIBLOGGING_STDLOG -extern stdlog_channel_t stdlog_hdl; -#endif /* interfaces */ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ - uchar* (*GetWorkDir)(void); - int (*GetMaxLine)(void); + uchar* (*GetWorkDir)(rsconf_t *cnf); + int (*GetMaxLine)(rsconf_t *cnf); #define SIMP_PROP(name, dataType) \ dataType (*Get##name)(void); \ rsRetVal (*Set##name)(dataType); SIMP_PROP(OptimizeUniProc, int) SIMP_PROP(PreserveFQDN, int) - SIMP_PROP(DefPFFamily, int) - SIMP_PROP(DropMalPTRMsgs, int) - SIMP_PROP(Option_DisallowWarning, int) - SIMP_PROP(DisableDNS, int) SIMP_PROP(LocalFQDNName, uchar*) SIMP_PROP(mainqCnfObj, struct cnfobj*) SIMP_PROP(LocalHostName, uchar*) SIMP_PROP(LocalDomain, uchar*) - SIMP_PROP(StripDomains, char**) - SIMP_PROP(LocalHosts, char**) - SIMP_PROP(DfltNetstrmDrvr, uchar*) - SIMP_PROP(DfltNetstrmDrvrCAF, uchar*) - SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*) - SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) - SIMP_PROP(ParserControlCharacterEscapePrefix, uchar) - SIMP_PROP(ParserDropTrailingLFOnReception, int) - SIMP_PROP(ParserEscapeControlCharactersOnReceive, int) - SIMP_PROP(ParserSpaceLFOnReceive, int) - SIMP_PROP(ParserEscape8BitCharactersOnReceive, int) - SIMP_PROP(ParserEscapeControlCharacterTab, int) - SIMP_PROP(ParserEscapeControlCharactersCStyle, int) /* added v3, 2009-06-30 */ rsRetVal (*GenerateLocalHostNameProperty)(void); prop_t* (*GetLocalHostNameProp)(void); @@ -86,7 +64,6 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ int (*GetGlobalInputTermState)(void); void (*SetGlobalInputTermination)(void); /* added v5, 2009-11-03 */ - SIMP_PROP(ParseHOSTNAMEandTAG, int) /* note: v4, v5 are already used by more recent versions, so we need to skip them! */ /* added v6, 2009-11-16 as part of varmojfekoj's "unlimited select()" patch * Note that it must be always present, otherwise the interface would have different @@ -103,31 +80,47 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ uchar* (*GetSourceIPofLocalClient)(void); /* [ar] */ rsRetVal (*SetSourceIPofLocalClient)(uchar*); /* [ar] */ /* v9 - 2015-01-12 SetMaxLine method removed */ + /* v10 - global variables should be moved to the rsconf_t data structure, so + * dynamic configuration reload can be introduced. This is why each getter needs additional + * parameter specifying a configuration it belongs to(either loadConf or runConf) + */ +#undef SIMP_PROP +#define SIMP_PROP(name, dataType) \ + dataType (*Get##name)(rsconf_t *cnf); \ + rsRetVal (*Set##name)(dataType); + + SIMP_PROP(DropMalPTRMsgs, int) + SIMP_PROP(DfltNetstrmDrvrCAF, uchar*) + SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) + SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*) + SIMP_PROP(DfltNetstrmDrvr, uchar*) + SIMP_PROP(DefPFFamily, int) + SIMP_PROP(DisableDNS, int) + SIMP_PROP(ParserControlCharacterEscapePrefix, uchar) + SIMP_PROP(ParserDropTrailingLFOnReception, int) + SIMP_PROP(ParserEscapeControlCharactersOnReceive, int) + SIMP_PROP(ParserSpaceLFOnReceive, int) + SIMP_PROP(ParserEscape8BitCharactersOnReceive, int) + SIMP_PROP(ParserEscapeControlCharacterTab, int) + SIMP_PROP(ParserEscapeControlCharactersCStyle, int) + SIMP_PROP(ParseHOSTNAMEandTAG, int) + SIMP_PROP(OptionDisallowWarning, int) + #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 10 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ PROTOTYPEObj(glbl); -extern int glblDebugOnShutdown; /* start debug log when we are shut down */ -extern int glblReportNewSenders; -extern int glblReportGoneAwaySenders; -extern int glblSenderStatsTimeout; -extern int glblSenderKeepTrack; extern int glblUnloadModules; extern short janitorInterval; -extern int glblIntMsgRateLimitItv; -extern int glblIntMsgRateLimitBurst; extern char** glblDbgFiles; extern size_t glblDbgFilesNum; extern int glblDbgWhitelist; extern int glblPermitCtlC; -extern int glblInputTimeoutShutdown; -extern int glblIntMsgsSeverityFilter; extern int bTerminateInputs; -extern int glblShutdownQueueDoubleSize; #ifndef HAVE_ATOMIC_BUILTINS extern DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); #endif @@ -141,7 +134,6 @@ extern DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); */ #define DEV_OPTION_KEEP_RUNNING_ON_HARD_CONF_ERROR 1 #define DEV_OPTION_8_1905_HANG_TEST 2 // TODO: remove - temporary for bughunt -extern uint64_t glblDevOptions; #define glblGetOurPid() glbl_ourpid #define glblSetOurPid(pid) { glbl_ourpid = (pid); } @@ -152,15 +144,15 @@ void glblProcessTimezone(struct cnfobj *o); void glblProcessMainQCnf(struct cnfobj *o); void glblDestructMainqCnfObj(void); rsRetVal glblDoneLoadCnf(void); -const uchar * glblGetWorkDirRaw(void); +const uchar * glblGetWorkDirRaw(rsconf_t *cnf); tzinfo_t* glblFindTimezoneInfo(char *id); -int GetGnuTLSLoglevel(void); -int glblGetMaxLine(void); +int GetGnuTLSLoglevel(rsconf_t *cnf); +int glblGetMaxLine(rsconf_t *cnf); int bs_arrcmp_glblDbgFiles(const void *s1, const void *s2); -uchar* glblGetOversizeMsgErrorFile(void); -const uchar* glblGetOperatingStateFile(void); -int glblGetOversizeMsgInputMode(void); -int glblReportOversizeMessage(void); -void glblReportChildProcessExit(const uchar *name, pid_t pid, int status); +uchar* glblGetOversizeMsgErrorFile(rsconf_t *cnf); +const uchar* glblGetOperatingStateFile(rsconf_t *cnf); +int glblGetOversizeMsgInputMode(rsconf_t *cnf); +int glblReportOversizeMessage(rsconf_t *cnf); +void glblReportChildProcessExit(rsconf_t *cnf, const uchar *name, pid_t pid, int status); #endif /* #ifndef GLBL_H_INCLUDED */ diff --git a/runtime/msg.c b/runtime/msg.c index ee0d757352..87e7e6703a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1261,7 +1261,7 @@ static rsRetVal MsgSerialize(smsg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pCSAPPNAME, CSTR); objSerializePTR(pStrm, pCSPROCID, CSTR); objSerializePTR(pStrm, pCSMSGID, CSTR); - + objSerializePTR(pStrm, pszUUID, PSZ); if(pThis->pRuleset != NULL) { @@ -1514,7 +1514,7 @@ static rsRetVal aquirePROCIDFromTAG(smsg_t * const pM) ++i; if(!(i < pM->iLenTAG)) return RS_RET_OK; /* no [, so can not emulate... */ - + ++i; /* skip '[' */ /* now obtain the PROCID string... */ @@ -1569,7 +1569,7 @@ aquireProgramName(smsg_t * const pM) ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) && (pszTag[i] != '\0') && (pszTag[i] != ':') && (pszTag[i] != '[') - && (bPermitSlashInProgramname || (pszTag[i] != '/')) + && (runConf->globals.parser.bPermitSlashInProgramname || (pszTag[i] != '/')) ; ++i) ; /* just search end of PROGNAME */ if(i < CONF_PROGNAME_BUFSIZE) { @@ -2492,7 +2492,7 @@ tryEmulateTAG(smsg_t *const pM, const sbool bLockMutex) MsgUnlock(pM); return; /* done, no need to emulate */ } - + if(msgGetProtocolVersion(pM) == 1) { if(!strcmp(getPROCID(pM, MUTEX_ALREADY_LOCKED), "-")) { /* no process ID, use APP-NAME only */ @@ -2905,7 +2905,7 @@ void ATTR_NONNULL() MsgTruncateToMaxSize(smsg_t *const pThis) { ISOBJ_TYPE_assert(pThis, msg); - const int maxMsgSize = glblGetMaxLine(); + const int maxMsgSize = glblGetMaxLine(runConf); assert(pThis->iLenRawMsg > maxMsgSize); const int deltaSize = pThis->iLenRawMsg - maxMsgSize; @@ -3914,12 +3914,12 @@ uchar *MsgGetProp(smsg_t *__restrict__ const pMsg, struct templateEntry *__restr *pPropLen = (bufLen == -1) ? (int) ustrlen(pRes) : bufLen; return pRes; } - + /* Now check if we need to make "temporary" transformations (these * are transformations that do not go back into the message - * memory must be allocated for them!). */ - + /* substring extraction */ /* first we check if we need to extract by field number * rgerhards, 2005-12-22 diff --git a/runtime/net.c b/runtime/net.c index d98395a77c..2818193cc1 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -62,6 +62,7 @@ #include "net.h" #include "dnscache.h" #include "prop.h" +#include "rsconf.h" #ifdef OS_SOLARIS #include @@ -95,10 +96,6 @@ struct AllowedSenders *pAllowedSenders_GSS = NULL; static struct AllowedSenders *pLastAllowedSenders_GSS = NULL; #endif -int ACLAddHostnameOnFail = 0; /* add hostname to acl when DNS resolving has failed */ -int ACLDontResolve = 0; /* add hostname to acl instead of resolving it to IP(s) */ - - /* ------------------------------ begin permitted peers code ------------------------------ */ @@ -494,10 +491,10 @@ PermittedPeerWildcardMatch(permittedPeers_t *const pPeer, */ static void MaskIP6 (struct in6_addr *addr, uint8_t bits) { register uint8_t i; - + assert (addr != NULL); assert (bits <= 128); - + i = bits/32; if (bits%32) addr->s6_addr32[i++] &= htonl(0xffffffff << (32 - (bits % 32))); @@ -506,10 +503,10 @@ static void MaskIP6 (struct in6_addr *addr, uint8_t bits) { } static void MaskIP4 (struct in_addr *addr, uint8_t bits) { - + assert (addr != NULL); assert (bits <=32 ); - + addr->s_addr &= htonl(0xffffffff << (32 - bits)); } @@ -556,11 +553,11 @@ static rsRetVal AddAllowedSenderEntry(struct AllowedSenders **ppRoot, struct All if((pEntry = (struct AllowedSenders*) calloc(1, sizeof(struct AllowedSenders))) == NULL) { return RS_RET_OUT_OF_MEMORY; /* no options left :( */ } - + memcpy(&(pEntry->allowedSender), iAllow, sizeof (struct NetAddr)); pEntry->pNext = NULL; pEntry->SignificantBits = iSignificantBits; - + /* enqueue */ if(*ppRoot == NULL) { *ppRoot = pEntry; @@ -568,7 +565,7 @@ static rsRetVal AddAllowedSenderEntry(struct AllowedSenders **ppRoot, struct All (*ppLast)->pNext = pEntry; } *ppLast = pEntry; - + return RS_RET_OK; } @@ -585,7 +582,7 @@ clearAllowedSenders(uchar *pszType) if(setAllowRoot(&pCurr, pszType) != RS_RET_OK) return; /* if something went wrong, so let's leave */ - + while(pCurr != NULL) { pPrev = pCurr; pCurr = pCurr->pNext; @@ -668,14 +665,14 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iRet = AddAllowedSenderEntry(ppRoot, ppLast, iAllow, iSignificantBits); } else { /* we need to process a hostname ACL */ - if(glbl.GetDisableDNS()) { + if(glbl.GetDisableDNS(loadConf)) { LogError(0, NO_ERRCODE, "Ignoring hostname based ACLs because DNS is disabled."); ABORT_FINALIZE(RS_RET_OK); } if (!strchr (iAllow->addr.HostWildcard, '*') && !strchr (iAllow->addr.HostWildcard, '?') && - ACLDontResolve == 0) { + loadConf->globals.ACLDontResolve == 0) { /* single host - in this case, we pull its IP addresses from DNS * and add IP-based ACLs. */ @@ -692,7 +689,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS if (getaddrinfo (iAllow->addr.HostWildcard, NULL, &hints, &res) != 0) { LogError(0, NO_ERRCODE, "DNS error: Can't resolve \"%s\"", iAllow->addr.HostWildcard); - if (ACLAddHostnameOnFail) { + if (loadConf->globals.ACLAddHostnameOnFail) { LogError(0, NO_ERRCODE, "Adding hostname \"%s\" to ACL as a wildcard " "entry.", iAllow->addr.HostWildcard); iRet = AddAllowedSenderEntry(ppRoot, ppLast, iAllow, iSignificantBits); @@ -799,7 +796,7 @@ PrintAllowedSenders(int iListToPrint) #define iListToPrint_MAX 2 #endif assert((iListToPrint > 0) && (iListToPrint <= iListToPrint_MAX)); - + dbgprintf("Allowed %s Senders:\n", SENDER_TEXT[iListToPrint]); pSender = (iListToPrint == 1) ? pAllowedSenders_UDP : @@ -1006,13 +1003,13 @@ static int isAllowedSender2(uchar *pszType, struct sockaddr *pFrom, const char * int ret; assert(pFrom != NULL); - + if(setAllowRoot(&pAllowRoot, pszType) != RS_RET_OK) return 0; /* if something went wrong, we deny access - that's the better choice... */ if(pAllowRoot == NULL) return 1; /* checking disabled, everything is valid! */ - + /* now we loop through the list of allowed senders. As soon as * we find a match, we return back (indicating allowed). We loop * until we are out of allowed senders. If so, we fall through the @@ -1186,7 +1183,7 @@ getLocalHostname(uchar **ppName) char *dot = strstr(hnbuf, "."); struct addrinfo *res = NULL; - if(!empty_hostname && dot == NULL && !glbl.GetDisableDNS()) { + if(!empty_hostname && dot == NULL && runConf != NULL && !glbl.GetDisableDNS(runConf)) { /* we need to (try) to find the real name via resolver */ struct addrinfo flags; memset(&flags, 0, sizeof(flags)); @@ -1475,7 +1472,7 @@ create_udp_socket(uchar *hostname, hints.ai_flags = AI_PASSIVE | AI_NUMERICSERV; else hints.ai_flags = AI_NUMERICSERV; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runConf); hints.ai_socktype = SOCK_DGRAM; # if defined (_AIX) /* AIXPORT : SOCK_DGRAM has the protocol IPPROTO_UDP @@ -1690,9 +1687,6 @@ CODESTARTobjQueryInterface(net) pIf->CmpHost = CmpHost; pIf->HasRestrictions = HasRestrictions; pIf->GetIFIPAddr = getIFIPAddr; - /* data members */ - pIf->pACLAddHostnameOnFail = &ACLAddHostnameOnFail; - pIf->pACLDontResolve = &ACLDontResolve; finalize_it: ENDobjQueryInterface(net) diff --git a/runtime/net.h b/runtime/net.h index 43ee04f4a3..c2847f65f0 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -165,13 +165,11 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ int (*isAllowedSender2)(uchar *pszType, struct sockaddr *pFrom, const char *pszFromHost, int bChkDNS); /* v7 interface additions - 2012-03-06 */ rsRetVal (*GetIFIPAddr)(uchar *szif, int family, uchar *pszbuf, int lenBuf); - /* data members - these should go away over time... TODO */ - int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ - int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ /* v8 cvthname() signature change -- rgerhards, 2013-01-18 */ /* v9 create_udp_socket() signature change -- dsahern, 2016-11-11 */ + /* v10 moved data members to rsconf_t -- alakatos, 2021-12-29 */ ENDinterface(net) -#define netCURR_IF_VERSION 9 /* increment whenever you change the interface structure! */ +#define netCURR_IF_VERSION 10 /* increment whenever you change the interface structure! */ /* prototypes */ PROTOTYPEObj(net); diff --git a/runtime/netstrms.c b/runtime/netstrms.c index caf57242cd..387021d3f8 100644 --- a/runtime/netstrms.c +++ b/runtime/netstrms.c @@ -37,6 +37,7 @@ #include "nssel.h" #include "nspoll.h" #include "netstrms.h" +#include "rsconf.h" MODULE_TYPE_LIB MODULE_TYPE_NOKEEP @@ -64,7 +65,7 @@ loadDrvr(netstrms_t *pThis) pBaseDrvrName = pThis->pBaseDrvrName; if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ - pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(runConf); if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsd_%s", pBaseDrvrName) == sizeof(szDrvrName)) ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 6fc300c525..a9725a8e12 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -53,6 +53,7 @@ #include "nsdsel_gtls.h" #include "nsd_gtls.h" #include "unicode-helper.h" +#include "rsconf.h" /* things to move to some better place/functionality - TODO */ #define CRLFILE "crl.pem" @@ -189,8 +190,8 @@ gtlsLoadOurCertKey(nsd_gtls_t *pThis) ISOBJ_TYPE_assert(pThis, nsd_gtls); - certFile = (pThis->pszCertFile == NULL) ? glbl.GetDfltNetstrmDrvrCertFile() : pThis->pszCertFile; - keyFile = (pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile() : pThis->pszKeyFile; + certFile = (pThis->pszCertFile == NULL) ? glbl.GetDfltNetstrmDrvrCertFile(runConf) : pThis->pszCertFile; + keyFile = (pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile(runConf) : pThis->pszKeyFile; if(certFile == NULL || keyFile == NULL) { /* in this case, we can not set our certificate. If we are @@ -610,8 +611,8 @@ gtlsAddOurCert(nsd_gtls_t *const pThis) uchar *pGnuErr; /* for GnuTLS error reporting */ DEFiRet; - certFile = (pThis->pszCertFile == NULL) ? glbl.GetDfltNetstrmDrvrCertFile() : pThis->pszCertFile; - keyFile = (pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile() : pThis->pszKeyFile; + certFile = (pThis->pszCertFile == NULL) ? glbl.GetDfltNetstrmDrvrCertFile(runConf) : pThis->pszCertFile; + keyFile = (pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile(runConf) : pThis->pszKeyFile; dbgprintf("GTLS certificate file: '%s'\n", certFile); dbgprintf("GTLS key file: '%s'\n", keyFile); if(certFile == NULL) { @@ -696,7 +697,7 @@ gtlsInitCred(nsd_gtls_t *const pThis ) CHKgnutls(gnutls_certificate_allocate_credentials(&pThis->xcred)); /* sets the trusted cas file */ - cafile = (pThis->pszCAFile == NULL) ? glbl.GetDfltNetstrmDrvrCAF() : pThis->pszCAFile; + cafile = (pThis->pszCAFile == NULL) ? glbl.GetDfltNetstrmDrvrCAF(runConf) : pThis->pszCAFile; if(cafile == NULL) { LogMsg(0, RS_RET_CA_CERT_MISSING, LOG_WARNING, "Warning: CA certificate is not set"); @@ -739,9 +740,9 @@ gtlsGlblInit(void) #endif CHKgnutls(gnutls_global_init()); - if(GetGnuTLSLoglevel() > 0){ + if(GetGnuTLSLoglevel(runConf) > 0){ gnutls_global_set_log_function(logFunction); - gnutls_global_set_log_level(GetGnuTLSLoglevel()); + gnutls_global_set_log_level(GetGnuTLSLoglevel(runConf)); /* 0 (no) to 9 (most), 10 everything */ } diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 03ebc0ab33..434389a302 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -53,6 +53,7 @@ #include "nsdsel_ossl.h" #include "nsd_ossl.h" #include "unicode-helper.h" +#include "rsconf.h" /* things to move to some better place/functionality - TODO */ // #define CRLFILE "crl.pem" @@ -1224,7 +1225,7 @@ osslInit_ctx(nsd_ossl_t *const pThis) int bHaveKey; const char *caFile, *certFile, *keyFile; /* Setup certificates */ - caFile = (char*) ((pThis->pszCAFile == NULL) ? glbl.GetDfltNetstrmDrvrCAF() : pThis->pszCAFile); + caFile = (char*) ((pThis->pszCAFile == NULL) ? glbl.GetDfltNetstrmDrvrCAF(runConf) : pThis->pszCAFile); if(caFile == NULL) { LogMsg(0, RS_RET_CA_CERT_MISSING, LOG_WARNING, "Warning: CA certificate is not set"); @@ -1232,7 +1233,8 @@ osslInit_ctx(nsd_ossl_t *const pThis) } else { bHaveCA = 1; } - certFile = (char*) ((pThis->pszCertFile == NULL) ? glbl.GetDfltNetstrmDrvrCertFile() : pThis->pszCertFile); + certFile = (char*) ((pThis->pszCertFile == NULL) ? + glbl.GetDfltNetstrmDrvrCertFile(runConf) : pThis->pszCertFile); if(certFile == NULL) { LogMsg(0, RS_RET_CERT_MISSING, LOG_WARNING, "Warning: Certificate file is not set"); @@ -1240,7 +1242,7 @@ osslInit_ctx(nsd_ossl_t *const pThis) } else { bHaveCert = 1; } - keyFile = (char*) ((pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile() : pThis->pszKeyFile); + keyFile = (char*) ((pThis->pszKeyFile == NULL) ? glbl.GetDfltNetstrmDrvrKeyFile(runConf) : pThis->pszKeyFile); if(keyFile == NULL) { LogMsg(0, RS_RET_CERTKEY_MISSING, LOG_WARNING, "Warning: Key file is not set"); diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index c44246cf4f..c5861ddd17 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -53,6 +53,7 @@ #include "nsd_ptcp.h" #include "prop.h" #include "dnscache.h" +#include "rsconf.h" MODULE_TYPE_LIB MODULE_TYPE_NOKEEP @@ -542,7 +543,7 @@ LstnInit(netstrms_t *const pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t* memset(&hints, 0, sizeof(hints)); hints.ai_flags = AI_PASSIVE; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runConf); hints.ai_socktype = SOCK_STREAM; error = getaddrinfo((const char*)cnf_params->pszAddr, (const char*) cnf_params->pszPort, &hints, &res); diff --git a/runtime/nspoll.c b/runtime/nspoll.c index d5c2a8d0ef..5e17bb175b 100644 --- a/runtime/nspoll.c +++ b/runtime/nspoll.c @@ -36,6 +36,7 @@ #include "module-template.h" #include "netstrm.h" #include "nspoll.h" +#include "rsconf.h" /* static data */ DEFobjStaticHelpers @@ -63,7 +64,7 @@ loadDrvr(nspoll_t *pThis) pBaseDrvrName = pThis->pBaseDrvrName; if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ - pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(runConf); if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdpoll_%s", pBaseDrvrName) == sizeof(szDrvrName)) ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); diff --git a/runtime/nssel.c b/runtime/nssel.c index 6a7cc4c061..bcfbfeed17 100644 --- a/runtime/nssel.c +++ b/runtime/nssel.c @@ -41,6 +41,7 @@ #include "module-template.h" #include "netstrm.h" #include "nssel.h" +#include "rsconf.h" /* static data */ DEFobjStaticHelpers @@ -68,7 +69,7 @@ loadDrvr(nssel_t *pThis) pBaseDrvrName = pThis->pBaseDrvrName; if(pBaseDrvrName == NULL) /* if no drvr name is set, use system default */ - pBaseDrvrName = glbl.GetDfltNetstrmDrvr(); + pBaseDrvrName = glbl.GetDfltNetstrmDrvr(runConf); if(snprintf((char*)szDrvrName, sizeof(szDrvrName), "lmnsdsel_%s", pBaseDrvrName) == sizeof(szDrvrName)) ABORT_FINALIZE(RS_RET_DRVRNAME_TOO_LONG); CHKmalloc(pThis->pDrvrName = (uchar*) strdup((char*)szDrvrName)); @@ -163,7 +164,7 @@ Add(nssel_t *pThis, netstrm_t *pStrm, nsdsel_waitOp_t waitOp) ISOBJ_TYPE_assert(pThis, nssel); ISOBJ_TYPE_assert(pStrm, netstrm); - + CHKiRet(pThis->Drvr.Add(pThis->pDrvrData, pStrm->pDrvrData, waitOp)); finalize_it: diff --git a/runtime/operatingstate.c b/runtime/operatingstate.c index e72e9d45d6..04b4cbd005 100644 --- a/runtime/operatingstate.c +++ b/runtime/operatingstate.c @@ -34,6 +34,7 @@ #include "rsyslog.h" #include "errmsg.h" #include "operatingstate.h" +#include "rsconf.h" #ifndef O_LARGEFILE #define O_LARGEFILE 0 @@ -53,7 +54,7 @@ static void osf_checkOnStartup(void) { int do_rename = 1; - const char *fn_osf = (const char*) glblGetOperatingStateFile(); + const char *fn_osf = (const char*) glblGetOperatingStateFile(loadConf); char iobuf[sizeof(STATE_CLEAN_CLOSE)]; const int len_clean_close = sizeof(STATE_CLEAN_CLOSE) - 1; assert(fn_osf != NULL); @@ -116,7 +117,7 @@ void osf_open(void) { assert(fd_osf == -1); - const char *fn_osf = (const char*) glblGetOperatingStateFile(); + const char *fn_osf = (const char*) glblGetOperatingStateFile(loadConf); assert(fn_osf != NULL); osf_checkOnStartup(); diff --git a/runtime/parser.c b/runtime/parser.c index 8fcddfb475..ea3514d3dd 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -68,8 +68,6 @@ parserList_t *pParsLstRoot = NULL; */ parserList_t *pDfltParsLst = NULL; -int bSupportCompressionExtension = 1; - /* intialize (but NOT allocate) a parser list. Primarily meant as a hook * which can be used to extend the list in the future. So far, just sets * it to NULL. @@ -324,7 +322,8 @@ static rsRetVal uncompressMessage(smsg_t *pMsg) /* we first need to check if we have a compressed record. If so, * we must decompress it. */ - if(lenMsg > 0 && *pszMsg == 'z' && bSupportCompressionExtension) { /* compressed data present? */ + if(lenMsg > 0 && *pszMsg == 'z' && + runConf->globals.bSupportCompressionExtension) { /* compressed data present? */ /* we have compressed data, so let's deflate it. We support a maximum * message size of iMaxLine. If it is larger, an error message is logged * and the message is dropped. We do NOT try to decompress larger messages @@ -333,7 +332,7 @@ static rsRetVal uncompressMessage(smsg_t *pMsg) * feature. */ int ret; - iLenDefBuf = glbl.GetMaxLine(); + iLenDefBuf = glbl.GetMaxLine(runConf); CHKmalloc(deflateBuf = malloc(iLenDefBuf + 1)); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", @@ -412,7 +411,7 @@ SanitizeMsg(smsg_t *pMsg) * compatible to recent IETF developments, we allow the user to * turn on/off this handling. rgerhards, 2007-07-23 */ - if(glbl.GetParserDropTrailingLFOnReception() + if(glbl.GetParserDropTrailingLFOnReception(runConf) && lenMsg > 0 && pszMsg[lenMsg-1] == '\n') { DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); lenMsg--; @@ -433,15 +432,15 @@ SanitizeMsg(smsg_t *pMsg) int bNeedSanitize = 0; for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { if(pszMsg[iSrc] < 32) { - if(glbl.GetParserSpaceLFOnReceive() && pszMsg[iSrc] == '\n') { + if(glbl.GetParserSpaceLFOnReceive(runConf) && pszMsg[iSrc] == '\n') { pszMsg[iSrc] = ' '; - } else if(pszMsg[iSrc] == '\0' || glbl.GetParserEscapeControlCharactersOnReceive()) { + } else if(pszMsg[iSrc] == '\0' || glbl.GetParserEscapeControlCharactersOnReceive(runConf)) { bNeedSanitize = 1; - if (!glbl.GetParserSpaceLFOnReceive()) { + if (!glbl.GetParserSpaceLFOnReceive(runConf)) { break; } } - } else if(pszMsg[iSrc] > 127 && glbl.GetParserEscape8BitCharactersOnReceive()) { + } else if(pszMsg[iSrc] > 127 && glbl.GetParserEscape8BitCharactersOnReceive(runConf)) { bNeedSanitize = 1; break; } @@ -456,7 +455,7 @@ SanitizeMsg(smsg_t *pMsg) /* now copy over the message and sanitize it. Note that up to iSrc-1 there was * obviously no need to sanitize, so we can go over that quickly... */ - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runConf); maxDest = lenMsg * 4; /* message can grow at most four-fold */ if(maxDest > iMaxLine) @@ -477,16 +476,16 @@ SanitizeMsg(smsg_t *pMsg) } iDst = iSrc; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ - if((pszMsg[iSrc] < 32) && (pszMsg[iSrc] != '\t' || glbl.GetParserEscapeControlCharacterTab())) { + if((pszMsg[iSrc] < 32) && (pszMsg[iSrc] != '\t' || glbl.GetParserEscapeControlCharacterTab(runConf))) { /* note: \0 must always be escaped, the rest of the code currently * can not handle it! -- rgerhards, 2009-08-26 */ - if(pszMsg[iSrc] == '\0' || glbl.GetParserEscapeControlCharactersOnReceive()) { + if(pszMsg[iSrc] == '\0' || glbl.GetParserEscapeControlCharactersOnReceive(runConf)) { /* we are configured to escape control characters. Please note * that this most probably break non-western character sets like * Japanese, Korean or Chinese. rgerhards, 2007-07-17 */ - if (glbl.GetParserEscapeControlCharactersCStyle()) { + if (glbl.GetParserEscapeControlCharactersCStyle(runConf)) { pDst[iDst++] = '\\'; switch (pszMsg[iSrc]) { @@ -528,15 +527,15 @@ SanitizeMsg(smsg_t *pMsg) } } else { - pDst[iDst++] = glbl.GetParserControlCharacterEscapePrefix(); + pDst[iDst++] = glbl.GetParserControlCharacterEscapePrefix(runConf); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } } - } else if(pszMsg[iSrc] > 127 && glbl.GetParserEscape8BitCharactersOnReceive()) { - if (glbl.GetParserEscapeControlCharactersCStyle()) { + } else if(pszMsg[iSrc] > 127 && glbl.GetParserEscape8BitCharactersOnReceive(runConf)) { + if (glbl.GetParserEscapeControlCharactersCStyle(runConf)) { pDst[iDst++] = '\\'; pDst[iDst++] = 'x'; @@ -548,7 +547,7 @@ SanitizeMsg(smsg_t *pMsg) /* In this case, we also do the conversion. Note that this most * probably breaks European languages. -- rgerhards, 2010-01-27 */ - pDst[iDst++] = glbl.GetParserControlCharacterEscapePrefix(); + pDst[iDst++] = glbl.GetParserControlCharacterEscapePrefix(runConf); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); diff --git a/runtime/parser.h b/runtime/parser.h index df0802b40b..166a483e1f 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -66,8 +66,6 @@ ENDinterface(parser) void printParserList(parserList_t *pList); -extern int bSupportCompressionExtension; - /* prototypes */ PROTOTYPEObj(parser); rsRetVal parserConstructViaModAndName(modInfo_t *pMod, uchar *const pName, void *parserInst); diff --git a/runtime/queue.c b/runtime/queue.c index 5339370e05..78a4618fb5 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -62,6 +62,7 @@ #include "unicode-helper.h" #include "statsobj.h" #include "parserif.h" +#include "rsconf.h" #ifdef OS_SOLARIS # include @@ -84,17 +85,6 @@ unsigned int iOverallQueueSize = 0; #define OVERSIZE_QUEUE_WATERMARK 500000 /* when is a queue considered to be "overly large"? */ -/* overridable default values (via global config) */ -int actq_dflt_toQShutdown = 10; /* queue shutdown */ -int actq_dflt_toActShutdown = 1000; /* action shutdown (in phase 2) */ -int actq_dflt_toEnq = 2000; /* timeout for queue enque */ -int actq_dflt_toWrkShutdown = 60000; /* timeout for worker thread shutdown */ - -int ruleset_dflt_toQShutdown = 1500; /* queue shutdown */ -int ruleset_dflt_toActShutdown = 1000; /* action shutdown (in phase 2) */ -int ruleset_dflt_toEnq = 2000; /* timeout for queue enque */ -int ruleset_dflt_toWrkShutdown = 60000; /* timeout for worker thread shutdown */ - /* forward-definitions */ static rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, smsg_t *pMsg); @@ -1482,7 +1472,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread { DEFiRet; qqueue_t *pThis; - const uchar *const workDir = glblGetWorkDirRaw(); + const uchar *const workDir = glblGetWorkDirRaw(ourConf); assert(ppThis != NULL); assert(pConsumer != NULL); @@ -1549,10 +1539,10 @@ qqueueSetDefaultsActionQueue(qqueue_t *pThis) pThis->iMaxFileSize = 1024*1024; pThis->iPersistUpdCnt = 0; /* persist queue info every n updates */ pThis->bSyncQueueFiles = 0; - pThis->toQShutdown = actq_dflt_toQShutdown; /* queue shutdown */ - pThis->toActShutdown = actq_dflt_toActShutdown; /* action shutdown (in phase 2) */ - pThis->toEnq = actq_dflt_toEnq; /* timeout for queue enque */ - pThis->toWrkShutdown = actq_dflt_toWrkShutdown; /* timeout for worker thread shutdown */ + pThis->toQShutdown = loadConf->globals.actq_dflt_toQShutdown; /* queue shutdown */ + pThis->toActShutdown = loadConf->globals.actq_dflt_toActShutdown; /* action shutdown (in phase 2) */ + pThis->toEnq = loadConf->globals.actq_dflt_toEnq; /* timeout for queue enque */ + pThis->toWrkShutdown = loadConf->globals.actq_dflt_toWrkShutdown; /* timeout for worker thread shutdown */ pThis->iMinMsgsPerWrkr = -1; /* minimum messages per worker needed to start a new one */ pThis->bSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ pThis->sizeOnDiskMax = 0; /* unlimited */ @@ -1582,10 +1572,10 @@ qqueueSetDefaultsRulesetQueue(qqueue_t *pThis) pThis->iMaxFileSize = 16*1024*1024; pThis->iPersistUpdCnt = 0; /* persist queue info every n updates */ pThis->bSyncQueueFiles = 0; - pThis->toQShutdown = ruleset_dflt_toQShutdown; - pThis->toActShutdown = ruleset_dflt_toActShutdown; - pThis->toEnq = ruleset_dflt_toEnq; - pThis->toWrkShutdown = ruleset_dflt_toWrkShutdown; + pThis->toQShutdown = ourConf->globals.ruleset_dflt_toQShutdown; + pThis->toActShutdown = ourConf->globals.ruleset_dflt_toActShutdown; + pThis->toEnq = ourConf->globals.ruleset_dflt_toEnq; + pThis->toWrkShutdown = ourConf->globals.ruleset_dflt_toWrkShutdown; pThis->iMinMsgsPerWrkr = -1; /* minimum messages per worker needed to start a new one */ pThis->bSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ pThis->sizeOnDiskMax = 0; /* unlimited */ @@ -2351,7 +2341,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* note: we need to pick the path so late as we do not have * the workdir during early config load */ - if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) + if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir(runConf))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); } @@ -2820,7 +2810,7 @@ DoSaveOnShutdown(qqueue_t *pThis) BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) DBGOPRINT((obj_t*) pThis, "shutdown: begin to destruct queue\n"); - if(glblShutdownQueueDoubleSize) { + if(ourConf->globals.shutdownQueueDoubleSize) { pThis->iHighWtrMrk *= 2; pThis->iMaxQueueSize *= 2; } diff --git a/runtime/queue.h b/runtime/queue.h index f84f2aaf96..2367c34c9c 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -237,17 +237,6 @@ PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); #define qqueueGetID(pThis) ((unsigned long) pThis) -/* overridable default values (via global config) */ -extern int actq_dflt_toQShutdown; -extern int actq_dflt_toActShutdown; -extern int actq_dflt_toEnq; -extern int actq_dflt_toWrkShutdown; - -extern int ruleset_dflt_toQShutdown; -extern int ruleset_dflt_toActShutdown; -extern int ruleset_dflt_toEnq; -extern int ruleset_dflt_toWrkShutdown; - #ifdef ENABLE_IMDIAG extern unsigned int iOverallQueueSize; #endif diff --git a/runtime/ratelimit.c b/runtime/ratelimit.c index 363036c64f..1669962f6e 100644 --- a/runtime/ratelimit.c +++ b/runtime/ratelimit.c @@ -305,7 +305,7 @@ ratelimitAddMsg(ratelimit_t *ratelimit, multi_submit_t *pMultiSub, smsg_t *pMsg) CHKiRet(multiSubmitMsg2(pMultiSub)); } CHKiRet(localRet); - if(pMsg->iLenRawMsg > glblGetMaxLine()) { + if(pMsg->iLenRawMsg > glblGetMaxLine(runConf)) { /* oversize message needs special processing. We keep * at least the previous batch as batch... */ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 42195e2c5c..036ba3b6e2 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "rsyslog.h" #include "obj.h" @@ -163,6 +164,62 @@ static void cnfSetDefaults(rsconf_t *pThis) pThis->templates.last = NULL; pThis->templates.lastStatic = NULL; pThis->actions.nbrActions = 0; + pThis->globals.pszWorkDir = NULL; + pThis->globals.bDropMalPTRMsgs = 0; + pThis->globals.operatingStateFile = NULL; + pThis->globals.iGnuTLSLoglevel = 0; + pThis->globals.debugOnShutdown = 0; + pThis->globals.pszDfltNetstrmDrvrCAF = NULL; + pThis->globals.pszDfltNetstrmDrvrCertFile = NULL; + pThis->globals.pszDfltNetstrmDrvrKeyFile = NULL; + pThis->globals.pszDfltNetstrmDrvr = NULL; + pThis->globals.oversizeMsgErrorFile = NULL; + pThis->globals.reportOversizeMsg = 1; + pThis->globals.oversizeMsgInputMode = 0; + pThis->globals.reportChildProcessExits = REPORT_CHILD_PROCESS_EXITS_ERRORS; + pThis->globals.bActionReportSuspension = 1; + pThis->globals.bActionReportSuspensionCont = 0; + pThis->globals.janitorInterval = 10; + pThis->globals.reportNewSenders = 0; + pThis->globals.reportGoneAwaySenders = 0; + pThis->globals.senderStatsTimeout = 12 * 60 * 60; /* 12 hr timeout for senders */ + pThis->globals.senderKeepTrack = 0; + pThis->globals.inputTimeoutShutdown = 1000; + pThis->globals.iDefPFFamily = PF_UNSPEC; + pThis->globals.ACLAddHostnameOnFail = 0; + pThis->globals.ACLDontResolve = 0; + pThis->globals.bDisableDNS = 0; + pThis->globals.bProcessInternalMessages = 0; + const char *const log_dflt = getenv("RSYSLOG_DFLT_LOG_INTERNAL"); + if(log_dflt != NULL && !strcmp(log_dflt, "1")) + pThis->globals.bProcessInternalMessages = 1; + pThis->globals.glblDevOptions = 0; + pThis->globals.intMsgRateLimitItv = 5; + pThis->globals.intMsgRateLimitBurst = 500; + pThis->globals.intMsgsSeverityFilter = DFLT_INT_MSGS_SEV_FILTER; + pThis->globals.permitCtlC = glblPermitCtlC; + + pThis->globals.actq_dflt_toQShutdown = 10; + pThis->globals.actq_dflt_toActShutdown = 1000; + pThis->globals.actq_dflt_toEnq = 2000; + pThis->globals.actq_dflt_toWrkShutdown = 60000; + + pThis->globals.ruleset_dflt_toQShutdown = 1500; + pThis->globals.ruleset_dflt_toActShutdown = 1000; + pThis->globals.ruleset_dflt_toEnq = 2000; + pThis->globals.ruleset_dflt_toWrkShutdown = 60000; + + pThis->globals.dnscacheDefaultTTL = 24 * 60 * 60; + pThis->globals.dnscacheEnableTTL = 0; + pThis->globals.shutdownQueueDoubleSize = 0; + pThis->globals.optionDisallowWarning = 1; + pThis->globals.bSupportCompressionExtension = 1; + #ifdef ENABLE_LIBLOGGING_STDLOG + pThis->globals.stdlog_hdl = stdlog_open("rsyslogd", 0, STDLOG_SYSLOG, NULL); + pThis->globals.stdlog_chanspec = NULL; + #endif + pThis->globals.iMaxLine = 8096; + /* queue params */ pThis->globals.mainQ.iMainMsgQueueSize = 100000; pThis->globals.mainQ.iMainMsgQHighWtrMark = 80000; @@ -186,6 +243,16 @@ static void cnfSetDefaults(rsconf_t *pThis) pThis->globals.mainQ.bMainMsgQSaveOnShutdown = 1; pThis->globals.mainQ.iMainMsgQueueDeqtWinFromHr = 0; pThis->globals.mainQ.iMainMsgQueueDeqtWinToHr = 25; + + pThis->globals.parser.cCCEscapeChar = '#'; + pThis->globals.parser.bDropTrailingLF = 1; + pThis->globals.parser.bEscapeCCOnRcv = 1; + pThis->globals.parser.bSpaceLFOnRcv = 0; + pThis->globals.parser.bEscape8BitChars = 0; + pThis->globals.parser.bEscapeTab = 1; + pThis->globals.parser.bParserEscapeCCCStyle = 0; + pThis->globals.parser.bPermitSlashInProgramname = 0; + pThis->globals.parser.bParseHOSTNAMEandTAG = 1; } @@ -243,6 +310,17 @@ CODESTARTobjDestruct(rsconf) ochDeleteAll(); free(pThis->globals.mainQ.pszMainMsgQFName); free(pThis->globals.pszConfDAGFile); + free(pThis->globals.pszWorkDir); + free(pThis->globals.operatingStateFile); + free(pThis->globals.pszDfltNetstrmDrvrCAF); + free(pThis->globals.pszDfltNetstrmDrvrCertFile); + free(pThis->globals.pszDfltNetstrmDrvrKeyFile); + free(pThis->globals.pszDfltNetstrmDrvr); + free(pThis->globals.oversizeMsgErrorFile); + #ifdef ENABLE_LIBLOGGING_STDLOG + stdlog_close(pThis->globals.stdlog_hdl); + free(pThis->globals.stdlog_chanspec); + #endif lookupDestroyCnf(); llDestroy(&(pThis->rulesets.llRulesets)); ENDobjDestruct(rsconf) @@ -266,7 +344,7 @@ BEGINobjDebugPrint(rsconf) /* be sure to specify the object type also in END and dbgprintf(" bErrMsgToStderr.....................: %d\n", pThis->globals.bErrMsgToStderr); dbgprintf(" drop Msgs with malicious PTR Record : %d\n", - glbl.GetDropMalPTRMsgs()); + glbl.GetDropMalPTRMsgs(pThis)); ruleset.DebugPrintAll(pThis); dbgprintf("\n"); if(pThis->globals.bDebugPrintTemplateList) @@ -297,7 +375,7 @@ BEGINobjDebugPrint(rsconf) /* be sure to specify the object type also in END and setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ - dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); + dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir(pThis)); ochPrintList(pThis); dbgprintf("Modules used in this configuration:\n"); for(modNode = pThis->modules.root ; modNode != NULL ; modNode = modNode->next) { @@ -1299,7 +1377,7 @@ validateConf(rsconf_t *cnf) if(cnf->globals.mainQ.MainMsgQueType == QUEUETYPE_DISK) { errno = 0; /* for logerror! */ - if(glbl.GetWorkDir() == NULL) { + if(glbl.GetWorkDir(cnf) == NULL) { LogError(0, NO_ERRCODE, "No $WorkDirectory specified - can not run main " "message queue in 'disk' mode. Using 'FixedArray' instead.\n"); cnf->globals.mainQ.MainMsgQueType = QUEUETYPE_FIXED_ARRAY; diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 598da1483b..16e3786a5f 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -31,6 +31,14 @@ /* --- configuration objects (the plan is to have ALL upper layers in this file) --- */ +#define REPORT_CHILD_PROCESS_EXITS_NONE 0 +#define REPORT_CHILD_PROCESS_EXITS_ERRORS 1 +#define REPORT_CHILD_PROCESS_EXITS_ALL 2 + +#ifndef DFLT_INT_MSGS_SEV_FILTER + #define DFLT_INT_MSGS_SEV_FILTER 6 /* Warning level and more important */ +#endif + /* queue config parameters. TODO: move to queue.c? */ struct queuecnf_s { int iMainMsgQueueSize; /* size of the main message queue above */ @@ -57,6 +65,19 @@ struct queuecnf_s { int iMainMsgQueueDeqtWinToHr; /* hour begin of time frame when queue is to be dequeued */ }; +/* parser config parameters */ +struct parsercnf_s { + uchar cCCEscapeChar; /* character to be used to start an escape sequence for control chars */ + int bDropTrailingLF; /* drop trailing LF's on reception? */ + int bEscapeCCOnRcv; /* escape control characters on reception: 0 - no, 1 - yes */ + int bSpaceLFOnRcv; /* replace newlines with spaces on reception: 0 - no, 1 - yes */ + int bEscape8BitChars; /* escape characters > 127 on reception: 0 - no, 1 - yes */ + int bEscapeTab; /* escape tab control character when doing CC escapes: 0 - no, 1 - yes */ + int bParserEscapeCCCStyle; /* escape control characters in c style: 0 - no, 1 - yes */ + int bPermitSlashInProgramname; + int bParseHOSTNAMEandTAG; /* parser modification (based on startup params!) */ +}; + /* globals are data items that are really global, and can be set only * once (at least in theory, because the legacy system permits them to * be re-set as often as the user likes). @@ -77,12 +98,68 @@ struct globals_s { int abortOnIDResolutionFail; int umask; /* umask to use */ uchar *pszConfDAGFile; /* name of config DAG file, non-NULL means generate one */ + uchar *pszWorkDir; + int bDropMalPTRMsgs;/* Drop messages which have malicious PTR records during DNS lookup */ + uchar *operatingStateFile; + int debugOnShutdown; /* start debug log when we are shut down */ + int iGnuTLSLoglevel;/* Sets GNUTLS Debug Level */ + uchar *pszDfltNetstrmDrvrCAF; /* default CA file for the netstrm driver */ + uchar *pszDfltNetstrmDrvrCertFile;/* default cert file for the netstrm driver (server) */ + uchar *pszDfltNetstrmDrvrKeyFile; /* default key file for the netstrm driver (server) */ + uchar *pszDfltNetstrmDrvr; /* module name of default netstream driver */ + uchar *oversizeMsgErrorFile; /* File where oversize messages are written to */ + int reportOversizeMsg; /* shall error messages be generated for oversize messages? */ + int oversizeMsgInputMode; /* Mode which oversize messages will be forwarded */ + int reportChildProcessExits; + int bActionReportSuspension; + int bActionReportSuspensionCont; + short janitorInterval; /* interval (in minutes) at which the janitor runs */ + int reportNewSenders; + int reportGoneAwaySenders; + int senderStatsTimeout; + int senderKeepTrack; /* keep track of known senders? */ + int inputTimeoutShutdown; /* input shutdown timeout in ms */ + int iDefPFFamily; /* protocol family (IPv4, IPv6 or both) */ + int ACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ + int ACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ + int bDisableDNS; /* don't look up IP addresses of remote messages */ + int bProcessInternalMessages; /* Should rsyslog itself process internal messages? + * 1 - yes + * 0 - send them to libstdlog (e.g. to push to journal) or syslog() + */ + uint64_t glblDevOptions; /* to be used by developers only */ + int intMsgRateLimitItv; + int intMsgRateLimitBurst; + int intMsgsSeverityFilter;/* filter for logging internal messages by syslog sev. */ + int permitCtlC; + + int actq_dflt_toQShutdown; /* queue shutdown */ + int actq_dflt_toActShutdown; /* action shutdown (in phase 2) */ + int actq_dflt_toEnq; /* timeout for queue enque */ + int actq_dflt_toWrkShutdown; /* timeout for worker thread shutdown */ + + int ruleset_dflt_toQShutdown; /* queue shutdown */ + int ruleset_dflt_toActShutdown; /* action shutdown (in phase 2) */ + int ruleset_dflt_toEnq; /* timeout for queue enque */ + int ruleset_dflt_toWrkShutdown; /* timeout for worker thread shutdown */ + + unsigned dnscacheDefaultTTL; /* 24 hrs default TTL */ + int dnscacheEnableTTL; /* expire entries or not (0) ? */ + int shutdownQueueDoubleSize; + int optionDisallowWarning; /* complain if message from disallowed sender is received */ + int bSupportCompressionExtension; + #ifdef ENABLE_LIBLOGGING_STDLOG + stdlog_channel_t stdlog_hdl; /* handle to be used for stdlog */ + uchar *stdlog_chanspec; + #endif + int iMaxLine; /* maximum length of a syslog message */ // TODO are the following ones defaults? int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ //TODO: other representation for main queue? Or just load it differently? queuecnf_t mainQ; /* main queue parameters */ + parsercnf_t parser; /* parser parameters */ }; /* (global) defaults are global in the sense that they are accessible diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index afce8b6780..c736fa89d8 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -143,7 +143,6 @@ rsrtInit(const char **ppErrObj, obj_if_t *pObjIF) /* init runtime only if not yet done */ #ifdef ENABLE_LIBLOGGING_STDLOG stdlog_init(0); - stdlog_hdl = stdlog_open("rsyslogd", 0, STDLOG_SYSLOG, NULL); #endif ret = pthread_attr_init(&default_thread_attr); if(ret != 0) { diff --git a/runtime/srutils.c b/runtime/srutils.c index 8548001cee..b2107f1cb7 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -48,6 +48,7 @@ #include "obj.h" #include "errmsg.h" #include "glbl.h" +#include "rsconf.h" #if _POSIX_TIMERS <= 0 #include @@ -176,7 +177,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len) uchar *pNew; assert(pOld != NULL); - + if((pNew = malloc(len + 1)) != NULL) memcpy(pNew, pOld, len + 1); @@ -296,7 +297,7 @@ int execProg(uchar *program, int bWait, uchar *arg) been reaped by the rsyslogd main loop (see rsyslogd.c) */ int status; if(waitpid(pid, &status, 0) == pid) { - glblReportChildProcessExit(program, pid, status); + glblReportChildProcessExit(runConf, program, pid, status); } else if(errno != ECHILD) { /* we do not use logerror(), because * that might bring us into an endless @@ -389,7 +390,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ if((pName = malloc(lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - + /* got memory, now construct string */ memcpy(pName, pDirName, lenDirName); pNameWork = pName + lenDirName; diff --git a/runtime/statsobj.c b/runtime/statsobj.c index 0f96100b70..ad959f5e54 100644 --- a/runtime/statsobj.c +++ b/runtime/statsobj.c @@ -42,6 +42,7 @@ #include "errmsg.h" #include "hashtable.h" #include "hashtable_itr.h" +#include "rsconf.h" /* externally-visiable data (see statsobj.h for explanation) */ @@ -362,7 +363,7 @@ getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, const statsFmtType_t fmt, co CHKmalloc(root = json_object_new_object()); CHKiRet(addContextForReporting(root, UCHAR_CONSTANT("name"), pThis->name)); - + if(pThis->origin != NULL) { CHKiRet(addContextForReporting(root, UCHAR_CONSTANT("origin"), pThis->origin)); } @@ -418,7 +419,7 @@ getStatsLineCEE(statsobj_t *pThis, cstr_t **ppcstr, const statsFmtType_t fmt, co if (values != NULL) { json_object_put(values); } - + RETiRet; } @@ -586,7 +587,7 @@ statsRecordSender(const uchar *sender, unsigned nMsgs, time_t lastSeen) CHKmalloc(stat = calloc(1, sizeof(struct sender_stats))); stat->sender = (const uchar*)strdup((const char*)sender); stat->nMsgs = 0; - if(glblReportNewSenders) { + if(runConf->globals.reportNewSenders) { LogMsg(0, RS_RET_SENDER_APPEARED, LOG_INFO, "new sender '%s'", stat->sender); } @@ -640,7 +641,7 @@ checkGoneAwaySenders(const time_t tCurr) { struct hashtable_itr *itr = NULL; struct sender_stats *stat; - const time_t rqdLast = tCurr - glblSenderStatsTimeout; + const time_t rqdLast = tCurr - runConf->globals.senderStatsTimeout; struct tm tm; pthread_mutex_lock(&mutSenders); @@ -653,7 +654,7 @@ checkGoneAwaySenders(const time_t tCurr) do { stat = (struct sender_stats*)hashtable_iterator_value(itr); if(stat->lastSeen < rqdLast) { - if(glblReportGoneAwaySenders) { + if(runConf->globals.reportGoneAwaySenders) { localtime_r(&stat->lastSeen, &tm); LogMsg(0, RS_RET_SENDER_GONE_AWAY, LOG_WARNING, diff --git a/runtime/stream.c b/runtime/stream.c index 23cde86a07..27010aba53 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -64,6 +64,7 @@ #include "errmsg.h" #include "cryprov.h" #include "datetime.h" +#include "rsconf.h" /* some platforms do not have large file support :( */ #ifndef O_LARGEFILE @@ -161,7 +162,7 @@ resolveFileSizeLimit(strm_t *pThis, uchar *pszCurrFName) if(pThis->pszSizeLimitCmd == NULL) { ABORT_FINALIZE(RS_RET_NON_SIZELIMITCMD); /* nothing we can do in this case... */ } - + /* we first check if we have command line parameters. We assume this, * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. @@ -349,7 +350,7 @@ strmSetCurrFName(strm_t *pThis) finalize_it: RETiRet; } - + /* This function checks if the actual file has changed and, if so, resets the * offset. This is support for monitoring files. It should be called after * deserializing the strm object and before doing any other operation on it @@ -400,7 +401,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); CHKiRet(strmSetCurrFName(pThis)); - + CHKiRet(doPhysOpen(pThis)); pThis->iCurrOffs = 0; @@ -824,7 +825,7 @@ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { int padBytes = 0; /* in crypto mode, we may have some padding (non-data) bytes */ DEFiRet; - + assert(pThis != NULL); assert(pC != NULL); @@ -836,7 +837,7 @@ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) pThis->iUngetC = -1; ABORT_FINALIZE(RS_RET_OK); } - + /* do we need to obtain a new buffer? */ if(pThis->iBufPtr >= pThis->iBufPtrMax) { CHKiRet(strmReadBuf(pThis, &padBytes)); @@ -1071,7 +1072,7 @@ strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *start_preg, regex_t * cstr_t *thisLine = NULL; rsRetVal readCharRet; const time_t tCurr = pThis->readTimeout ? getTime(NULL) : 0; - int maxMsgSize = glblGetMaxLine(); + int maxMsgSize = glblGetMaxLine(runConf); DEFiRet; do { @@ -1732,7 +1733,7 @@ syncFile(strm_t *pThis) DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", pThis->fd, err, errStr); } - + if(pThis->fdDir != -1) { if(fsync(pThis->fdDir) != 0) DBGPRINTF("stream/syncFile: fsync returned error, ignoring\n"); @@ -2260,7 +2261,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) assert(pThis != NULL); assert(pszName != NULL); - + if(iLenName < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -2290,7 +2291,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) assert(pThis != NULL); assert(pszDir != NULL); - + if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -2442,7 +2443,7 @@ strmDup(strm_t *const pThis, strm_t **ppNew) pNew->iFileNumDigits = pThis->iFileNumDigits; pNew->bDeleteOnClose = pThis->bDeleteOnClose; pNew->iCurrOffs = pThis->iCurrOffs; - + *ppNew = pNew; pNew = NULL; diff --git a/runtime/tcps_sess.c b/runtime/tcps_sess.c index 9e5dbcc5cb..0efa2c23c4 100644 --- a/runtime/tcps_sess.c +++ b/runtime/tcps_sess.c @@ -49,6 +49,7 @@ #include "prop.h" #include "ratelimit.h" #include "debug.h" +#include "rsconf.h" /* static data */ @@ -69,7 +70,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->inputState = eAtStrtFram; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ /* now allocate the message reception buffer */ - CHKmalloc(pThis->pMsg = (uchar*) malloc(glbl.GetMaxLine() + 1)); + CHKmalloc(pThis->pMsg = (uchar*) malloc(glbl.GetMaxLine(runConf) + 1)); finalize_it: ENDobjConstruct(tcps_sess) @@ -236,7 +237,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG ISOBJ_TYPE_assert(pThis, tcps_sess); const tcpLstnParams_t *const cnf_params = pThis->pLstnInfo->cnf_params; - + if(pThis->iMsg == 0) { DBGPRINTF("discarding zero-sized message\n"); FINALIZE; @@ -291,7 +292,7 @@ PrepareClose(tcps_sess_t *pThis) DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); - + if(pThis->inputState == eAtStrtFram) { /* this is how it should be. There is no unprocessed * data left and such we have nothing to do. For simplicity @@ -360,7 +361,7 @@ processDataRcvd(tcps_sess_t *pThis, DEFiRet; const tcpLstnParams_t *const cnf_params = pThis->pLstnInfo->cnf_params; ISOBJ_TYPE_assert(pThis, tcps_sess); - int iMaxLine = glbl.GetMaxLine(); + int iMaxLine = glbl.GetMaxLine(runConf); uchar *propPeerName = NULL; int lenPeerName = 0; uchar *propPeerIP = NULL; @@ -544,7 +545,7 @@ DataRcvd(tcps_sess_t *pThis, char *pData, const size_t iLen) } iRet = multiSubmitFlush(&multiSub); - if(glblSenderKeepTrack) + if(runConf->globals.senderKeepTrack) statsRecordSender(propGetSzStr(pThis->fromHost), nMsgs, ttGenTime); finalize_it: diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c index 61c9444c7c..6b46510091 100644 --- a/runtime/tcpsrv.c +++ b/runtime/tcpsrv.c @@ -494,7 +494,7 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, */ if(!pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { DBGPRINTF("%s is not an allowed sender\n", fromHostFQDN); - if(glbl.GetOption_DisallowWarning()) { + if(glbl.GetOptionDisallowWarning(runConf)) { errno = 0; LogError(0, RS_RET_HOST_NOT_PERMITTED, "TCP message from disallowed " "sender %s discarded", fromHostFQDN); diff --git a/runtime/typedefs.h b/runtime/typedefs.h index ffc601ba9f..772f272887 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -96,6 +96,7 @@ typedef void (*statsobj_read_notifier_t)(statsobj_t *, void *); typedef struct nsd_epworkset_s nsd_epworkset_t; typedef struct templates_s templates_t; typedef struct queuecnf_s queuecnf_t; +typedef struct parsercnf_s parsercnf_t; typedef struct rulesets_s rulesets_t; typedef struct globals_s globals_t; typedef struct defaults_s defaults_t; diff --git a/threads.c b/threads.c index 5375906c98..0e4875d2a1 100644 --- a/threads.c +++ b/threads.c @@ -43,6 +43,7 @@ #include "errmsg.h" #include "glbl.h" #include "unicode-helper.h" +#include "rsconf.h" /* linked list of currently-known threads */ static linkedList_t llThrds; @@ -118,15 +119,15 @@ thrdTerminateNonCancel(thrdInfo_t *pThis) pThis->bShallStop = RSTRUE; d_pthread_mutex_lock(&pThis->mutThrd); - timeoutComp(&tTimeout, glblInputTimeoutShutdown); + timeoutComp(&tTimeout, runConf->globals.inputTimeoutShutdown); was_active = pThis->bIsActive; while(was_active) { if(dbgTimeoutToStderr) { fprintf(stderr, "rsyslogd debug: info: trying to cooperatively stop " - "input %s, timeout %d ms\n", pThis->name, glblInputTimeoutShutdown); + "input %s, timeout %d ms\n", pThis->name, runConf->globals.inputTimeoutShutdown); } DBGPRINTF("thread %s: initiating termination, timeout %d ms\n", - pThis->name, glblInputTimeoutShutdown); + pThis->name, runConf->globals.inputTimeoutShutdown); const int r = pthread_kill(pThis->thrdID, SIGTTIN); if(r != 0) { LogError(errno, RS_RET_INTERNAL_ERROR, "error terminating thread %s " diff --git a/tools/omfile.c b/tools/omfile.c index 338df30b01..6bb97e64dd 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -70,6 +70,7 @@ #include "cryprov.h" #include "parserif.h" #include "janitor.h" +#include "rsconf.h" MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP @@ -648,7 +649,7 @@ prepareFile(instanceData *__restrict__ const pData, const uchar *__restrict__ co if(pData->useSigprov) sigprovPrepare(pData, szNameBuf); - + finalize_it: if(iRet != RS_RET_OK) { if(pData->pStrm != NULL) { @@ -699,7 +700,7 @@ prepareDynFile(instanceData *__restrict__ const pData, const uchar *__restrict__ * we do not know if we will otherwise come back to this file to flush it * at end of TX. see https://github.com/rsyslog/rsyslog/issues/2502 */ - if(((glblDevOptions & DEV_OPTION_8_1905_HANG_TEST) == 0) && + if(((runModConf->pConf->globals.glblDevOptions & DEV_OPTION_8_1905_HANG_TEST) == 0) && pData->bFlushOnTXEnd && pData->pStrm != NULL) { CHKiRet(strm.Flush(pData->pStrm)); } @@ -947,7 +948,7 @@ janitorChkDynaFiles(instanceData *__restrict__ const pData) if(pData->iCurrElt == i) pData->iCurrElt = -1; /* no longer available! */ } else { - pCache[i]->nInactive += janitorInterval; + pCache[i]->nInactive += runModConf->pConf->globals.janitorInterval; } } } @@ -968,7 +969,7 @@ janitorCB(void *pUsr) STATSCOUNTER_INC(pData->ctrCloseTimeouts, pData->mutCtrCloseTimeouts); closeFile(pData); } else { - pData->nInactive += janitorInterval; + pData->nInactive += runModConf->pConf->globals.janitorInterval; } } } diff --git a/tools/omfwd.c b/tools/omfwd.c index de860e0a27..d1729b52f3 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -837,7 +837,7 @@ static rsRetVal TCPSendInit(void *pvData) if(pData->gnutlsPriorityString != NULL) { CHKiRet(netstrm.SetGnutlsPriorityString(pWrkrData->pNetstrm, pData->gnutlsPriorityString)); } - CHKiRet(netstrm.Connect(pWrkrData->pNetstrm, glbl.GetDefPFFamily(), + CHKiRet(netstrm.Connect(pWrkrData->pNetstrm, glbl.GetDefPFFamily(runModConf->pConf), (uchar*)pData->port, (uchar*)pData->target, pData->device)); /* set keep-alive if enabled */ @@ -964,7 +964,7 @@ static rsRetVal doTryResume(wrkrInstanceData_t *pWrkrData) memset(&hints, 0, sizeof(hints)); /* port must be numeric, because config file syntax requires this */ hints.ai_flags = AI_NUMERICSERV; - hints.ai_family = glbl.GetDefPFFamily(); + hints.ai_family = glbl.GetDefPFFamily(runModConf->pConf); hints.ai_socktype = SOCK_DGRAM; if((iErr = (getaddrinfo(pData->target, pData->port, &hints, &res))) != 0) { LogError(0, RS_RET_SUSPENDED, @@ -1050,7 +1050,7 @@ processMsg(wrkrInstanceData_t *__restrict__ const pWrkrData, instanceData *__restrict__ const pData = pWrkrData->pData; DEFiRet; - iMaxLine = glbl.GetMaxLine(); + iMaxLine = glbl.GetMaxLine(runModConf->pConf); psz = iparam->param; l = iparam->lenStr; diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c index 3b772302c7..11515f378f 100644 --- a/tools/pmrfc3164.c +++ b/tools/pmrfc3164.c @@ -42,6 +42,7 @@ #include "parser.h" #include "datetime.h" #include "unicode-helper.h" +#include "rsconf.h" MODULE_TYPE_PARSER MODULE_TYPE_NOKEEP PARSER_NAME("rsyslog.rfc3164") @@ -108,7 +109,7 @@ createInstance(instanceConf_t **pinst) inst->bPermitAtSignsInHostname = 0; inst->bForceTagEndingByColon = 0; inst->bRemoveMsgFirstSpace = 0; - bParseHOSTNAMEandTAG=glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); *pinst = inst; finalize_it: RETiRet; @@ -408,7 +409,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); DBGPRINTF("rfc3164 parser init called\n"); - bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(loadConf); /* cache value, is set only during rsyslogd option processing */ diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index 01e1bf4b51..c635842f71 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -320,7 +320,7 @@ checkStartupOK(void) fprintf(stderr, "rsyslogd: error reading pid file, cannot start up\n"); ABORT_FINALIZE(RS_RET_ERR); } - + /* ok, we got a pid, let's check if the process is running */ const pid_t pid = (pid_t) pf_pid; if(kill(pid, 0) == 0 || errno != ESRCH) { @@ -559,7 +559,8 @@ rsyslogd_InitStdRatelimiters(void) CHKiRet(ratelimitNew(&dflt_ratelimiter, "rsyslogd", "dflt")); CHKiRet(ratelimitNew(&internalMsg_ratelimiter, "rsyslogd", "internal_messages")); ratelimitSetThreadSafe(internalMsg_ratelimiter); - ratelimitSetLinuxLike(internalMsg_ratelimiter, glblIntMsgRateLimitItv, glblIntMsgRateLimitBurst); + ratelimitSetLinuxLike(internalMsg_ratelimiter, + loadConf->globals.intMsgRateLimitItv, loadConf->globals.intMsgRateLimitBurst); /* TODO: make internalMsg ratelimit settings configurable */ finalize_it: RETiRet; @@ -851,8 +852,8 @@ static void logmsgInternal_doWrite(smsg_t *pMsg) { const int pri = getPRIi(pMsg); - if(pri % 8 <= glblIntMsgsSeverityFilter) { - if(bProcessInternalMessages) { + if(pri % 8 <= runConf->globals.intMsgsSeverityFilter) { + if(runConf->globals.bProcessInternalMessages) { submitMsg2(pMsg); pMsg = NULL; /* msg obj handed over; do not destruct */ } else { @@ -865,13 +866,13 @@ logmsgInternal_doWrite(smsg_t *pMsg) */ static warnmsg_emitted = 0; if(warnmsg_emitted == 0) { - stdlog_log(stdlog_hdl, LOG_WARNING, "%s", + stdlog_log(runConf->globals.stdlog_hdl, LOG_WARNING, "%s", "RSYSLOG WARNING: liblogging-stdlog " "functionality will go away soon. For details see " "https://github.com/rsyslog/rsyslog/issues/2706"); warnmsg_emitted = 1; } - stdlog_log(stdlog_hdl, pri2sev(pri), "%s", (char*)msg); + stdlog_log(runConf->globals.stdlog_hdl, pri2sev(pri), "%s", (char*)msg); #else syslog(pri, "%s", msg); #endif @@ -1004,7 +1005,7 @@ splitOversizeMessage(smsg_t *const pMsg) const char *rawmsg; int nsegments; int len_rawmsg; - const int maxlen = glblGetMaxLine(); + const int maxlen = glblGetMaxLine(runConf); ISOBJ_TYPE_assert(pMsg, msg); getRawMsg(pMsg, (uchar**) &rawmsg, &len_rawmsg); @@ -1048,31 +1049,31 @@ submitMsg2(smsg_t *pMsg) ISOBJ_TYPE_assert(pMsg, msg); - if(getRawMsgLen(pMsg) > glblGetMaxLine()){ + if(getRawMsgLen(pMsg) > glblGetMaxLine(runConf)){ uchar *rawmsg; int dummy; getRawMsg(pMsg, &rawmsg, &dummy); - if(glblReportOversizeMessage()) { + if(glblReportOversizeMessage(runConf)) { LogMsg(0, RS_RET_OVERSIZE_MSG, LOG_WARNING, "message too long (%d) with configured size %d, begin of " "message is: %.80s", - getRawMsgLen(pMsg), glblGetMaxLine(), rawmsg); + getRawMsgLen(pMsg), glblGetMaxLine(runConf), rawmsg); } writeOversizeMessageLog(pMsg); - if(glblGetOversizeMsgInputMode() == glblOversizeMsgInputMode_Split) { + if(glblGetOversizeMsgInputMode(runConf) == glblOversizeMsgInputMode_Split) { splitOversizeMessage(pMsg); /* we have submitted the message segments recursively, so we * can just deleted the original msg object and terminate. */ msgDestruct(&pMsg); FINALIZE; - } else if(glblGetOversizeMsgInputMode() == glblOversizeMsgInputMode_Truncate) { + } else if(glblGetOversizeMsgInputMode(runConf) == glblOversizeMsgInputMode_Truncate) { MsgTruncateToMaxSize(pMsg); } else { /* in "accept" mode, we do nothing, simply because "accept" means * to use as-is. */ - assert(glblGetOversizeMsgInputMode() == glblOversizeMsgInputMode_Accept); + assert(glblGetOversizeMsgInputMode(runConf) == glblOversizeMsgInputMode_Accept); } } @@ -1403,16 +1404,14 @@ initAll(int argc, char **argv) DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': - fprintf (stderr, "rsyslogd: the -4 command line option will go away " - "soon.\nPlease use the global(net.ipprotocol=\"ipv4-only\") " + fprintf (stderr, "rsyslogd: the -4 command line option has gone away.\n" + "Please use the global(net.ipprotocol=\"ipv4-only\") " "configuration parameter instead.\n"); - glbl.SetDefPFFamily(PF_INET); break; case '6': - fprintf (stderr, "rsyslogd: the -6 command line option will go away " - "soon.\nPlease use the global(net.ipprotocol=\"ipv6-only\") " + fprintf (stderr, "rsyslogd: the -6 command line option will has gone away.\n" + "Please use the global(net.ipprotocol=\"ipv6-only\") " "configuration parameter instead.\n"); - glbl.SetDefPFFamily(PF_INET6); break; case 'A': fprintf (stderr, "rsyslogd: the -A command line option will go away " @@ -1472,16 +1471,14 @@ initAll(int argc, char **argv) } break; case 'q': /* add hostname if DNS resolving has failed */ - fprintf (stderr, "rsyslogd: the -q command line option will go away " - "soon.\nPlease use the global(net.aclAddHostnameOnFail=\"on\") " + fprintf (stderr, "rsyslogd: the -q command line option has gone away.\n" + "Please use the global(net.aclAddHostnameOnFail=\"on\") " "configuration parameter instead.\n"); - *(net.pACLAddHostnameOnFail) = 1; break; case 'Q': /* dont resolve hostnames in ACL to IPs */ - fprintf (stderr, "rsyslogd: the -Q command line option will go away " - "soon.\nPlease use the global(net.aclResolveHostname=\"off\") " + fprintf (stderr, "rsyslogd: the -Q command line option has gone away.\n" + "Please use the global(net.aclResolveHostname=\"off\") " "configuration parameter instead.\n"); - *(net.pACLDontResolve) = 1; break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ if(arg == NULL) { @@ -1503,12 +1500,10 @@ initAll(int argc, char **argv) case 'u': /* misc user settings */ iHelperUOpt = (arg == NULL) ? 0 : atoi(arg); if(iHelperUOpt & 0x01) { - fprintf (stderr, "rsyslogd: the -u command line option will go away " - "soon.\n" + fprintf (stderr, "rsyslogd: the -u command line option has gone away.\n" "For the 0x01 bit, please use the " "global(parser.parseHostnameAndTag=\"off\") " "configuration parameter instead.\n"); - glbl.SetParseHOSTNAMEandTAG(0); } if(iHelperUOpt & 0x02) { fprintf (stderr, "rsyslogd: the -u command line option will go away " @@ -1521,16 +1516,14 @@ initAll(int argc, char **argv) bChDirRoot = 0; break; case 'w': /* disable disallowed host warnigs */ - fprintf (stderr, "rsyslogd: the -w command line option will go away " - "soon.\nPlease use the global(net.permitWarning=\"off\") " + fprintf (stderr, "rsyslogd: the -w command line option has gone away.\n" + "Please use the global(net.permitWarning=\"off\") " "configuration parameter instead.\n"); - glbl.SetOption_DisallowWarning(0); break; case 'x': /* disable dns for remote messages */ - fprintf (stderr, "rsyslogd: the -x command line option will go away " - "soon.\nPlease use the global(net.enableDNS=\"off\") " + fprintf (stderr, "rsyslogd: the -x command line option has gone away.\n" + "Please use the global(net.enableDNS=\"off\") " "configuration parameter instead.\n"); - glbl.SetDisableDNS(1); break; case 'h': case '?': @@ -1565,7 +1558,7 @@ initAll(int argc, char **argv) * even on hard config errors. Note that this may lead to segfaults * or other malfunction further down the road. */ - if((glblDevOptions & DEV_OPTION_KEEP_RUNNING_ON_HARD_CONF_ERROR) == 1) { + if((loadConf->globals.glblDevOptions & DEV_OPTION_KEEP_RUNNING_ON_HARD_CONF_ERROR) == 1) { fprintf(stderr, "rsyslogd: NOTE: developer-only option set to keep rsyslog " "running where it should abort - this can lead to " "more problems later in the run.\n"); @@ -1592,7 +1585,7 @@ initAll(int argc, char **argv) localRet = RS_RET_OK; } CHKiRet(localRet); - + CHKiRet(rsyslogd_InitStdRatelimiters()); if(bChDirRoot) { @@ -1613,7 +1606,7 @@ initAll(int argc, char **argv) hdlr_enable(SIGPIPE, SIG_IGN); hdlr_enable(SIGXFSZ, SIG_IGN); - if(Debug || glblPermitCtlC) { + if(Debug || loadConf->globals.permitCtlC) { hdlr_enable(SIGUSR1, rsyslogdDebugSwitch); hdlr_enable(SIGINT, rsyslogdDoDie); hdlr_enable(SIGQUIT, rsyslogdDoDie); @@ -1820,7 +1813,7 @@ rsyslogdDoDie(int sig) abort(); } bFinished = sig; - if(glblDebugOnShutdown) { + if(runConf->globals.debugOnShutdown) { /* kind of hackish - set to 0, so that debug_swith will enable * and AND emit the "start debug log" message. */ @@ -1842,7 +1835,7 @@ wait_timeout(const sigset_t *sigmask) { struct timespec tvSelectTimeout; - tvSelectTimeout.tv_sec = janitorInterval * 60; /* interval is in minutes! */ + tvSelectTimeout.tv_sec = runConf->globals.janitorInterval * 60; /* interval is in minutes! */ tvSelectTimeout.tv_nsec = 0; #ifdef _AIX @@ -1856,7 +1849,7 @@ wait_timeout(const sigset_t *sigmask) * in useful subsecond steps. */ const long wait_period = 500000000; /* wait period in nanoseconds */ - int timeout = janitorInterval * 60 * (1000000000 / wait_period); + int timeout = runConf->globals.janitorInterval * 60 * (1000000000 / wait_period); tvSelectTimeout.tv_sec = 0; tvSelectTimeout.tv_nsec = wait_period; @@ -1934,7 +1927,7 @@ reapChild(void) int status; child = waitpid(-1, &status, WNOHANG); if(child != -1 && child != 0) { - glblReportChildProcessExit(NULL, child, status); + glblReportChildProcessExit(runConf, NULL, child, status); } } while(child > 0); } @@ -2018,7 +2011,7 @@ deinitAll(void) /* close the inputs */ DBGPRINTF("Terminating input threads...\n"); glbl.SetGlobalInputTermination(); - + thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ @@ -2137,16 +2130,13 @@ main(int argc, char **argv) /* disable case-sensitive comparisons in variable subsystem: */ fjson_global_do_case_sensitive_comparison(0); - const char *const log_dflt = getenv("RSYSLOG_DFLT_LOG_INTERNAL"); - if(log_dflt != NULL && !strcmp(log_dflt, "1")) - bProcessInternalMessages = 1; dbgClassInit(); initAll(argc, argv); #ifdef HAVE_LIBSYSTEMD sd_notify(0, "READY=1"); dbgprintf("done signaling to systemd that we are ready!\n"); #endif - DBGPRINTF("max message size: %d\n", glblGetMaxLine()); + DBGPRINTF("max message size: %d\n", glblGetMaxLine(runConf)); DBGPRINTF("----RSYSLOGD INITIALIZED\n"); LogMsg(0, RS_RET_OK, LOG_DEBUG, "rsyslogd fully started up and initialized " "- begin actual processing"); @@ -2154,9 +2144,6 @@ main(int argc, char **argv) mainloop(); LogMsg(0, RS_RET_OK, LOG_DEBUG, "rsyslogd shutting down"); deinitAll(); -#ifdef ENABLE_LIBLOGGING_STDLOG - stdlog_close(stdlog_hdl); -#endif osf_close(); return 0; } From feb6420148c351072a190990622b58124fd44506 Mon Sep 17 00:00:00 2001 From: Kailash Sethuraman Date: Thu, 13 Jan 2022 13:52:46 -0500 Subject: [PATCH 012/134] Fixes #4395 by correctly checking for EPIPE. kmsg is a unique device, which can recover from EPIPE errors. The original code checked for this, but checked the return value for the libc read call, which always returns -1 and sets the appropriate errno. This meant that when an EPIPE error actually happened, the fd was infinitely retried. The 'for loop' was broken out of, but the readikmsg() function is repeatedly called. Note: there is an additional bug here. The readikmsg function needs better error checking on the fd. I suspect that this was rarely an issue because /dev/kmsg goes truly invalid when the system is actually shutting down. The fix here is to check the return value as well as the errno. --- contrib/imkmsg/kmsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/imkmsg/kmsg.c b/contrib/imkmsg/kmsg.c index beb4076ef3..5a3f45e49c 100644 --- a/contrib/imkmsg/kmsg.c +++ b/contrib/imkmsg/kmsg.c @@ -214,7 +214,7 @@ readkmsg(void) if (i > 0) { /* successful read of message of nonzero length */ pRcv[i] = '\0'; - } else if (i == -EPIPE) { + } else if (i < 0 && errno == EPIPE) { imkmsgLogIntMsg(LOG_WARNING, "imkmsg: some messages in circular buffer got overwritten"); continue; From 3edbe334e7fc5d937f951cec739e05a47c70ea39 Mon Sep 17 00:00:00 2001 From: alakatos Date: Tue, 4 Jan 2022 15:46:50 +0100 Subject: [PATCH 013/134] Move timezone specific variables to rsconf --- runtime/Makefile.am | 32 +++---- runtime/datetime.c | 6 +- runtime/glbl.c | 166 +---------------------------------- runtime/glbl.h | 2 - runtime/rsconf.c | 6 ++ runtime/rsconf.h | 2 + runtime/timezones.c | 205 ++++++++++++++++++++++++++++++++++++++++++++ runtime/timezones.h | 38 ++++++++ runtime/typedefs.h | 1 + 9 files changed, 277 insertions(+), 181 deletions(-) create mode 100644 runtime/timezones.c create mode 100644 runtime/timezones.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 6a8ea22bd5..0e31a01064 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -1,5 +1,5 @@ sbin_PROGRAMS = -man_MANS = +man_MANS = noinst_LTLIBRARIES = librsyslog.la pkglib_LTLIBRARIES = #pkglib_LTLIBRARIES = librsyslog.la @@ -102,7 +102,9 @@ librsyslog_la_SOURCES = \ ../outchannel.c \ ../outchannel.h \ ../template.c \ - ../template.h + ../template.h \ + timezones.c \ + timezones.h # the files with ../ we need to work on - so that they either become part of the # runtime or will no longer be needed. -- rgerhards, 2008-06-13 # @@ -128,13 +130,13 @@ librsyslog_la_CPPFLAGS += -I\$(top_srcdir)/tools # # regular expression support -# +# if ENABLE_REGEXP pkglib_LTLIBRARIES += lmregexp.la lmregexp_la_SOURCES = regexp.c regexp.h lmregexp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmregexp_la_LDFLAGS = -module -avoid-version -lmregexp_la_LIBADD = +lmregexp_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmregexp_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -145,12 +147,12 @@ endif # # zlib support -# +# pkglib_LTLIBRARIES += lmzlibw.la lmzlibw_la_SOURCES = zlibw.c zlibw.h lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmzlibw_la_LDFLAGS = -module -avoid-version $(ZLIB_LIBS) -lmzlibw_la_LIBADD = +lmzlibw_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmzlibw_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -162,11 +164,11 @@ if ENABLE_INET pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # # network support -# +# lmnet_la_SOURCES = net.c net.h lmnet_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnet_la_LDFLAGS = -module -avoid-version ../compat/compat_la-getifaddrs.lo -lmnet_la_LIBADD = +lmnet_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmnet_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -180,7 +182,7 @@ lmnetstrms_la_SOURCES = netstrms.c netstrms.h \ nspoll.c nspoll.h lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version -lmnetstrms_la_LIBADD = +lmnetstrms_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmnetstrms_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -196,7 +198,7 @@ lmnsd_ptcp_la_SOURCES = nsd_ptcp.c nsd_ptcp.h \ nsdpoll_ptcp.c nsdpoll_ptcp.h lmnsd_ptcp_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnsd_ptcp_la_LDFLAGS = -module -avoid-version -lmnsd_ptcp_la_LIBADD = +lmnsd_ptcp_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmnsd_ptcp_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -207,7 +209,7 @@ endif # if ENABLE_INET # # openssl netstream driver -# +# if ENABLE_OPENSSL pkglib_LTLIBRARIES += lmnsd_ossl.la lmnsd_ossl_la_SOURCES = nsd_ossl.c nsd_ossl.h nsdsel_ossl.c nsdsel_ossl.h @@ -218,7 +220,7 @@ endif # # GnuTLS netstream driver -# +# if ENABLE_GNUTLS pkglib_LTLIBRARIES += lmnsd_gtls.la lmnsd_gtls_la_SOURCES = nsd_gtls.c nsd_gtls.h nsdsel_gtls.c nsdsel_gtls.h @@ -245,7 +247,7 @@ endif # # gssapi support -# +# if ENABLE_GSSAPI pkglib_LTLIBRARIES += lmgssutil.la lmgssutil_la_SOURCES = gss-misc.c gss-misc.h @@ -266,7 +268,7 @@ lmtcpsrv_la_SOURCES = \ tcpsrv.h lmtcpsrv_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmtcpsrv_la_LDFLAGS = -module -avoid-version -lmtcpsrv_la_LIBADD = +lmtcpsrv_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmtcpsrv_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) @@ -281,7 +283,7 @@ lmtcpclt_la_SOURCES = \ tcpclt.h lmtcpclt_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmtcpclt_la_LDFLAGS = -module -avoid-version -lmtcpclt_la_LIBADD = +lmtcpclt_la_LIBADD = if ENABLE_LIBLOGGING_STDLOG lmtcpclt_la_CPPFLAGS += $(LIBLOGGING_STDLOG_CFLAGS) diff --git a/runtime/datetime.c b/runtime/datetime.c index cc7b3c90eb..29f5415237 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -42,6 +42,8 @@ #include "srUtils.h" #include "stringbuf.h" #include "errmsg.h" +#include "rsconf.h" +#include "timezones.h" /* static data */ DEFobjStaticHelpers @@ -731,7 +733,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr, /* found TZ, apply it */ tzinfo_t* tzinfo; tzstring[i] = '\0'; - if((tzinfo = glblFindTimezoneInfo((char*) tzstring)) == NULL) { + if((tzinfo = glblFindTimezone(runConf, (char*) tzstring)) == NULL) { DBGPRINTF("ParseTIMESTAMP3164: invalid TZ string '%s' -- ignored\n", tzstring); } else { @@ -1016,7 +1018,7 @@ formatTimestamp3164(struct syslogTime *ts, char* pBuf, int bBuggyDay) int iDay; assert(ts != NULL); assert(pBuf != NULL); - + pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; pBuf[2] = monthNames[(ts->month - 1) % 12][2]; diff --git a/runtime/glbl.c b/runtime/glbl.c index 39f5fa7c1e..e46d8695d7 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -58,6 +58,7 @@ #include "queue.h" #include "dnscache.h" #include "parser.h" +#include "timezones.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -100,9 +101,6 @@ static int iFdSetSize = howmany(FD_SETSIZE, __NFDBITS) * sizeof (fd_mask); /* si #endif static uchar *SourceIPofLocalClient = NULL; /* [ar] Source IP for local client to be used on multihomed host */ -tzinfo_t *tzinfos = NULL; -static int ntzinfos; - /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfparamdescr[] = { { "workdirectory", eCmdHdlrString, 0 }, @@ -184,16 +182,6 @@ static struct cnfparamblk paramblk = cnfparamdescr }; -static struct cnfparamdescr timezonecnfparamdescr[] = { - { "id", eCmdHdlrString, CNFPARAM_REQUIRED}, - { "offset", eCmdHdlrGetWord, CNFPARAM_REQUIRED } -}; -static struct cnfparamblk timezonepblk = - { CNFPARAMBLK_VERSION, - sizeof(timezonecnfparamdescr)/sizeof(struct cnfparamdescr), - timezonecnfparamdescr - }; - static struct cnfparamvals *cnfparamvals = NULL; /* we need to support multiple calls into our param block, so we need * to persist the current settings. Note that this must be re-set @@ -981,66 +969,6 @@ glblPrepCnf(void) cnfparamvals = NULL; } - -static void -freeTimezoneInfo(void) -{ - int i; - for(i = 0 ; i < ntzinfos ; ++i) - free(tzinfos[i].id); - free(tzinfos); - tzinfos = NULL; -} - -static void -displayTzinfos(void) -{ - int i; - if(!Debug) - return; - for(i = 0 ; i < ntzinfos ; ++i) - dbgprintf("tzinfo: '%s':%c%2.2d:%2.2d\n", - tzinfos[i].id, tzinfos[i].offsMode, - tzinfos[i].offsHour, tzinfos[i].offsMin); -} - - -/* Note: this function is NOT thread-safe! - * This is currently not needed as used only during - * initialization. - */ -static rsRetVal -addTimezoneInfo(uchar *tzid, char offsMode, int8_t offsHour, int8_t offsMin) -{ - DEFiRet; - tzinfo_t *newti; - CHKmalloc(newti = realloc(tzinfos, (ntzinfos+1)*sizeof(tzinfo_t))); - if((newti[ntzinfos].id = strdup((char*)tzid)) == NULL) { - free(newti); - DBGPRINTF("addTimezoneInfo: strdup failed with OOM\n"); - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - newti[ntzinfos].offsMode = offsMode; - newti[ntzinfos].offsHour = offsHour; - newti[ntzinfos].offsMin = offsMin; - ++ntzinfos, tzinfos = newti; -finalize_it: - RETiRet; -} - - -static int -bs_arrcmp_tzinfo(const void *s1, const void *s2) -{ - return strcmp((char*)s1, (char*)((tzinfo_t*)s2)->id); -} -/* returns matching timezone info or NULL if no entry exists */ -tzinfo_t* -glblFindTimezoneInfo(char *id) -{ - return (tzinfo_t*) bsearch(id, tzinfos, ntzinfos, sizeof(tzinfo_t), bs_arrcmp_tzinfo); -} - /* handle the timezone() object. Each incarnation adds one additional * zone info to the global table of time zones. */ @@ -1051,81 +979,7 @@ bs_arrcmp_glblDbgFiles(const void *s1, const void *s2) return strcmp((char*)s1, *(char**)s2); } -void -glblProcessTimezone(struct cnfobj *o) -{ - struct cnfparamvals *pvals; - uchar *id = NULL; - uchar *offset = NULL; - char offsMode; - int8_t offsHour; - int8_t offsMin; - int i; - - pvals = nvlstGetParams(o->nvlst, &timezonepblk, NULL); - if(pvals == NULL) { - LogError(0, RS_RET_MISSING_CNFPARAMS, "error processing timezone " - "config parameters"); - goto done; - } - if(Debug) { - dbgprintf("timezone param blk after glblProcessTimezone:\n"); - cnfparamsPrint(&timezonepblk, pvals); - } - - for(i = 0 ; i < timezonepblk.nParams ; ++i) { - if(!pvals[i].bUsed) - continue; - if(!strcmp(timezonepblk.descr[i].name, "id")) { - id = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); - } else if(!strcmp(timezonepblk.descr[i].name, "offset")) { - offset = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); - } else { - dbgprintf("glblProcessTimezone: program error, non-handled " - "param '%s'\n", timezonepblk.descr[i].name); - } - } - - /* note: the following two checks for NULL are not strictly necessary - * as these are required parameters for the config block. But we keep - * them to make the clang static analyzer happy, which also helps - * guard against logic errors. - */ - if(offset == NULL) { - parser_errmsg("offset parameter missing (logic error?), timezone config ignored"); - goto done; - } - if(id == NULL) { - parser_errmsg("id parameter missing (logic error?), timezone config ignored"); - goto done; - } - if( strlen((char*)offset) != 6 - || !(offset[0] == '-' || offset[0] == '+') - || !(isdigit(offset[1]) && isdigit(offset[2])) - || offset[3] != ':' - || !(isdigit(offset[4]) && isdigit(offset[5])) - ) { - parser_errmsg("timezone offset has invalid format. Must be +/-hh:mm, e.g. \"-07:00\"."); - goto done; - } - - offsHour = (offset[1] - '0') * 10 + offset[2] - '0'; - offsMin = (offset[4] - '0') * 10 + offset[5] - '0'; - offsMode = offset[0]; - - if(offsHour > 12 || offsMin > 59) { - parser_errmsg("timezone offset outside of supported range (hours 0..12, minutes 0..59)"); - goto done; - } - - addTimezoneInfo(id, offsMode, offsHour, offsMin); - -done: - cnfparamvalsDestruct(pvals, &timezonepblk); - free(id); - free(offset); -} /* handle a global config object. Note that multiple global config statements * are permitted (because of plugin support), so once we got a param block, @@ -1221,15 +1075,6 @@ glblDestructMainqCnfObj(void) } } -/* comparison function for qsort() and string array compare - * this is for the string lookup table type - */ -static int -qs_arrcmp_tzinfo(const void *s1, const void *s2) -{ - return strcmp(((tzinfo_t*)s1)->id, ((tzinfo_t*)s2)->id); -} - static int qs_arrcmp_glblDbgFiles(const void *s1, const void *s2) { @@ -1288,11 +1133,9 @@ glblDoneLoadCnf(void) DEFiRet; CHKiRet(objUse(net, CORE_COMPONENT)); - if(ntzinfos > 0) { - qsort(tzinfos, ntzinfos, sizeof(tzinfo_t), qs_arrcmp_tzinfo); - } - DBGPRINTF("Timezone information table (%d entries):\n", ntzinfos); - displayTzinfos(); + sortTimezones(loadConf); + DBGPRINTF("Timezone information table (%d entries):\n", loadConf->timezones.ntzinfos); + displayTimezones(loadConf); if(cnfparamvals == NULL) goto finalize_it; @@ -1575,7 +1418,6 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(LocalHostName); free(LocalHostNameOverride); free(LocalFQDNName); - freeTimezoneInfo(); objRelease(prop, CORE_COMPONENT); if(propLocalHostNameToDelete != NULL) prop.Destruct(&propLocalHostNameToDelete); diff --git a/runtime/glbl.h b/runtime/glbl.h index f90b426fb3..fcd13c2d04 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -140,12 +140,10 @@ extern DEF_ATOMIC_HELPER_MUT(mutTerminateInputs); void glblPrepCnf(void); void glblProcessCnf(struct cnfobj *o); -void glblProcessTimezone(struct cnfobj *o); void glblProcessMainQCnf(struct cnfobj *o); void glblDestructMainqCnfObj(void); rsRetVal glblDoneLoadCnf(void); const uchar * glblGetWorkDirRaw(rsconf_t *cnf); -tzinfo_t* glblFindTimezoneInfo(char *id); int GetGnuTLSLoglevel(rsconf_t *cnf); int glblGetMaxLine(rsconf_t *cnf); int bs_arrcmp_glblDbgFiles(const void *s1, const void *s2); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 036ba3b6e2..ef5a52588f 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -68,6 +68,7 @@ #include "modules.h" #include "dirty.h" #include "template.h" +#include "timezones.h" extern char* yytext; /* static data */ @@ -220,6 +221,10 @@ static void cnfSetDefaults(rsconf_t *pThis) #endif pThis->globals.iMaxLine = 8096; + /* timezone specific*/ + pThis->timezones.tzinfos = NULL; + pThis->timezones.ntzinfos = 0; + /* queue params */ pThis->globals.mainQ.iMainMsgQueueSize = 100000; pThis->globals.mainQ.iMainMsgQHighWtrMark = 80000; @@ -308,6 +313,7 @@ CODESTARTobjDestruct(rsconf) dynstats_destroyAllBuckets(); perctileBucketsDestruct(); ochDeleteAll(); + freeTimezones(pThis); free(pThis->globals.mainQ.pszMainMsgQFName); free(pThis->globals.pszConfDAGFile); free(pThis->globals.pszWorkDir); diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 16e3786a5f..078f5dc866 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -28,6 +28,7 @@ #include "lookup.h" #include "dynstats.h" #include "perctile_stats.h" +#include "timezones.h" /* --- configuration objects (the plan is to have ALL upper layers in this file) --- */ @@ -239,6 +240,7 @@ struct rsconf_s { * - actions * Of course, we need to debate if we shall change that some time... */ + timezones_t timezones; }; diff --git a/runtime/timezones.c b/runtime/timezones.c new file mode 100644 index 0000000000..e1e039418d --- /dev/null +++ b/runtime/timezones.c @@ -0,0 +1,205 @@ +/* timezones.c + * Support for timezones in RainerScript. + * + * Copyright 2022 Attila Lakatos and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "unicode-helper.h" +#include "errmsg.h" +#include "parserif.h" +#include "rainerscript.h" +#include "srUtils.h" +#include "rsconf.h" + + +static struct cnfparamdescr timezonecnfparamdescr[] = { + { "id", eCmdHdlrString, CNFPARAM_REQUIRED}, + { "offset", eCmdHdlrGetWord, CNFPARAM_REQUIRED } +}; +static struct cnfparamblk timezonepblk = { + CNFPARAMBLK_VERSION, + sizeof(timezonecnfparamdescr)/sizeof(struct cnfparamdescr), + timezonecnfparamdescr +}; + +/* Note: this function is NOT thread-safe! + * This is currently not needed as used only during + * initialization. + */ +static rsRetVal +addTimezoneInfo(rsconf_t *cnf, uchar *tzid, char offsMode, int8_t offsHour, int8_t offsMin) +{ + DEFiRet; + tzinfo_t *newti; + CHKmalloc(newti = realloc(cnf->timezones.tzinfos, (cnf->timezones.ntzinfos+1)*sizeof(tzinfo_t))); + if((newti[cnf->timezones.ntzinfos].id = strdup((char*)tzid)) == NULL) { + free(newti); + DBGPRINTF("addTimezoneInfo: strdup failed with OOM\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + newti[cnf->timezones.ntzinfos].offsMode = offsMode; + newti[cnf->timezones.ntzinfos].offsHour = offsHour; + newti[cnf->timezones.ntzinfos].offsMin = offsMin; + ++cnf->timezones.ntzinfos, cnf->timezones.tzinfos = newti; +finalize_it: + RETiRet; +} + +void +glblProcessTimezone(struct cnfobj *o) +{ + struct cnfparamvals *pvals; + uchar *id = NULL; + uchar *offset = NULL; + char offsMode; + int8_t offsHour; + int8_t offsMin; + int i; + + pvals = nvlstGetParams(o->nvlst, &timezonepblk, NULL); + if(pvals == NULL) { + LogError(0, RS_RET_MISSING_CNFPARAMS, "error processing timezone " + "config parameters"); + goto done; + } + if(Debug) { + dbgprintf("timezone param blk after glblProcessTimezone:\n"); + cnfparamsPrint(&timezonepblk, pvals); + } + + for(i = 0 ; i < timezonepblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(timezonepblk.descr[i].name, "id")) { + id = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(timezonepblk.descr[i].name, "offset")) { + offset = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + } else { + dbgprintf("glblProcessTimezone: program error, non-handled " + "param '%s'\n", timezonepblk.descr[i].name); + } + } + + /* note: the following two checks for NULL are not strictly necessary + * as these are required parameters for the config block. But we keep + * them to make the clang static analyzer happy, which also helps + * guard against logic errors. + */ + if(offset == NULL) { + parser_errmsg("offset parameter missing (logic error?), timezone config ignored"); + goto done; + } + if(id == NULL) { + parser_errmsg("id parameter missing (logic error?), timezone config ignored"); + goto done; + } + + if( strlen((char*)offset) != 6 + || !(offset[0] == '-' || offset[0] == '+') + || !(isdigit(offset[1]) && isdigit(offset[2])) + || offset[3] != ':' + || !(isdigit(offset[4]) && isdigit(offset[5])) + ) { + parser_errmsg("timezone offset has invalid format. Must be +/-hh:mm, e.g. \"-07:00\"."); + goto done; + } + + offsHour = (offset[1] - '0') * 10 + offset[2] - '0'; + offsMin = (offset[4] - '0') * 10 + offset[5] - '0'; + offsMode = offset[0]; + + if(offsHour > 12 || offsMin > 59) { + parser_errmsg("timezone offset outside of supported range (hours 0..12, minutes 0..59)"); + goto done; + } + + addTimezoneInfo(loadConf, id, offsMode, offsHour, offsMin); + +done: + cnfparamvalsDestruct(pvals, &timezonepblk); + free(id); + free(offset); +} + +/* comparison function for qsort() and string array compare + * this is for the string lookup table type + */ +static int +qs_arrcmp_tzinfo(const void *s1, const void *s2) +{ + return strcmp(((tzinfo_t*)s1)->id, ((tzinfo_t*)s2)->id); +} + +void sortTimezones(rsconf_t *cnf) +{ + if (cnf->timezones.ntzinfos > 0) { + qsort(cnf->timezones.tzinfos, cnf->timezones.ntzinfos, + sizeof(tzinfo_t), qs_arrcmp_tzinfo); + } +} + +void +displayTimezones(rsconf_t *cnf) +{ + if(!Debug) + return; + for(int i = 0 ; i < cnf->timezones.ntzinfos ; ++i) + dbgprintf("tzinfo: '%s':%c%2.2d:%2.2d\n", + cnf->timezones.tzinfos[i].id, cnf->timezones.tzinfos[i].offsMode, + cnf->timezones.tzinfos[i].offsHour, cnf->timezones.tzinfos[i].offsMin); +} + +static int +bs_arrcmp_tzinfo(const void *s1, const void *s2) +{ + return strcmp((char*)s1, (char*)((tzinfo_t*)s2)->id); +} + +/* returns matching timezone info or NULL if no entry exists */ +tzinfo_t* +glblFindTimezone(rsconf_t *cnf, char *id) +{ + return (tzinfo_t*) bsearch( + id, cnf->timezones.tzinfos, cnf->timezones.ntzinfos, sizeof(tzinfo_t), bs_arrcmp_tzinfo); +} + +static void +freeTimezone(tzinfo_t *tzinfo) +{ + free(tzinfo->id); +} + +void +freeTimezones(rsconf_t *cnf) +{ + for(int i = 0; i < cnf->timezones.ntzinfos ; ++i) + freeTimezone(&cnf->timezones.tzinfos[i]); + if (cnf->timezones.ntzinfos > 0) + free(cnf->timezones.tzinfos); + cnf->timezones.tzinfos = NULL; +} diff --git a/runtime/timezones.h b/runtime/timezones.h new file mode 100644 index 0000000000..d2bcdb4fad --- /dev/null +++ b/runtime/timezones.h @@ -0,0 +1,38 @@ +/* header for timezones.c + * + * Copyright 2022 Attila Lakatos and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_TIMEZONES_H +#define INCLUDED_TIMEZONES_H + +#include "rsconf.h" + +/* timezone specific parameters*/ +struct timezones_s { + tzinfo_t *tzinfos; + int ntzinfos; +}; + +void displayTimezones(rsconf_t *cnf); +void sortTimezones(rsconf_t *cnf); +void glblProcessTimezone(struct cnfobj *o); +tzinfo_t* glblFindTimezone(rsconf_t *cnf, char *id); +void freeTimezones(rsconf_t *cnf); + +#endif diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 772f272887..706ab3b2da 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -101,6 +101,7 @@ typedef struct rulesets_s rulesets_t; typedef struct globals_s globals_t; typedef struct defaults_s defaults_t; typedef struct actions_s actions_t; +typedef struct timezones_s timezones_t; typedef struct rsconf_s rsconf_t; typedef struct cfgmodules_s cfgmodules_t; typedef struct cfgmodules_etry_s cfgmodules_etry_t; From 178a36c9b497a78855e1eda03550c3089473ace7 Mon Sep 17 00:00:00 2001 From: Gabor Orosz Date: Fri, 14 Jan 2022 19:58:17 +0000 Subject: [PATCH 014/134] Terminate all tcpsrv threads properly Graceful shutdown of Rsyslog could lead to segmentation faults when multiple imtcp inputs are being used. That is because the rest of the tcpsrv threads are left behind running, while their underlying objects are being disposed by the main thread as part of the module de-initialization. Signed-off-by: Gabor Orosz --- plugins/imtcp/imtcp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 0e689a28d8..a6256acad8 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -1007,6 +1007,7 @@ CODESTARTrunInput iRet = tcpsrv.Run(tcpsrv_root->tcpsrv); /* de-init remaining servers */ + etry = tcpsrv_root->next; while(etry != NULL) { stopSrvWrkr(etry); etry = etry->next; From f7bfb9b83a3e0ed603fcf5f9eec9b978397f59ef Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 17 Jan 2022 10:44:29 +0100 Subject: [PATCH 015/134] CI: remove fedora 33 based testing We have some hang conditions solely with Fedora 33, which we do not see with other Fedora versions. As we already have f 34 inside CI, we decide to not necessarily analyze this further but rather remove this environment from CI. If time permits, we still try to find the root cause, but so far we are very sure it's not directly in rsyslog. --- .github/workflows/run_fedora_33.yml | 59 ----------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/run_fedora_33.yml diff --git a/.github/workflows/run_fedora_33.yml b/.github/workflows/run_fedora_33.yml deleted file mode 100644 index e95cd9d9a5..0000000000 --- a/.github/workflows/run_fedora_33.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2020 Rainer Gerhards and Others -# -# https://github.com/rsyslog/rsyslog-pkg-ubuntu -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# References: -# -# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options -# https://github.com/settings/notifications -# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog - - ---- -name: check fedora 33 - -on: - pull_request: - -jobs: - check_run: - runs-on: ubuntu-18.04 - timeout-minutes: 50 - - steps: - - name: git checkout project - uses: actions/checkout@v1 - - - name: run container CI pipeline - run: | - chmod -R go+rw . - export RSYSLOG_CONTAINER_UID="" # use default - export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' - export CFLAGS='-g' - export CC='gcc' - export USE_AUTO_DEBUG='off' - export CI_MAKE_OPT='-j20' - export CI_MAKE_CHECK_OPT='-j8' - export CI_CHECK_CMD='check' - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:33' - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ - --disable-kafka-tests --enable-debug" - devtools/devcontainer.sh --rm devtools/run-ci.sh - - - name: show error logs (if we errored) - if: ${{ failure() || cancelled() }} - run: | - devtools/gather-check-logs.sh - cat failed-tests.log From 90882d52aacdddd67db01539b3ad027fa1fc5b88 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 17 Jan 2022 12:40:26 +0100 Subject: [PATCH 016/134] maintain ChangeLog --- ChangeLog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index d15c4b793f..1901f0e1fd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,13 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-01-17: imtcp bugfix: worker threads were not properly terminated + Graceful shutdown of Rsyslog could lead to segmentation faults when + multiple imtcp inputs were being used. That is because the rest of the + tcpsrv threads are left behind running, while their underlying objects + are being disposed by the main thread as part of the module + de-initialization. + closes: https://github.com/rsyslog/rsyslog/issues/4776 + Thanks to Gabor Orosz for the analysis and patch. - 2022-01-07: omlibdbi bugfix: use-after-free bug This occurred in when sqllite driver was used. Depending on circumstances, this had no visible issues (often) up to rsyslog segfault. The busier rsyslog is, the more From 320c673d9a8d4340b0e60d69c9cf10e8a018994b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 17 Jan 2022 15:44:45 +0100 Subject: [PATCH 017/134] maintain ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1901f0e1fd..8cf888c5a2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-01-17: imkmsg bugfix: effectively disabled input on error reading kmsg + Due to a program bug, imkmsg could not recover from an kmsg read error. + Note that recovering is possible and was intended. + Thanks to Kailash Sethuraman for the patch. - 2022-01-17: imtcp bugfix: worker threads were not properly terminated Graceful shutdown of Rsyslog could lead to segmentation faults when multiple imtcp inputs were being used. That is because the rest of the From 87e3a90d2910a2ca146863085e52a6e282ea1cf3 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Thu, 13 Jan 2022 13:24:06 +0100 Subject: [PATCH 018/134] testbench: adjust ElasticSearch startup parameters Newer versions seem to require some new defaults. Maybe this also helps with some hard-to-explain flakiness in ES test runs. --- tests/diag.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/diag.sh b/tests/diag.sh index fb2e83c70b..6ee1fdf7a7 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -2178,8 +2178,8 @@ ensure_elasticsearch_ready() { # $2, if set, is the number of additional ES instances start_elasticsearch() { - # Heap Size (limit to 128MB for testbench! defaults is way to HIGH) - export ES_JAVA_OPTS="-Xms128m -Xmx128m" + # Heap Size (limit to 256MB for testbench! defaults is way to HIGH) + export ES_JAVA_OPTS="-Xms256m -Xmx256m" dep_work_dir=$(readlink -f .dep_wrk) dep_work_es_config="es.yml" @@ -2193,7 +2193,7 @@ start_elasticsearch() { printf 'elasticsearch pid is %s\n' "$(cat $dep_work_es_pidfile)" # Wait for startup with hardcoded timeout - timeoutend=60 + timeoutend=120 timeseconds=0 # Loop until elasticsearch port is reachable or until # timeout is reached! @@ -2204,6 +2204,16 @@ start_elasticsearch() { if [ "$timeseconds" -gt "$timeoutend" ]; then echo "--- TIMEOUT ( $timeseconds ) reached!!!" + if [ ! -d $dep_work_dir/es ]; then + echo "ElasticSearch $dep_work_dir/es does not exist, no ElasticSearch debuglog" + else + echo "Dumping rsyslog-testbench.log from ElasticSearch instance $1" + echo "=========================================" + cat $dep_work_dir/es/logs/rsyslog-testbench.log + echo "=========================================" +# printf 'non-info is:\n' +# grep --invert-match '^\[.* INFO ' $dep_work_dir/kafka/logs/server.log | grep '^\[' + fi error_exit 1 fi done From e98e691ad82cde0e0ed68138cfe1f83bfeb638f6 Mon Sep 17 00:00:00 2001 From: Ameer Ghani Date: Fri, 31 Dec 2021 16:55:48 -0500 Subject: [PATCH 019/134] ompgsql: allow connection params via connection string Happy new years! Add another action parameter `conninfo` that allows specifying a postgres connection string. This enables us to use any of the postgres connection parameters, particularly `sslmode` and `sslrootcert`. Per the postgres docs, this connection string can be a URI or several key-value pairs. https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING Here's an example: ``` module(load="ompgsql") action( type="ompgsql" conninfo="postgresql://postgres:password@localhost/Syslog?sslmode=require" ) ``` This addresses https://github.com/rsyslog/rsyslog/issues/4741 by allowing the user to specify ssl options as part of the connection string. `libpq` will take care of the rest. This also addresses https://github.com/rsyslog/rsyslog/issues/4698 because `libpq` is not constrained by MAXHOSTNAMELEN. Long hostnames will work. `conninfo` can be specified in lieu of the other parameters. `ompgsql` will prioritize using `conninfo` to connect over the other parameters. Signed-off-by: Ameer Ghani --- plugins/ompgsql/ompgsql.c | 69 +++++++++++++++++++++++--------- runtime/syslogd-types.h | 11 +++-- tests/pgsql-basic-conninfo-vg.sh | 27 +++++++++++++ 3 files changed, 83 insertions(+), 24 deletions(-) create mode 100755 tests/pgsql-basic-conninfo-vg.sh diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 78bdf7d051..6eaa7a82b2 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -59,10 +59,11 @@ MODULE_CNFNAME("ompgsql") DEF_OMOD_STATIC_DATA typedef struct _instanceData { - char srv[MAXHOSTNAMELEN+1]; /* IP or hostname of DB server*/ - char dbname[_DB_MAXDBLEN+1]; /* DB name */ - char user[_DB_MAXUNAMELEN+1]; /* DB user */ - char pass[_DB_MAXPWDLEN+1]; /* DB user's password */ + char srv[MAXHOSTNAMELEN+1]; /* IP or hostname of DB server*/ + char dbname[_DB_MAXDBLEN+1]; /* DB name */ + char user[_DB_MAXUNAMELEN+1]; /* DB user */ + char pass[_DB_MAXPWDLEN+1]; /* DB user's password */ + char conninfo[_DB_MAXCONNINFOLEN+1]; /* Connection parameters or URI */ unsigned int trans_age; unsigned int trans_commit; unsigned short multi_row; @@ -78,8 +79,8 @@ typedef struct wrkrInstanceData { /* action (instance) parameters */ static struct cnfparamdescr actpdescr[] = { - { "server", eCmdHdlrGetWord, 1 }, - { "db", eCmdHdlrGetWord, 1 }, + { "server", eCmdHdlrGetWord, 0 }, + { "db", eCmdHdlrGetWord, 0 }, { "user", eCmdHdlrGetWord, 0 }, { "uid", eCmdHdlrGetWord, 0 }, { "pass", eCmdHdlrGetWord, 0 }, @@ -89,7 +90,8 @@ static struct cnfparamdescr actpdescr[] = { { "trans_age", eCmdHdlrInt, 0 }, { "serverport", eCmdHdlrInt, 0 }, { "port", eCmdHdlrInt, 0 }, - { "template", eCmdHdlrGetWord, 0 } + { "template", eCmdHdlrGetWord, 0 }, + { "conninfo", eCmdHdlrGetWord, 0 } }; @@ -190,6 +192,7 @@ static void reportDBError(wrkrInstanceData_t *pWrkrData, int bSilent) */ static rsRetVal initPgSQL(wrkrInstanceData_t *pWrkrData, int bSilent) { + int sslStatus; instanceData *pData; DEFiRet; @@ -197,23 +200,38 @@ static rsRetVal initPgSQL(wrkrInstanceData_t *pWrkrData, int bSilent) assert(pData != NULL); assert(pWrkrData->f_hpgsql == NULL); - dbgprintf("host=%s port=%d dbname=%s uid=%s\n",pData->srv, pData->port, pData->dbname, pData->user); - - /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may - * get all sorts of side effects (e.g.: backslash escapes) and warnings - */ - const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + if (strlen(pData->conninfo) > 0) { + /* Don't log the whole connection string, because it contains the DB password */ + dbgprintf("initPgSQL: using connection string provided by conninfo\n"); + pWrkrData->f_hpgsql = PQconnectdb(pData->conninfo); + } else { + dbgprintf("initPgSQL: host=%s port=%d dbname=%s uid=%s\n", pData->srv, pData->port, + pData->dbname, pData->user); + + /* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may + * get all sorts of side effects (e.g.: backslash escapes) and warnings + * + * Note: PostgreSQL versions since 9.3 have this already on by default. + */ + const char *PgConnectionOptions = "-c standard_conforming_strings=on"; + + /* Connect to database */ + char port[6]; + snprintf(port, sizeof(port), "%d", pData->port); + + pWrkrData->f_hpgsql = PQsetdbLogin(pData->srv, port, PgConnectionOptions, NULL, + pData->dbname, pData->user, pData->pass); + } - /* Connect to database */ - char port[6]; - snprintf(port, sizeof(port), "%d", pData->port); - if ((pWrkrData->f_hpgsql=PQsetdbLogin(pData->srv, port, PgConnectionOptions, NULL, - pData->dbname, pData->user, pData->pass)) == NULL) { + if (pWrkrData->f_hpgsql == NULL) { reportDBError(pWrkrData, bSilent); closePgSQL(pWrkrData); /* ignore any error we may get */ iRet = RS_RET_SUSPENDED; } + sslStatus = PQsslInUse(pWrkrData->f_hpgsql); + dbgprintf("initPgSQL: ssl status: %d\n", sslStatus); + RETiRet; } @@ -431,12 +449,27 @@ CODESTARTnewActInst free(cstr); } else if (!strcmp(actpblk.descr[i].name, "template")) { pData->tpl = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if (!strcmp(actpblk.descr[i].name, "conninfo")) { + cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + len = es_strlen(pvals[i].val.d.estr); + if(len >= sizeof(pData->conninfo)-1) { + parser_errmsg("ompgsql: conninfo parameter longer than supported " + "maximum of %d characters", (int)sizeof(pData->conninfo)-1); + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + } + memcpy(pData->conninfo, cstr, len+1); + free(cstr); } else { dbgprintf("ompgsql: program error, non-handled " "param '%s'\n", actpblk.descr[i].name); } } + if (strlen(pData->conninfo) == 0 && (strlen(pData->srv) == 0 || strlen(pData->dbname) == 0)) { + parser_errmsg("ompgsql: must provide conninfo or server and dbname"); + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + if (pData->tpl == NULL) { CHKiRet(OMSRsetEntry(*ppOMSR, 0, (uchar*) strdup(" StdPgSQLFmt"), OMSR_RQD_TPL_OPT_SQL)); } else { diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index 679f5535c0..0924a3c802 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -34,12 +34,11 @@ #define MAXFNAME 4096 /* max file pathname length */ -#define _DB_MAXDBLEN 128 /* maximum number of db */ -#define _DB_MAXUNAMELEN 128 /* maximum number of user name */ -#define _DB_MAXPWDLEN 128 /* maximum number of user's pass */ -#define _DB_DELAYTIMEONERROR 20 /* If an error occur we stop logging until - a delayed time is over */ - +#define _DB_MAXCONNINFOLEN 2048 /* maximum length connection string */ +#define _DB_MAXDBLEN 128 /* maximum number of db */ +#define _DB_MAXUNAMELEN 128 /* maximum number of user name */ +#define _DB_MAXPWDLEN 128 /* maximum number of user's pass */ +#define _DB_DELAYTIMEONERROR 20 /* If an error occur we stop logging until a delayed time is over */ /* we define features of the syslog code. This features can be used * to check if modules are compatible with them - and possible other diff --git a/tests/pgsql-basic-conninfo-vg.sh b/tests/pgsql-basic-conninfo-vg.sh new file mode 100755 index 0000000000..4a994632d4 --- /dev/null +++ b/tests/pgsql-basic-conninfo-vg.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# This file is part of the rsyslog project, released under GPLv3 + +. ${srcdir:=.}/diag.sh init + +psql -h localhost -U postgres -f testsuites/pgsql-basic.sql + +generate_conf +add_conf ' +module(load="../plugins/ompgsql/.libs/ompgsql") +if $msg contains "msgnum" then { + action(type="ompgsql" + conninfo="postgresql://postgres:testbench@localhost/syslogtest") +}' +startup_vg +injectmsg 0 5000 +shutdown_when_empty +wait_shutdown_vg +check_exit_vg + +psql -h localhost -U postgres -d syslogtest -f testsuites/pgsql-select-msg.sql -t -A >$RSYSLOG_OUT_LOG +seq_check 0 4999 + +echo cleaning up test database +psql -h localhost -U postgres -c 'DROP DATABASE IF EXISTS syslogtest;' + +exit_test From fd96e556a26361a810d07e09eba301845b6f1836 Mon Sep 17 00:00:00 2001 From: Ameer Ghani Date: Mon, 17 Jan 2022 15:10:42 -0600 Subject: [PATCH 020/134] ompgsql: PGsslInUse not supported on old distros Check for existence of PGsslInUse and fallback to PGgetssl if not present. Use of PGgetssl in this way is not supported for libpq 9.5+, but we still need to support distros that package the older libpq. This may be a bit overkill just for a debug message, but being able to easily check connection TLS state client-side is helpful. --- configure.ac | 5 +++++ plugins/ompgsql/ompgsql.c | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index f19532fa23..b4faae6dfe 100644 --- a/configure.ac +++ b/configure.ac @@ -891,6 +891,11 @@ if test "x$enable_pgsql" = "xyes"; then [-L`$PG_CONFIG --libdir`] ) ]) + AC_CHECK_LIB( + [pq], + [PGsslInUse], + [AC_DEFINE([HAVE_PGSSLINUSE], [1], [PGsslInUse function available])] + ) fi AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 6eaa7a82b2..0dd449c578 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -229,7 +229,11 @@ static rsRetVal initPgSQL(wrkrInstanceData_t *pWrkrData, int bSilent) iRet = RS_RET_SUSPENDED; } +#ifdef HAVE_PGSSLINUSE sslStatus = PQsslInUse(pWrkrData->f_hpgsql); +#else + sslStatus = PQgetssl(pWrkrData->f_hpgsql) == NULL ? 0 : 1; +#endif dbgprintf("initPgSQL: ssl status: %d\n", sslStatus); RETiRet; From 9428a9f666576be566c43340cd2827ea7de7dc12 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Wed, 26 Jan 2022 17:04:59 +0100 Subject: [PATCH 021/134] testbench: adapt contentcheck for tlscommands tests Under io / cpu stress, the OpenSSL tls error can be SSL_ERROR_SYSCALL instead of SSL_ERROR_SSL. The outcome it the same from the test perspective. closes: https://github.com/rsyslog/rsyslog/issues/4784 --- tests/imtcp-tls-ossl-basic-tlscommands.sh | 12 +++++++++--- tests/sndrcv_tls_ossl_certvalid_tlscommand.sh | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/imtcp-tls-ossl-basic-tlscommands.sh b/tests/imtcp-tls-ossl-basic-tlscommands.sh index 6418734989..6f2e760c2f 100755 --- a/tests/imtcp-tls-ossl-basic-tlscommands.sh +++ b/tests/imtcp-tls-ossl-basic-tlscommands.sh @@ -39,9 +39,15 @@ then echo "SKIP: TLS library does not support SSL_CONF_cmd" skip_test else - # Kindly check for a failed session - content_check "SSL_ERROR_SSL" - content_check "OpenSSL Error Stack:" + if content_check --check-only "SSL_ERROR_SYSCALL" + then + # Found SSL_ERROR_SYSCALL errorcode, no further check needed + exit_test + else + # Check for a SSL_ERROR_SSL error code + content_check "SSL_ERROR_SSL" + content_check "OpenSSL Error Stack:" + fi fi exit_test diff --git a/tests/sndrcv_tls_ossl_certvalid_tlscommand.sh b/tests/sndrcv_tls_ossl_certvalid_tlscommand.sh index bfa8bf660e..3e2e0e855c 100755 --- a/tests/sndrcv_tls_ossl_certvalid_tlscommand.sh +++ b/tests/sndrcv_tls_ossl_certvalid_tlscommand.sh @@ -1,7 +1,9 @@ #!/bin/bash # This file is part of the rsyslog project, released under ASL 2.0 . ${srcdir:=.}/diag.sh init +# export RSYSLOG_DEBUG="debug nologfuncflow noprintmutexaction nostdout" # start up the instances +# export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.receiver.debuglog" generate_conf add_conf ' global( defaultNetstreamDriverCAFile="'$srcdir/tls-certs/ca.pem'" @@ -64,9 +66,16 @@ if [ $ret == 0 ]; then echo "SKIP: TLS library does not support SSL_CONF_cmd" skip_test else - # Kindly check for a failed session - content_check "SSL_ERROR_SSL" - content_check "OpenSSL Error Stack:" + content_check --check-only "SSL_ERROR_SYSCALL" + ret=$? + if [ $ret == 0 ]; then + # Found SSL_ERROR_SYSCALL errorcode, no further check needed + exit_test + else + # Check for a SSL_ERROR_SSL error code + content_check "SSL_ERROR_SSL" + content_check "OpenSSL Error Stack:" + fi fi exit_test From 63acea336a37d5164141d4fa15fdac4884586a07 Mon Sep 17 00:00:00 2001 From: Mattia Barbon Date: Mon, 24 Jan 2022 15:11:28 +0100 Subject: [PATCH 022/134] omelasticsearch: add optional timeout for the index operation --- plugins/omelasticsearch/omelasticsearch.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index 0808c6054e..d7d6c68e60 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -116,6 +116,7 @@ typedef struct instanceConf_s { uchar **serverBaseUrls; int numServers; long healthCheckTimeout; + long indexTimeout; uchar *uid; uchar *pwd; uchar *authBuf; @@ -187,6 +188,7 @@ static struct cnfparamdescr actpdescr[] = { { "server", eCmdHdlrArray, 0 }, { "serverport", eCmdHdlrInt, 0 }, { "healthchecktimeout", eCmdHdlrInt, 0 }, + { "indextimeout", eCmdHdlrInt, 0 }, { "uid", eCmdHdlrGetWord, 0 }, { "pwd", eCmdHdlrGetWord, 0 }, { "searchindex", eCmdHdlrGetWord, 0 }, @@ -355,6 +357,7 @@ CODESTARTdbgPrintInstInfo dbgprintf("\ttemplate='%s'\n", pData->tplName); dbgprintf("\tnumServers=%d\n", pData->numServers); dbgprintf("\thealthCheckTimeout=%lu\n", pData->healthCheckTimeout); + dbgprintf("\tindexTimeout=%lu\n", pData->indexTimeout); dbgprintf("\tserverBaseUrls="); for(i = 0 ; i < pData->numServers ; ++i) dbgprintf("%c'%s'", i == 0 ? '[' : ' ', pData->serverBaseUrls[i]); @@ -1768,6 +1771,8 @@ curlPostSetup(wrkrInstanceData_t *const pWrkrData) PTR_ASSERT_SET_TYPE(pWrkrData, WRKR_DATA_TYPE_ES); curlSetupCommon(pWrkrData, pWrkrData->curlPostHandle); curl_easy_setopt(pWrkrData->curlPostHandle, CURLOPT_POST, 1); + curl_easy_setopt(pWrkrData->curlPostHandle, + CURLOPT_TIMEOUT_MS, pWrkrData->pData->indexTimeout); } #define CONTENT_JSON "Content-Type: application/json; charset=utf-8" @@ -1797,6 +1802,7 @@ setInstParamDefaults(instanceData *const pData) pData->serverBaseUrls = NULL; pData->defaultPort = 9200; pData->healthCheckTimeout = 3500; + pData->indexTimeout = 0; pData->uid = NULL; pData->pwd = NULL; pData->authBuf = NULL; @@ -1865,6 +1871,8 @@ CODESTARTnewActInst pData->defaultPort = (int) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "healthchecktimeout")) { pData->healthCheckTimeout = (long) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "indextimeout")) { + pData->indexTimeout = (long) pvals[i].val.d.n; } else if(!strcmp(actpblk.descr[i].name, "uid")) { pData->uid = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "pwd")) { From 5cec5dd634e0033e71ddc3ca9806f7f03819a8e0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 28 Jan 2022 12:47:07 +0100 Subject: [PATCH 023/134] rscript: literal numbers were not compared correctly This problem occurred when numbers were used in rsyslog.conf in the set statement, e.g. set $nbr = 1234; In this case, during comparisons, the number was actually interpreted as a string with digits. Thus numerical comparisons lead to unexpected results. Even more so, as in other places of the code they were treated as native numbers. This is now fixed. We cannot outrule that this causes, in border cases, change of behavior to existing configs. But it is unlikely and the previous behaviour was a clear bug and very unintuitive. This in our opinion it is justified to risk a breaking change for an expected very minor subset of installations, if any such exists at all. The fix was combined with code refactoring. We did this, because the fix itself would have been quite hard to read, and the need for refactoring became obvious. closes https://github.com/rsyslog/rsyslog/issues/4770 --- grammar/rainerscript.c | 219 +++++++---------------------------------- 1 file changed, 38 insertions(+), 181 deletions(-) diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 67c159bb1d..492eac8f10 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -2,7 +2,7 @@ * * Module begun 2011-07-01 by Rainer Gerhards * - * Copyright 2011-2019 Rainer Gerhards and Others. + * Copyright 2011-2022 Rainer Gerhards and Others. * * This file is part of the rsyslog runtime library. * @@ -3046,6 +3046,39 @@ evalStrArrayCmp(es_str_t *const estr_l, if(bMustFree2) es_deleteStr(estr_l); \ varFreeMembers(&l) +/* helper to evaluate comparison in a strcmp() like manner. Result is + * to be used for final truth value evaluation. + */ +static int eval_strcmp_like(const struct cnfexpr *__restrict__ const expr, + void *__restrict__ const usrptr, + wti_t *__restrict__ const pWti) +{ + es_str_t *__restrict__ estr_r, *__restrict__ estr_l; + int bMustFree, bMustFree2; + int64_t n_r, n_l; + int convok_r, convok_l; + struct svar r, l; /* memory for subexpression results */ + int ret; + + cnfexprEval(expr->l, &l, usrptr, pWti); + cnfexprEval(expr->r, &r, usrptr, pWti); + n_l = var2Number(&l, &convok_l); + if(convok_l) { + n_r = var2Number(&r, &convok_r); + } + if(convok_l && convok_r) { + ret = n_l - n_r; + } else { + estr_l = var2String(&l, &bMustFree); + estr_r = var2String(&r, &bMustFree2); + ret = es_strcmp(estr_l, estr_r) < 0; /*CMP*/ + if(bMustFree) es_deleteStr(estr_l); + if(bMustFree2) es_deleteStr(estr_r); + } + FREE_BOTH_RET; + return ret; +} + /* evaluate an expression. * Note that we try to avoid malloc whenever possible (because of * the large overhead it has, especially on highly threaded programs). @@ -3196,196 +3229,20 @@ cnfexprEval(const struct cnfexpr *__restrict__ const expr, FREE_BOTH_RET; break; case CMP_LE: - cnfexprEval(expr->l, &l, usrptr, pWti); - cnfexprEval(expr->r, &r, usrptr, pWti); ret->datatype = 'N'; - if(l.datatype == 'S') { - if(r.datatype == 'S') { - ret->d.n = es_strcmp(l.d.estr, r.d.estr) <= 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l <= r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree); - ret->d.n = es_strcmp(l.d.estr, estr_r) <= 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_r); - } - } - } else if(l.datatype == 'J') { - estr_l = var2String(&l, &bMustFree); - if(r.datatype == 'S') { - ret->d.n = es_strcmp(estr_l, r.d.estr) <= 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l <= r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree2); - ret->d.n = es_strcmp(estr_l, estr_r) <= 0; /*CMP*/ - if(bMustFree2) es_deleteStr(estr_r); - } - } - if(bMustFree) es_deleteStr(estr_l); - } else { - if(r.datatype == 'S') { - n_r = var2Number(&r, &convok_r); - if(convok_r) { - ret->d.n = (l.d.n <= n_r); /*CMP*/ - } else { - estr_l = var2String(&l, &bMustFree); - ret->d.n = es_strcmp(r.d.estr, estr_l) <= 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_l); - } - } else { - ret->d.n = (l.d.n <= r.d.n); /*CMP*/ - } - } - FREE_BOTH_RET; + ret->d.n = eval_strcmp_like(expr, usrptr, pWti) <= 0; break; case CMP_GE: - cnfexprEval(expr->l, &l, usrptr, pWti); - cnfexprEval(expr->r, &r, usrptr, pWti); ret->datatype = 'N'; - if(l.datatype == 'S') { - if(r.datatype == 'S') { - ret->d.n = es_strcmp(l.d.estr, r.d.estr) >= 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l >= r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree); - ret->d.n = es_strcmp(l.d.estr, estr_r) >= 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_r); - } - } - } else if(l.datatype == 'J') { - estr_l = var2String(&l, &bMustFree); - if(r.datatype == 'S') { - ret->d.n = es_strcmp(estr_l, r.d.estr) >= 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l >= r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree2); - ret->d.n = es_strcmp(estr_l, estr_r) >= 0; /*CMP*/ - if(bMustFree2) es_deleteStr(estr_r); - } - } - if(bMustFree) es_deleteStr(estr_l); - } else { - if(r.datatype == 'S') { - n_r = var2Number(&r, &convok_r); - if(convok_r) { - ret->d.n = (l.d.n >= n_r); /*CMP*/ - } else { - estr_l = var2String(&l, &bMustFree); - ret->d.n = es_strcmp(r.d.estr, estr_l) >= 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_l); - } - } else { - ret->d.n = (l.d.n >= r.d.n); /*CMP*/ - } - } - FREE_BOTH_RET; + ret->d.n = eval_strcmp_like(expr, usrptr, pWti) >= 0; break; case CMP_LT: - cnfexprEval(expr->l, &l, usrptr, pWti); - cnfexprEval(expr->r, &r, usrptr, pWti); ret->datatype = 'N'; - if(l.datatype == 'S') { - if(r.datatype == 'S') { - ret->d.n = es_strcmp(l.d.estr, r.d.estr) < 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l < r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree); - ret->d.n = es_strcmp(l.d.estr, estr_r) < 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_r); - } - } - } else if(l.datatype == 'J') { - estr_l = var2String(&l, &bMustFree); - if(r.datatype == 'S') { - ret->d.n = es_strcmp(estr_l, r.d.estr) < 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l < r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree2); - ret->d.n = es_strcmp(estr_l, estr_r) < 0; /*CMP*/ - if(bMustFree2) es_deleteStr(estr_r); - } - } - if(bMustFree) es_deleteStr(estr_l); - } else { - if(r.datatype == 'S') { - n_r = var2Number(&r, &convok_r); - if(convok_r) { - ret->d.n = (l.d.n < n_r); /*CMP*/ - } else { - estr_l = var2String(&l, &bMustFree); - ret->d.n = es_strcmp(r.d.estr, estr_l) < 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_l); - } - } else { - ret->d.n = (l.d.n < r.d.n); /*CMP*/ - } - } - FREE_BOTH_RET; + ret->d.n = eval_strcmp_like(expr, usrptr, pWti) < 0; break; case CMP_GT: - cnfexprEval(expr->l, &l, usrptr, pWti); - cnfexprEval(expr->r, &r, usrptr, pWti); ret->datatype = 'N'; - if(l.datatype == 'S') { - if(r.datatype == 'S') { - ret->d.n = es_strcmp(l.d.estr, r.d.estr) > 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l > r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree); - ret->d.n = es_strcmp(l.d.estr, estr_r) > 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_r); - } - } - } else if(l.datatype == 'J') { - estr_l = var2String(&l, &bMustFree); - if(r.datatype == 'S') { - ret->d.n = es_strcmp(estr_l, r.d.estr) > 0; /*CMP*/ - } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l > r.d.n); /*CMP*/ - } else { - estr_r = var2String(&r, &bMustFree2); - ret->d.n = es_strcmp(estr_l, estr_r) > 0; /*CMP*/ - if(bMustFree2) es_deleteStr(estr_r); - } - } - if(bMustFree) es_deleteStr(estr_l); - } else { - if(r.datatype == 'S') { - n_r = var2Number(&r, &convok_r); - if(convok_r) { - ret->d.n = (l.d.n > n_r); /*CMP*/ - } else { - estr_l = var2String(&l, &bMustFree); - ret->d.n = es_strcmp(r.d.estr, estr_l) > 0; /*CMP*/ - if(bMustFree) es_deleteStr(estr_l); - } - } else { - ret->d.n = (l.d.n > r.d.n); /*CMP*/ - } - } - FREE_BOTH_RET; + ret->d.n = eval_strcmp_like(expr, usrptr, pWti) > 0; break; case CMP_STARTSWITH: PREP_TWO_STRINGS; From 504cc9fba3b56325d75143fcbe8230730a04ab1b Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Thu, 3 Feb 2022 15:57:56 +0100 Subject: [PATCH 024/134] omelasticsearch: Fix indexSuccess impstats counter in bulkmode When bulkmode is enabled, and a batch was processed without any failures (errors is false), the code that increments the indexSuccess impstats counter was never reached. As fix, we obtain the numitems first and add them to the indexSuccess impstats counter if errors is false. This fix may solve some of the random elasticsearch testbench failures. closes: https://github.com/rsyslog/rsyslog/issues/4794 --- plugins/omelasticsearch/omelasticsearch.c | 18 ++++++++++-------- tests/es-bulk-retry.sh | 6 ++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index 0808c6054e..ed9359732c 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -877,14 +877,6 @@ parseRequestAndResponseForContext(wrkrInstanceData_t *pWrkrData,fjson_object **p int i; int numitems; fjson_object *items=NULL, *jo_errors = NULL; - int errors = 0; - - if(fjson_object_object_get_ex(replyRoot, "errors", &jo_errors)) { - errors = fjson_object_get_boolean(jo_errors); - if (!errors && pWrkrData->pData->retryFailures) { - return RS_RET_OK; - } - } /*iterate over items*/ if(!fjson_object_object_get_ex(replyRoot, "items", &items)) { @@ -897,6 +889,15 @@ parseRequestAndResponseForContext(wrkrInstanceData_t *pWrkrData,fjson_object **p numitems = fjson_object_array_length(items); + int errors = 0; + if(fjson_object_object_get_ex(replyRoot, "errors", &jo_errors)) { + errors = fjson_object_get_boolean(jo_errors); + if (!errors && pWrkrData->pData->retryFailures) { + STATSCOUNTER_ADD(indexSuccess, mutIndexSuccess, numitems); + return RS_RET_OK; + } + } + if (reqmsg) { DBGPRINTF("omelasticsearch: Entire request %s\n", reqmsg); } else { @@ -1267,6 +1268,7 @@ getDataRetryFailures(context *ctx,int itemStatus,char *request,char *response, response); } } + need_free_omes = 0; CHKiRet(msgAddJSON(msg, (uchar*)".omes", omes, 0, 0)); MsgSetRuleset(msg, ctx->retryRuleset); diff --git a/tests/es-bulk-retry.sh b/tests/es-bulk-retry.sh index e94d0b78d2..b4bff85559 100755 --- a/tests/es-bulk-retry.sh +++ b/tests/es-bulk-retry.sh @@ -1,8 +1,13 @@ #!/bin/bash # This file is part of the rsyslog project, released under ASL 2.0 . ${srcdir:=.}/diag.sh init +#export RSYSLOG_DEBUG="debug nologfuncflow noprintmutexaction nostdout" +#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debuglog" + export ES_PORT=19200 export NUMMESSAGES=100 + +# export RSTB_GLOBAL_INPUT_SHUTDOWN_TIMEOUT=120000 override_test_timeout 120 #export USE_VALGRIND="YES" # to enable this to run under valgrind ensure_elasticsearch_ready --no-start @@ -152,6 +157,7 @@ badarg=50 injectmsg 0 $NUMMESSAGES ./msleep 1500; cat $RSYSLOG_OUT_LOG # debuging - we sometimes miss 1 message wait_content '"response.success": 50' $RSYSLOG_DYNNAME.spool/es-stats.log +wait_content '"response.badargument": 50' $RSYSLOG_DYNNAME.spool/es-stats.log shutdown_when_empty wait_shutdown es_getdata $NUMMESSAGES $ES_PORT From 34c40fbba3322f75ba7664447c4a10bb06e3c159 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Feb 2022 09:23:56 +0100 Subject: [PATCH 025/134] maintain ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8cf888c5a2..0b239db0af 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-02-04: omelasticsearch bugfix: indexSuccess impstats counter in bulkmode wrong + When bulkmode is enabled, and a batch was processed without any + failures (errors is false), the code that increments the indexSuccess + impstats counter was never reached. + closes: https://github.com/rsyslog/rsyslog/issues/4794 - 2022-01-17: imkmsg bugfix: effectively disabled input on error reading kmsg Due to a program bug, imkmsg could not recover from an kmsg read error. Note that recovering is possible and was intended. From 36d70a00fd5f671bbf058a9734f27c6fc2c19843 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Feb 2022 14:56:57 +0100 Subject: [PATCH 026/134] maintain ChangeLog --- ChangeLog | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0b239db0af..1a9d54ee7a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,19 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-02-04: rscript bugfix: literal numbers were not compared correctly + This problem occurred when numbers were used in rsyslog.conf in + the set statement, e.g. + set $nbr = 1234; + In this case, during comparisons, the number was actually interpreted + as a string with digits. Thus numerical comparisons lead to unexpected + results. Even more so, as in other places of the code they were + treated as native numbers. + This is now fixed. We cannot outrule that this causes, in border cases, + change of behavior to existing configs. But it is unlikely and the + previous behaviour was a clear bug and very unintuitive. This in our + opinion it is justified to risk a breaking change for an expected + very minor subset of installations, if any such exists at all. + closes https://github.com/rsyslog/rsyslog/issues/4770 - 2022-02-04: omelasticsearch bugfix: indexSuccess impstats counter in bulkmode wrong When bulkmode is enabled, and a batch was processed without any failures (errors is false), the code that increments the indexSuccess From 7836d298b5bac8b059f5e6a7243a2939a9e6ff2c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 3 Feb 2022 12:34:29 +0100 Subject: [PATCH 027/134] testbench: test cases for rscript comparisons This covers failure cases from practice which were not yet covered. see also: https://github.com/rsyslog/rsyslog/issues/4770 --- tests/Makefile.am | 10 +++++ tests/rscript_number_comparison_LE-vg.sh | 3 ++ tests/rscript_number_comparison_LE.sh | 24 ++++++++++ tests/rscript_number_comparison_LT.sh | 24 ++++++++++ tests/rscript_number_comparison_all-vg.sh | 3 ++ tests/rscript_number_comparison_all.sh | 53 +++++++++++++++++++++++ 6 files changed, 117 insertions(+) create mode 100755 tests/rscript_number_comparison_LE-vg.sh create mode 100755 tests/rscript_number_comparison_LE.sh create mode 100755 tests/rscript_number_comparison_LT.sh create mode 100755 tests/rscript_number_comparison_all-vg.sh create mode 100755 tests/rscript_number_comparison_all.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 3caf70b6f0..ed06736dd1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -431,6 +431,9 @@ TESTS += \ rscript_lt_var.sh \ rscript_ne.sh \ rscript_ne_var.sh \ + rscript_number_comparison_all.sh \ + rscript_number_comparison_LE.sh \ + rscript_number_comparison_LT.sh \ rscript_num2ipv4.sh \ rscript_int2Hex.sh \ rscript_trim.sh \ @@ -531,6 +534,8 @@ TESTS += \ udp-msgreduc-orgmsg-vg.sh \ tcp-msgreduc-vg.sh \ rscript_field-vg.sh \ + rscript_number_comparison_all-vg.sh \ + rscript_number_comparison_LE-vg.sh \ unused_lookup_table-vg.sh \ lookup_table-vg.sh \ lookup_table_no_hup_reload-vg.sh \ @@ -1841,6 +1846,11 @@ EXTRA_DIST= \ rscript_gt_var.sh \ rscript_ne.sh \ rscript_ne_var.sh \ + rscript_number_comparison_all.sh \ + rscript_number_comparison_all-vg.sh \ + rscript_number_comparison_LE.sh \ + rscript_number_comparison_LE-vg.sh \ + rscript_number_comparison_LT.sh \ rscript_num2ipv4.sh \ rscript_int2Hex.sh \ rscript_trim.sh \ diff --git a/tests/rscript_number_comparison_LE-vg.sh b/tests/rscript_number_comparison_LE-vg.sh new file mode 100755 index 0000000000..5493137502 --- /dev/null +++ b/tests/rscript_number_comparison_LE-vg.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export USE_VALGRIND="YES" +source ${srcdir:-.}/rscript_number_comparison_LE.sh diff --git a/tests/rscript_number_comparison_LE.sh b/tests/rscript_number_comparison_LE.sh new file mode 100755 index 0000000000..acb9865ff0 --- /dev/null +++ b/tests/rscript_number_comparison_LE.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# added by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +template(name="outfmt" type="list") { + property(name="$!result") + constant(value="\n") +} +set $!lower_nr = 1111; +set $!higher_nr = 2222; + +if $!lower_nr <= $!higher_nr + then { set $!result = "RIGHT"; } + else { set $!result = "WRONG"; } + +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") +' +startup +shutdown_when_empty +wait_shutdown +content_check 'RIGHT' +exit_test diff --git a/tests/rscript_number_comparison_LT.sh b/tests/rscript_number_comparison_LT.sh new file mode 100755 index 0000000000..5cb729a3c1 --- /dev/null +++ b/tests/rscript_number_comparison_LT.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# added 2022-01-27 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +template(name="outfmt" type="list") { + property(name="$!result") + constant(value="\n") +} +set $!lower_nr = 1111; +set $!higher_nr = 2222; + +if $!higher_nr < $!lower_nr + then { set $!result = "WRONG"; } + else { set $!result = "RIGHT"; } + +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") +' +startup +shutdown_when_empty +wait_shutdown +content_check 'RIGHT' +exit_test diff --git a/tests/rscript_number_comparison_all-vg.sh b/tests/rscript_number_comparison_all-vg.sh new file mode 100755 index 0000000000..944d32003b --- /dev/null +++ b/tests/rscript_number_comparison_all-vg.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export USE_VALGRIND="YES" +source ${srcdir:-.}/rscript_number_comparison_all.sh diff --git a/tests/rscript_number_comparison_all.sh b/tests/rscript_number_comparison_all.sh new file mode 100755 index 0000000000..4b2a7def0f --- /dev/null +++ b/tests/rscript_number_comparison_all.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# added by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +template(name="outfmt" type="list") { + property(name="$!result") + constant(value="\n") +} +set $!lower_nr = 1111; +set $!higher_nr = 2222; + +if $!lower_nr <= $!higher_nr + then { set $!result = "<= RIGHT"; } + else { set $!result = "<= WRONG"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") + +if $!lower_nr < $!higher_nr + then { set $!result = "< RIGHT"; } + else { set $!result = "< WRONG"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") + +if $!higher_nr >= $!lower_nr + then { set $!result = ">= RIGHT"; } + else { set $!result = ">= WRONG"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") + +if $!higher_nr > $!lower_nr + then { set $!result = "> RIGHT"; } + else { set $!result = "> WRONG"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") + +if $!higher_nr != $!lower_nr + then { set $!result = "!= RIGHT"; } + else { set $!result = "!= WRONG"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") + +if $!higher_nr == $!lower_nr + then { set $!result = "== WRONG"; } + else { set $!result = "== RIGHT"; } +action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") +' +startup +shutdown_when_empty +wait_shutdown +content_check '<= RIGHT' +content_check '>= RIGHT' +content_check '!= RIGHT' +content_check '== RIGHT' +content_check '< RIGHT' +content_check '> RIGHT' +exit_test From 1a403f987bf8c51479991401969ba97b1da6f67d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 20 Sep 2021 12:50:59 +0200 Subject: [PATCH 028/134] testbench: restore original order of ES test execution The order of execution was changed to a less optimal (more startups, thus slower) order to work-around a testbench issue. This has been fixed and so we can restore the original order. --- .github/workflows/run_ubuntu_20_distcheck.yml | 8 +++----- tests/Makefile.am | 11 ++++------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run_ubuntu_20_distcheck.yml b/.github/workflows/run_ubuntu_20_distcheck.yml index bca318a1d4..85789818bf 100644 --- a/.github/workflows/run_ubuntu_20_distcheck.yml +++ b/.github/workflows/run_ubuntu_20_distcheck.yml @@ -59,16 +59,14 @@ jobs: echo PWD: $(pwd) ls -la ls -ld rsyslog-* - sudo chmod -R go+rw . + sudo chmod -vR go+rw . ls -la ls -ld rsyslog-* - echo FIND ..: + echo FIND TRS: find .. -name .dep_wrk - echo FIND: - find . -name .dep_wrk echo ls of curr dir: echo END CI DEBUGGING rm -rf $(find . -name .dep_wrk) - sudo chmod -R a+r . + sudo chmod -vR a+r . devtools/gather-check-logs.sh cat failed-tests.log diff --git a/tests/Makefile.am b/tests/Makefile.am index 3caf70b6f0..2060fff7ac 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -62,16 +62,13 @@ TESTS += \ es-duplicated-ruleset.sh # the following test need to be serialized: TESTS += \ - es-basic.sh \ es-basic-es6.0.sh \ es-basic-es7.14.sh \ + es-basic.sh \ es-basic-bulk.sh -#es-basic-es6.0.log: es-basic.log -#es-basic-es7.14.log: es-basic-es6.0.log -es-basic-es7.14.log: es-basic.log -es-basic-es6.0.log: es-basic-es7.14.log -es-basic-bulk.log: es-basic-es6.0.log -#es-basic-bulk.log: es-basic-es7.14.log +es-basic-es7.14.log: es-basic-es6.0.log +es-basic.log: es-basic-es7.14.log +es-basic-bulk.log: es-basic.log es-basic-server.log: es-basic-bulk.log # special "test" for stopping ES once all ES tests are done From 46c79edc53cdfa8e73c4690d39017315f9c888aa Mon Sep 17 00:00:00 2001 From: Masahiro Matsuya Date: Wed, 9 Feb 2022 16:36:24 +0900 Subject: [PATCH 029/134] imfile: Remove inotify watch descriptor on inode change detected --- plugins/imfile/imfile.c | 1 + tests/imfile-rename.sh | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 29fe8155e3..f79c10fb4c 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -1040,6 +1040,7 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted) } #ifdef HAVE_INOTIFY_INIT if(act->wd != -1) { + inotify_rm_watch(ino_fd, act->wd); wdmapDel(act->wd); } #endif diff --git a/tests/imfile-rename.sh b/tests/imfile-rename.sh index 28ea69dd47..6d249d3365 100755 --- a/tests/imfile-rename.sh +++ b/tests/imfile-rename.sh @@ -42,6 +42,11 @@ if $msg contains "msgnum:" then file=`echo $RSYSLOG_OUT_LOG` template="outfmt" ) +if $msg contains "imfile:" then + action( + type="omfile" + file="'$RSYSLOG_DYNNAME.errmsgs'" + ) ' # generate input file first. @@ -58,6 +63,18 @@ wait_file_lines $RSYSLOG_OUT_LOG $TESTMESSAGES $RETRIES mv $RSYSLOG_DYNNAME.input.1.log rsyslog.input.2.log ./msleep 500 + +# Write into the renamed file +echo 'testmessage1 +testmessage2' >> rsyslog.input.2.log + +./msleep 500 + +if grep "imfile: internal error? inotify provided watch descriptor" < "$RSYSLOG_DYNNAME.errmsgs" ; then + echo "Error: inotify event from renamed file" + exit 1 +fi + # generate some more input into moved file ./inputfilegen -m $TESTMESSAGES -i $TESTMESSAGES >> $RSYSLOG_DYNNAME.input.2.log ls -l $RSYSLOG_DYNNAME.input* From 6f7347342a4e920a34ea4cb1d86e95c9adee48ae Mon Sep 17 00:00:00 2001 From: alakatos Date: Fri, 11 Feb 2022 10:57:55 +0100 Subject: [PATCH 030/134] Make action counter part of the config --- action.c | 13 +++---------- action.h | 3 --- runtime/rsconf.c | 3 ++- runtime/rsconf.h | 9 ++++++++- runtime/wti.c | 11 ++++++----- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/action.c b/action.c index 90bfa53287..695967d46a 100644 --- a/action.c +++ b/action.c @@ -184,13 +184,6 @@ typedef struct configSettings_s { static configSettings_t cs; /* our current config settings */ -/* the counter below counts actions created. It is used to obtain unique IDs for the action. They - * should not be relied on for any long-term activity (e.g. disk queue names!), but they are nice - * to have during one instance of an rsyslogd run. For example, I use them to name actions when there - * is no better name available. - */ -int iActionNbr = 0; - /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfparamdescr[] = { { "name", eCmdHdlrGetWord, 0 }, /* legacy: actionname */ @@ -412,14 +405,14 @@ rsRetVal actionConstruct(action_t **ppThis) pThis->bReportSuspensionCont = -1; /* indicate "not yet set" */ pThis->bCopyMsg = 0; pThis->tLastOccur = datetime.GetTime(NULL); /* done once per action on startup only */ - pThis->iActionNbr = iActionNbr; + pThis->iActionNbr = loadConf->actions.iActionNbr; pthread_mutex_init(&pThis->mutErrFile, NULL); pthread_mutex_init(&pThis->mutAction, NULL); pthread_mutex_init(&pThis->mutWrkrDataTable, NULL); INIT_ATOMIC_HELPER_MUT(pThis->mutCAS); /* indicate we have a new action */ - ++iActionNbr; + loadConf->actions.iActionNbr++; finalize_it: *ppThis = pThis; @@ -1618,7 +1611,7 @@ actionCommitAllDirect(wti_t *__restrict__ const pWti) int i; action_t *pAction; - for(i = 0 ; i < iActionNbr ; ++i) { + for(i = 0 ; i < runConf->actions.iActionNbr ; ++i) { pAction = pWti->actWrkrInfo[i].pAction; if(pAction == NULL) continue; diff --git a/action.h b/action.h index 403226bc76..00de1cd6b9 100644 --- a/action.h +++ b/action.h @@ -111,7 +111,4 @@ void actionCommitAllDirect(wti_t *pWti); void actionRemoveWorker(action_t *const pAction, void *const actWrkrData); void releaseDoActionParams(action_t * const pAction, wti_t * const pWti, int action_destruct); -/* external data */ -extern int iActionNbr; - #endif /* #ifndef ACTION_H_INCLUDED */ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index ef5a52588f..6504708efc 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -165,6 +165,7 @@ static void cnfSetDefaults(rsconf_t *pThis) pThis->templates.last = NULL; pThis->templates.lastStatic = NULL; pThis->actions.nbrActions = 0; + pThis->actions.iActionNbr = 0; pThis->globals.pszWorkDir = NULL; pThis->globals.bDropMalPTRMsgs = 0; pThis->globals.operatingStateFile = NULL; @@ -1447,7 +1448,7 @@ load(rsconf_t **cnf, uchar *confFile) ABORT_FINALIZE(RS_RET_NO_ACTIONS); } tellLexEndParsing(); - DBGPRINTF("Number of actions in this configuration: %d\n", iActionNbr); + DBGPRINTF("Number of actions in this configuration: %d\n", loadConf->actions.iActionNbr); CHKiRet(tellCoreConfigLoadDone()); tellModulesConfigLoadDone(); diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 078f5dc866..7d0fd85619 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -206,7 +206,14 @@ struct templates_s { struct actions_s { - unsigned nbrActions; /* number of actions */ + /* number of active actions */ + unsigned nbrActions; + /* number of actions created. It is used to obtain unique IDs for the action. They + * should not be relied on for any long-term activity (e.g. disk queue names!), but they are nice + * to have during one instance of an rsyslogd run. For example, I use them to name actions when there + * is no better name available. + */ + int iActionNbr; }; diff --git a/runtime/wti.c b/runtime/wti.c index d3ca74a173..290bf96166 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -47,6 +47,7 @@ #include "glbl.h" #include "action.h" #include "atomic.h" +#include "rsconf.h" /* static data */ DEFobjStaticHelpers @@ -275,13 +276,13 @@ wtiConstructFinalize(wti_t *pThis) ISOBJ_TYPE_assert(pThis, wti); DBGPRINTF("%s: finalizing construction of worker instance data (for %d actions)\n", - wtiGetDbgHdr(pThis), iActionNbr); + wtiGetDbgHdr(pThis), runConf->actions.iActionNbr); /* initialize our thread instance descriptor (no concurrency here) */ pThis->bIsRunning = WRKTHRD_STOPPED; /* must use calloc as we need zero-init */ - CHKmalloc(pThis->actWrkrInfo = calloc(iActionNbr, sizeof(actWrkrInfo_t))); + CHKmalloc(pThis->actWrkrInfo = calloc(runConf->actions.iActionNbr, sizeof(actWrkrInfo_t))); if(pThis->pWtp == NULL) { dbgprintf("wtiConstructFinalize: pWtp not set, this may be intentional\n"); @@ -316,7 +317,7 @@ wtiWorkerCancelCleanup(void *arg) DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); pWtp->pfObjProcessed(pWtp->pUsr, pThis); DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); - + } @@ -445,7 +446,7 @@ wtiWorker(wti_t *__restrict__ const pThis) d_pthread_mutex_unlock(pWtp->pmutUsr); DBGPRINTF("DDDD: wti %p: worker cleanup action instances\n", pThis); - for(i = 0 ; i < iActionNbr ; ++i) { + for(i = 0 ; i < runConf->actions.iActionNbr ; ++i) { wrkrInfo = &(pThis->actWrkrInfo[i]); dbgprintf("wti %p, action %d, ptr %p\n", pThis, i, wrkrInfo->actWrkrData); if(wrkrInfo->actWrkrData != NULL) { @@ -496,7 +497,7 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, const size_t lenMsg) ISOBJ_TYPE_assert(pThis, wti); assert(pszMsg != NULL); - + if(lenMsg < 1) ABORT_FINALIZE(RS_RET_PARAM_ERROR); From 8ee3194b4cd6498549817adbef5e43ded8fb59e9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 14 Feb 2022 17:35:55 +0100 Subject: [PATCH 031/134] maintain ChangeLog --- ChangeLog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1a9d54ee7a..ad203bacd7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,16 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +- 2022-02-14: imfile bugfix: remove cause for "internal error message" (not causing harm) + When any message is output into a renamed input file, rsyslogd output the following: + message. + imfile: internal error? inotify provided watch descriptor 7 which we could not find + in our tables - ignored + When rsyslogd detects the inode change, it deletes the entry from wdmap[]. But, + the watch descriptor is not removed. Some application like sssd outputs some messages + (like "HUP signal was received!!") after HUP signal is received and before switching + into the new log file. And, the above messages can be output every log rotation. + This situation is now resolved. + Thanks to Masahiro Matsuya for the patch. - 2022-02-04: rscript bugfix: literal numbers were not compared correctly This problem occurred when numbers were used in rsyslog.conf in the set statement, e.g. From de068aa22ca7bfb1971563c572609f82062d10e5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 14 Feb 2022 17:41:29 +0100 Subject: [PATCH 032/134] prepare for 8.2022.0 release --- ChangeLog | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ad203bacd7..69af33342d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ ---------------------------------------------------------------------------------------- -Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-?? +Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-15 - 2022-02-14: imfile bugfix: remove cause for "internal error message" (not causing harm) When any message is output into a renamed input file, rsyslogd output the following: message. diff --git a/configure.ac b/configure.ac index b4faae6dfe..462213ff04 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2202.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2202.0],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 21, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 12, [month part of real rsyslog version]) # UPDATE on release From 6569133c759dc75583c49849919903eb977b7144 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Thu, 17 Feb 2022 10:54:12 +0100 Subject: [PATCH 033/134] Typo fixes (#4801) * typo fix: ambigious -> ambiguous * typo fix: aquire -> acquire * typo fix: assgined -> assigned * typo fix: cancelation -> cancellation * typo fix: childs -> children * typo fix: configuraton -> configuration * typo fix: delemiter -> delimiter * typo fix: forwardig -> forwarding * typo fix: initializiation -> initialization * typo fix: intializing -> initializing * typo fix: lengh -> length * typo fix: mesage -> message * typo fix: occured -> occurred * typo fix: occurence -> occurrence * typo fix: paramter -> parameter * typo fix: remaing -> remaining * typo fix: resetted -> reset * typo fix: suppored -> supported * typo fix: Sytem -> System * typo fix: uncommited -> uncommitted * typo fix: depricated -> deprecated * typo fix: stoping -> stopping * type fix: allow to -> allow one to --- .shellcheck.yaml | 2 +- ChangeLog | 108 +++++++++--------- action.c | 10 +- action.h | 6 +- contrib/imbatchreport/imbatchreport.c | 4 +- contrib/imkmsg/kmsg.c | 2 +- contrib/mmgrok/mmgrok.c | 2 +- contrib/omfile-hardened/omfile-hardened.c | 2 +- .../pmaixforwardedfrom/pmaixforwardedfrom.c | 2 +- contrib/pmcisconames/pmcisconames.c | 2 +- grammar/rainerscript.c | 12 +- plugins/imfile/imfile.c | 2 +- plugins/imjournal/imjournal.c | 2 +- plugins/impstats/statslog-analyzer.py | 4 +- plugins/impstats/svg.jquery.js | 2 +- plugins/imptcp/imptcp.c | 2 +- plugins/imudp/imudp.c | 2 +- plugins/imuxsock/imuxsock.c | 4 +- plugins/omgssapi/omgssapi.c | 2 +- plugins/omkafka/omkafka.c | 2 +- plugins/omlibdbi/omlibdbi.c | 4 +- plugins/ommongodb/ommongodb.c | 2 +- plugins/ommysql/ommysql.c | 6 +- plugins/ompgsql/ompgsql.c | 4 +- plugins/omsnmp/omsnmp.h | 6 +- runtime/cfsysline.c | 2 +- runtime/conf.c | 4 +- runtime/datetime.c | 10 +- runtime/debug.c | 2 +- runtime/glbl.c | 2 +- runtime/lookup.c | 2 +- runtime/modules.c | 2 +- runtime/msg.c | 18 +-- runtime/net.c | 2 +- runtime/netstrm.c | 2 +- runtime/nsd_ossl.c | 4 +- runtime/nsd_ptcp.c | 2 +- runtime/obj-types.h | 2 +- runtime/obj.c | 4 +- runtime/queue.c | 6 +- runtime/rsyslog.c | 2 +- runtime/rsyslog.h | 16 +-- runtime/srutils.c | 2 +- runtime/stream.c | 2 +- runtime/stringbuf.c | 4 +- runtime/var.c | 2 +- runtime/wti.c | 18 +-- runtime/wti.h | 2 +- runtime/wtp.c | 2 +- sample.conf | 2 +- template.c | 4 +- template.h | 2 +- tests/asynwr_deadlock2.sh | 2 +- tests/cfg1.cfgtest | 2 +- tests/cfg2.cfgtest | 2 +- tests/cfg3.cfgtest | 2 +- tests/diag.sh | 2 +- tests/historical/DiagTalker.java | 2 +- tests/imdocker-new-logs-from-start.sh | 2 +- tests/tcpflood.c | 2 +- tests/testbench.h | 2 +- tools/omfwd.c | 4 +- tools/pmrfc3164.c | 2 +- tools/rsyslog.conf.5 | 2 +- tools/rsyslogd.8 | 2 +- tools/rsyslogd.c | 2 +- 66 files changed, 174 insertions(+), 174 deletions(-) diff --git a/.shellcheck.yaml b/.shellcheck.yaml index bdf0eac7e4..b5378123e1 100644 --- a/.shellcheck.yaml +++ b/.shellcheck.yaml @@ -3,7 +3,7 @@ ignored: - SC2016 # shellcheck does not properly understand our intent, the issue # is raised e.g. here in config file creation: # binary="'$RSYSLOG_DYNNAME'.omprog-restart-terminated-bin.sh" - # we cannot explicitely permit this on each occurence. + # we cannot explicitely permit this on each occurrence. # things we do not (currently) care about - SC2046 diff --git a/ChangeLog b/ChangeLog index ad203bacd7..95f2bda28e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -772,7 +772,7 @@ Scheduled Release 8.2006.0 (aka 2020.06) 2020-06-23 Also adds two testbench tests for this case. closes: https://github.com/rsyslog/rsyslog/issues/4281 - 2020-05-12: ompipe bugfix: race during HUP - When HUP was received, the write mutex was not aquired. This could + When HUP was received, the write mutex was not acquired. This could lead to unexpected invalidation of the output file descriptor. Thanks to Julien Thomas for alerting us on this issue. see also https://github.com/rsyslog/rsyslog/pull/4136#issuecomment-578326278 @@ -1023,7 +1023,7 @@ Scheduled Release 8.2001.0 (aka 2020.01) 2020-01-14 kafka component download improvements, and prevent dangling left-over test tool instances from aborted tests - 2020-01-07: tls subsystem bugfix: default for permitExpiredCerts was invalidly "on" - The problem occured with commit 3d9b8df in December 2018 and went into + The problem occurred with commit 3d9b8df in December 2018 and went into scheduled stable 8.1901.0. Unfortunately, the change in default was not detected until a year later. This commit re-enables the previous default ("off"), which is also the only sensible default from a security PoV. Unfortunately, new 2019 @@ -1623,7 +1623,7 @@ an easy way to identify which changes went into the respective daily stable. - core bugfix: message duplication copied incorrect timestamp MsgDup() placed timereported into timegenerated property, resulting in invalid property values. Original timegenerated was lost. This - occured always when a message needed to be duplicated. Most + occurred always when a message needed to be duplicated. Most importantly this is the case when queues are used. closes https://github.com/rsyslog/rsyslog/issues/3716 - core bugfix: segfault on startup depending on queue file names @@ -1748,7 +1748,7 @@ Version 8.1905.0 (aka 2019.05) 2019-05-28 see also https://github.com/rsyslog/rsyslog/issues/3639 - config processing bugfix: error messages if config.enabled="off" is used Using config.enabled="off" could lead to error messages on - "parameter xxx not known", which were invalid. They occured + "parameter xxx not known", which were invalid. They occurred because the config handler expected them to be used, which was not the case due to being disabled. This commit fixes that issue. @@ -1863,7 +1863,7 @@ Version 8.1904.0 (aka 2019.04) 2019-04-16 out some basic container metadata as id, name, image, labels. Thanks to Nelson Yen for the contribution. - mmtaghostname: new contributed module - This module allows to force hostname after parsing to the localhostname of + This module allows one to force hostname after parsing to the localhostname of rsyslog and/or add a tag to messages received from input modules without tag parameter. Thanks to Philippe Duveau for the contribution. @@ -1898,7 +1898,7 @@ Version 8.1904.0 (aka 2019.04) 2019-04-16 - pmnormalize: fix memory leaks, improve tests This patch fixes a set of problems plus provides more and enhanced tests for the module. - Most important problem was a memory leak that occured when a message + Most important problem was a memory leak that occurred when a message could not be passed at all. For each message that could not be parsed memory of at least the size the message is leaked. Depending on traffic pattern this can quickly lead to OOM. Note, however, that @@ -1973,7 +1973,7 @@ Version 8.1903.0 (aka 2019.03) 2019-03-05 - AIX port: add AIX linking extensions on many plugins and contributions to allow building them on this os. - template: add Time-Related System Property $wday which is the day of week - This allow to get a week based rotation of log as AIX does. + This allows one to get a week based rotation of log as AIX does. - ksi subsystem: add high availability mode Note: ksi subsystem now REQUIRES libksi 3.19.0 or above Thanks to Allan Park for the patch. @@ -2103,7 +2103,7 @@ Version 8.1901.0 (aka 2019.01) 2019-01-22 we add configure switch --enable-helgrind. We need to turn helgrind off when we use clang coverage instrumentation. The instrumentation injects mt-unsafe counter updates which we seem to be unable to suppress. - Note: for gcc this was possible, because they all occured in a utility + Note: for gcc this was possible, because they all occurred in a utility function. For clang, they are inlined so we get many -and changing- violations. see also https://github.com/rsyslog/rsyslog/issues/3361#issuecomment-450502569 - imzmq3/omzmq3: remove modules @@ -2658,7 +2658,7 @@ Version 8.38.0 [v8-stable] 2018-09-18 - another two major leaks of similar magnitude could occur if "fromhost-ip" and/or "fromhost" properties were set - minor leaks upon termination. these were unproblematic as - static and only occured immediately before shutdown. + static and only occurred immediately before shutdown. But they triggered memory debugger errors. - fixed test which did not check for mem leaks albeit it should - core invalid handling of the "fromhost" property, if set via @@ -3264,7 +3264,7 @@ Version 8.35.0 [v8-stable] 2018-05-15 Thanks to github user jvymazal for the patch. - imuxsock: replace select() call by poll() While extremely unlikely, imuxsock could abort if a file descriptor - > 1024 was received during the startup phase (never occured in + > 1024 was received during the startup phase (never occurred in practice, but theoretically could if imfile monitored a large number of files and were loaded before imuxsock - and maybe other strange cases). @@ -3544,7 +3544,7 @@ Version 8.33.1 [v8-stable] 2018-03-06 resulting in make check failures if build from tarball. Thanks to Thomas D. (whissi), and Michael Biebl for alerting us on the problem, providing advise and some of the patches. We also added addt'l - patches ourselves. The problem occured as the CI check for tarball + patches ourselves. The problem occurred as the CI check for tarball completeness was more or less disabled a couple of weeks ago, which unfortunately went unnoticed. We have also applied some more safeguards to detect such problems in the future. @@ -3618,7 +3618,7 @@ Version 8.33.0 [v8-stable] 2018-02-20 - core bugfix: misaddressing in external command parser This parser is used whenever a module (e.g. omprog) needs to process command lines. If command parameters were given, memory misaddressing - occured. This could lead to a segfault. + occurred. This could lead to a segfault. This is a regression in 8.32.0. closes https://github.com/rsyslog/rsyslog/issues/2408 - core bugfix: small memory leak in external command parser @@ -3877,7 +3877,7 @@ Version 8.32.0 [v8-stable] 2018-01-09 This should not cause any change of behavior, but is worth noting in case we see a regression not caught by the CI system. - mmsnmptrapd bugfix: potential misaddressing - This only occured in situations when the system was totally out of memory. + This only occurred in situations when the system was totally out of memory. - imkafka: fix potential small resource leak If rdkafka handle cannot fully populated, cleanup is added. Previously, we could leak a handle if e.g. no brokers were available. Note that this was @@ -4055,7 +4055,7 @@ Version 8.31.0 [v8-stable] 2017-11-28 respected, but no "hard error". - core/template bugfix: potential NULL pointer access at config load could happen if the config was loaded and the parameters could not properly - be processed. If so, this occured only during the startup phase. + be processed. If so, this occurred only during the startup phase. Detected by Coverity scan, CID 185318 - core/json var subsystem bugfix: segfault in MsgSetPropsViaJSON Invalid libfastjson API use lead to double-free. This was most importantly @@ -4570,7 +4570,7 @@ Version 8.29.0 [v8-stable] 2017-08-08 messages from rsyslog, but that should actually improve the end user experience. - core: add parameters debug.file and debug.whitelist - allows to generate debug log output only of specific files + allows one to generate debug log output only of specific files Background information available at: https://www.linkedin.com/pulse/improving-rsyslog-debug-output-jan-gerhards - core/net.c: improve UDP socket creation error messages @@ -4931,7 +4931,7 @@ Version 8.26.0 [v8-stable] 2017-04-04 - bugfix omsnmp: improper handling of new-style configuration parameters Thanks to Radovan Sroka for the patch. - bugfix: rsyslog identifies itself as "liblogging-stdlog" in internal messages - This occured when liblogging-stdlog was used, and was used by default (without + This occurred when liblogging-stdlog was used, and was used by default (without explicit configuration). This is a regression of the new default, which does not correctly call stdlog_open() in the default case. closes https://github.com/rsyslog/rsyslog/issues/1442 @@ -5158,7 +5158,7 @@ Version 8.23.0 [v8-stable] 2016-11-15 This permits to keep batches below ES-configured limits. Thanks to github user barakshechter for the patch. - omelasticsearch: high availability addressing of ElasticSearch cluster - allow to specify an array of servers, which is tried until a working + allow one to specify an array of servers, which is tried until a working one is found (and given up only if none works). Thanks to github user barakshechter for the patch. - omelasticsearch: make compatible with ElasticSearch 2.x and 5.x @@ -7123,7 +7123,7 @@ Version 7.5.5 [devel] 2013-10-16 - now requires libestr 0.1.7 as early versions had a nasty bug in string comparisons - bugfix: mmanon did not detect all IP addresses in rewrite mode - The problem occured if two IPs were close to each other and the first one + The problem occurred if two IPs were close to each other and the first one was shrunk. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=485 Thanks to micah-at-riseup.net for reporting this bug @@ -7389,7 +7389,7 @@ Version 7.4.5 [v7.4-stable] 2013-10-22 support atomic instructions - in this case the associated mutexes also did not get properly initialized. - bugfix: mmanon did not detect all IP addresses in rewrite mode - The problem occured if two IPs were close to each other and the first one + The problem occurred if two IPs were close to each other and the first one was shrunk. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=485 Thanks to micah-at-riseup.net for reporting this bug @@ -7782,7 +7782,7 @@ Version 7.3.5 [devel] 2012-12-19 as the code path is smaller and requires slightly less memory writes. That probably outperforms the extra locking overhead (which in the low-end case always happens in user space, without need for kernel - support as we can always directly aquire the lock - there is no + support as we can always directly acquire the lock - there is no contention at all). - build system cleanup (thanks to Michael Biebl for this!) - bugfix: omelasticsearch did not properly compile on some platforms @@ -8020,7 +8020,7 @@ Version 7.2.4 [v7-stable] 2012-12-07 as the code path is smaller and requires slightly less memory writes. That probably outperforms the extra locking overhead (which in the low-end case always happens in user space, without need for kernel - support as we can always directly aquire the lock - there is no + support as we can always directly acquire the lock - there is no contention at all). ---------------------------------------------------------------------------- Version 7.2.3 [v7-stable] 2012-10-21 @@ -8284,7 +8284,7 @@ Version 6.6.1 [v6-stable] 2012-10-?? as the code path is smaller and requires slightly less memory writes. That probably outperforms the extra locking overhead (which in the low-end case always happens in user space, without need for kernel - support as we can always directly aquire the lock - there is no + support as we can always directly acquire the lock - there is no contention at all). - bugfix: invalid DST handling under Solaris Thanks to Scott Severtson for the patch. @@ -8857,7 +8857,7 @@ Version 6.1.4 [DEVEL] (rgerhards), 2011-02-18 circumstances. This has now been solved, at least for common situations. - bugfix: fixed compile problem due to empty structs - this occured only on some platforms/compilers. thanks to Dražen Kačar + this occurred only on some platforms/compilers. thanks to Dražen Kačar for the fix --------------------------------------------------------------------------- Version 6.1.3 [DEVEL] (rgerhards), 2011-02-01 @@ -8911,7 +8911,7 @@ Version 6.1.1 [DEVEL] (rgerhards), 2010-11-30 - merged imuxsock improvements from 5.7.1 (see there) - support for systemd officially added (import from 5.7.0) - bugfix: a couple of problems that imfile had on some platforms, namely - Ubuntu (not their fault, but occured there) + Ubuntu (not their fault, but occurred there) - bugfix: imfile utilizes 32 bit to track offset. Most importantly, this problem can not experienced on Fedora 64 bit OS (which has 64 bit long's!) @@ -9299,7 +9299,7 @@ Version 5.8.7 [V5-stable] 2012-01-17 - bugfix: instabilities when using RFC5424 header fields Thanks to Kaiwang Chen for the patch - bugfix: imuxsock did truncate part of received message if it did not - contain a proper date. The truncation occured because we removed that + contain a proper date. The truncation occurred because we removed that part of the messages that was expected to be the date. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=295 - bugfix: potential abort after reading invalid X.509 certificate @@ -9380,7 +9380,7 @@ Version 5.8.2 [V5-stable] (rgerhards), 2011-06-21 Version 5.8.1 [V5-stable] (rgerhards), 2011-05-19 - bugfix: invalid processing in QUEUE_FULL condition If the the multi-submit interface was used and a QUEUE_FULL condition - occured, the failed message was properly destructed. However, the + occurred, the failed message was properly destructed. However, the rest of the input batch, if it existed, was not processed. So this lead to potential loss of messages and a memory leak. The potential loss of messages was IMHO minor, because they would have been dropped @@ -9527,7 +9527,7 @@ Version 5.7.4 [V5-BETA] (rgerhards), 2011-02-17 circumstances. This has now been solved, at least for common situations. - bugfix: fixed compile problem due to empty structs - this occured only on some platforms/compilers. thanks to Dražen Kačar + this occurred only on some platforms/compilers. thanks to Dražen Kačar for the fix --------------------------------------------------------------------------- Version 5.7.3 [V5-BETA] (rgerhards), 2011-02-07 @@ -9669,7 +9669,7 @@ This release brings all changes and enhancements of the 5.5.x series to the v5-stable branch. - bugfix: a couple of problems that imfile had on some platforms, namely - Ubuntu (not their fault, but occured there) + Ubuntu (not their fault, but occurred there) - bugfix: imfile utilizes 32 bit to track offset. Most importantly, this problem can not experienced on Fedora 64 bit OS (which has 64 bit long's!) @@ -10109,7 +10109,7 @@ Version 5.1.2 [DEVEL] (rgerhards), 2009-07-08 --------------------------------------------------------------------------- Version 5.1.1 [DEVEL] (rgerhards), 2009-07-03 - bugfix: huge memory leak in queue engine (made rsyslogd unusable in - production). Occured if at least one queue was in direct mode + production). Occurred if at least one queue was in direct mode (the default for action queues) - imported many performance optimizations from v4-devel (4.5.0) - bugfix: subtle (and usually irrelevant) issue in timeout processing @@ -10199,7 +10199,7 @@ Version 4.7.3 [v4-devel] (rgerhards), 2010-11-25 - added imptcp, a simplified, Linux-specific and potentially fast syslog plain tcp input plugin (NOT supporting TLS!) - bugfix: a couple of problems that imfile had on some platforms, namely - Ubuntu (not their fault, but occured there) + Ubuntu (not their fault, but occurred there) - bugfix: imfile utilizes 32 bit to track offset. Most importantly, this problem can not experienced on Fedora 64 bit OS (which has 64 bit long's!) @@ -10269,7 +10269,7 @@ Version 4.6.6 [v4-stable] (rgerhards), 2011-06-24 of some data structures. [backport from v6, limited testing under v4] - bugfix: invalid processing in QUEUE_FULL condition If the the multi-submit interface was used and a QUEUE_FULL condition - occured, the failed message was properly destructed. However, the + occurred, the failed message was properly destructed. However, the rest of the input batch, if it existed, was not processed. So this lead to potential loss of messages and a memory leak. The potential loss of messages was IMHO minor, because they would have been dropped @@ -10306,7 +10306,7 @@ Version 4.6.6 [v4-stable] (rgerhards), 2011-06-24 code has now been removed, a file change (and thus resent) is only detected if the inode number changes. - bugfix: a couple of problems that imfile had on some platforms, namely - Ubuntu (not their fault, but occured there) + Ubuntu (not their fault, but occurred there) - bugfix: imfile utilizes 32 bit to track offset. Most importantly, this problem can not experienced on Fedora 64 bit OS (which has 64 bit long's!) @@ -10422,8 +10422,8 @@ Version 4.6.2 [v4-stable] (rgerhards), 2010-03-26 unlikely) could lead to some issues. We have not been able to describe any fatal cases, but there was some bug potential. Sequence has now been straighted out. -- bugfix: potential problem (loop, abort) when file write error occured - When a write error occured in stream.c, variable iWritten had the error +- bugfix: potential problem (loop, abort) when file write error occurred + When a write error occurred in stream.c, variable iWritten had the error code but this was handled as if it were the actual number of bytes written. That was used in pointer arithmetic later on, and thus could lead to all sorts of problems. However, this could only happen if the @@ -11257,7 +11257,7 @@ Version 3.20.1 [v3-stable] (rgerhards), 2008-12-04 - security bugfix: $AllowedSender was not honored, all senders were permitted instead - enhance: regex nomatch option "ZERO" has been added - This allows to return the string 0 if a regular expression is + This allows one to return the string 0 if a regular expression is not found. This is probably useful for storing numerical values into database columns. - bugfix: memory leak in gtls netstream driver fixed @@ -11862,7 +11862,7 @@ Version 3.12.3 (rgerhards), 2008-03-18 --------------------------------------------------------------------------- Version 3.12.2 (rgerhards), 2008-03-13 - added RSYSLOGD_MODDIR environment variable -- added -M rsyslogd option (allows to specify module directory location) +- added -M rsyslogd option (allows one to specify module directory location) - converted net.c into a loadable library plugin - bugfix: debug module now survives unload of loadable module when printing out function call data @@ -12328,7 +12328,7 @@ Version 1.19.7 (rgerhards), 2007-09-25 handled. - replaced some non-thread-safe function calls by their thread-safe counterparts -- fixed a minor memory leak that occured when the %APPNAME% property was +- fixed a minor memory leak that occurred when the %APPNAME% property was used (I think nobody used that in practice) - fixed a bug that caused signal handlers in cvthname() not to be restored when a malicious pointer record was detected and processing of the message been @@ -12428,7 +12428,7 @@ Version 1.18.1 (rgerhards), 2007-08-08 - applied patch from Michel Samia to fix compilation when the pthreads feature is disabled - some code cleanup (moved action object to its own file set) -- add config directive $MainMsgQueueSize, which now allows to configure the +- add config directive $MainMsgQueueSize, which now allows one to configure the queue size dynamically - all compile-time settings are now shown in rsyslogd -v, not just the active ones @@ -12461,7 +12461,7 @@ Version 1.17.6 (rgerhards), 2007-08-01 The initial fix and idea was developed by mildew, I fine-tuned it a bit. Thanks a lot for the fix, I'd probably had pulled out my hair to find the bug... -- added output of config file line number when a parsing error occured +- added output of config file line number when a parsing error occurred - fixed bug in objomsr.c that caused program to abort in debug mode with an invalid assertion (in some cases) - fixed a typo that caused the default template for MySQL to be wrong. @@ -12838,7 +12838,7 @@ Version 1.10.2 (RGer), 2005-09-27 (only if -d switch is given) - changed some function definitions to static, which eventually enables some compiler optimizations -- fixed a bug in MySQL code; when a SQL error occured, rsyslogd could +- fixed a bug in MySQL code; when a SQL error occurred, rsyslogd could run in a tight loop. This was due to invalid sequence of error reporting and is now fixed. --------------------------------------------------------------------------- @@ -12896,7 +12896,7 @@ Version 1.0.3 (RGer), 2005-11-14 was present in the syslog message --------------------------------------------------------------------------- Version 1.0.2 (RGer), 2005-10-05 -- fixed an issue with MySQL error reporting. When an error occured, +- fixed an issue with MySQL error reporting. When an error occurred, the MySQL driver went into an endless loop (at least in most cases). --------------------------------------------------------------------------- Version 1.0.1 (RGer), 2005-09-23 @@ -12964,7 +12964,7 @@ Version 0.9.5 (RGer), 2005-08-01 enables rsyslogd NOT to listen to the local socket. This is most helpful when multiple instances of rsyslogd (or rsyslogd and another syslogd) shall run on a single system. -- added new option "-i " which allows to specify the pidfile. +- added new option "-i " which allows one to specify the pidfile. This is needed when multiple instances of rsyslogd are to be run. - the new project home page is now online at www.rsyslog.com --------------------------------------------------------------------------- @@ -13271,7 +13271,7 @@ Version 1.19.7 (rgerhards), 2007-09-25 handled. - replaced some non-thread-safe function calls by their thread-safe counterparts -- fixed a minor memory leak that occured when the %APPNAME% property was +- fixed a minor memory leak that occurred when the %APPNAME% property was used (I think nobody used that in practice) - fixed a bug that caused signal handlers in cvthname() not to be restored when a malicious pointer record was detected and processing of the message been @@ -13371,7 +13371,7 @@ Version 1.18.1 (rgerhards), 2007-08-08 - applied patch from Michel Samia to fix compilation when the pthreads feature is disabled - some code cleanup (moved action object to its own file set) -- add config directive $MainMsgQueueSize, which now allows to configure the +- add config directive $MainMsgQueueSize, which now allows one to configure the queue size dynamically - all compile-time settings are now shown in rsyslogd -v, not just the active ones @@ -13404,7 +13404,7 @@ Version 1.17.6 (rgerhards), 2007-08-01 The initial fix and idea was developed by mildew, I fine-tuned it a bit. Thanks a lot for the fix, I'd probably had pulled out my hair to find the bug... -- added output of config file line number when a parsing error occured +- added output of config file line number when a parsing error occurred - fixed bug in objomsr.c that caused program to abort in debug mode with an invalid assertion (in some cases) - fixed a typo that caused the default template for MySQL to be wrong. @@ -13781,7 +13781,7 @@ Version 1.10.2 (RGer), 2005-09-27 (only if -d switch is given) - changed some function definitions to static, which eventually enables some compiler optimizations -- fixed a bug in MySQL code; when a SQL error occured, rsyslogd could +- fixed a bug in MySQL code; when a SQL error occurred, rsyslogd could run in a tight loop. This was due to invalid sequence of error reporting and is now fixed. --------------------------------------------------------------------------- @@ -13839,7 +13839,7 @@ Version 1.0.3 (RGer), 2005-11-14 was present in the syslog message --------------------------------------------------------------------------- Version 1.0.2 (RGer), 2005-10-05 -- fixed an issue with MySQL error reporting. When an error occured, +- fixed an issue with MySQL error reporting. When an error occurred, the MySQL driver went into an endless loop (at least in most cases). --------------------------------------------------------------------------- Version 1.0.1 (RGer), 2005-09-23 @@ -13907,7 +13907,7 @@ Version 0.9.5 (RGer), 2005-08-01 enables rsyslogd NOT to listen to the local socket. This is most helpful when multiple instances of rsyslogd (or rsyslogd and another syslogd) shall run on a single system. -- added new option "-i " which allows to specify the pidfile. +- added new option "-i " which allows one to specify the pidfile. This is needed when multiple instances of rsyslogd are to be run. - the new project home page is now online at www.rsyslog.com --------------------------------------------------------------------------- @@ -14214,7 +14214,7 @@ Version 1.19.7 (rgerhards), 2007-09-25 handled. - replaced some non-thread-safe function calls by their thread-safe counterparts -- fixed a minor memory leak that occured when the %APPNAME% property was +- fixed a minor memory leak that occurred when the %APPNAME% property was used (I think nobody used that in practice) - fixed a bug that caused signal handlers in cvthname() not to be restored when a malicious pointer record was detected and processing of the message been @@ -14314,7 +14314,7 @@ Version 1.18.1 (rgerhards), 2007-08-08 - applied patch from Michel Samia to fix compilation when the pthreads feature is disabled - some code cleanup (moved action object to its own file set) -- add config directive $MainMsgQueueSize, which now allows to configure the +- add config directive $MainMsgQueueSize, which now allows one to configure the queue size dynamically - all compile-time settings are now shown in rsyslogd -v, not just the active ones @@ -14347,7 +14347,7 @@ Version 1.17.6 (rgerhards), 2007-08-01 The initial fix and idea was developed by mildew, I fine-tuned it a bit. Thanks a lot for the fix, I'd probably had pulled out my hair to find the bug... -- added output of config file line number when a parsing error occured +- added output of config file line number when a parsing error occurred - fixed bug in objomsr.c that caused program to abort in debug mode with an invalid assertion (in some cases) - fixed a typo that caused the default template for MySQL to be wrong. @@ -14724,7 +14724,7 @@ Version 1.10.2 (RGer), 2005-09-27 (only if -d switch is given) - changed some function definitions to static, which eventually enables some compiler optimizations -- fixed a bug in MySQL code; when a SQL error occured, rsyslogd could +- fixed a bug in MySQL code; when a SQL error occurred, rsyslogd could run in a tight loop. This was due to invalid sequence of error reporting and is now fixed. --------------------------------------------------------------------------- @@ -14782,7 +14782,7 @@ Version 1.0.3 (RGer), 2005-11-14 was present in the syslog message --------------------------------------------------------------------------- Version 1.0.2 (RGer), 2005-10-05 -- fixed an issue with MySQL error reporting. When an error occured, +- fixed an issue with MySQL error reporting. When an error occurred, the MySQL driver went into an endless loop (at least in most cases). --------------------------------------------------------------------------- Version 1.0.1 (RGer), 2005-09-23 @@ -14850,7 +14850,7 @@ Version 0.9.5 (RGer), 2005-08-01 enables rsyslogd NOT to listen to the local socket. This is most helpful when multiple instances of rsyslogd (or rsyslogd and another syslogd) shall run on a single system. -- added new option "-i " which allows to specify the pidfile. +- added new option "-i " which allows one to specify the pidfile. This is needed when multiple instances of rsyslogd are to be run. - the new project home page is now online at www.rsyslog.com --------------------------------------------------------------------------- diff --git a/action.c b/action.c index 695967d46a..62b4daaaee 100644 --- a/action.c +++ b/action.c @@ -149,8 +149,8 @@ typedef struct configSettings_s { int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ int bActionWriteAllMarkMsgs; /* should all mark messages be unconditionally written? */ int iActExecOnceInterval; /* execute action once every nn seconds */ - int iActExecEveryNthOccur; /* execute action every n-th occurence (0,1=always) */ - time_t iActExecEveryNthOccurTO; /* timeout for n-occurence setting (in seconds, 0=never) */ + int iActExecEveryNthOccur; /* execute action every n-th occurrence (0,1=always) */ + time_t iActExecEveryNthOccurTO; /* timeout for n-occurrence setting (in seconds, 0=never) */ int glbliActionResumeInterval; int glbliActionResumeRetryCount; /* how often should suspended actions be retried? */ int bActionRepMsgHasMsg; /* last messsage repeated... has msg fragment in it */ @@ -1565,7 +1565,7 @@ DBGPRINTF("actionCommit[%s]: return actionTryCommit %d\n", pThis->pszName, iRet) /* We still have some messages with suspend error. So now let's do our * "regular" retry and suspend processing. */ - DBGPRINTF("actionCommit[%s]: unhappy, we still have %d uncommited messages.\n", + DBGPRINTF("actionCommit[%s]: unhappy, we still have %d uncommitted messages.\n", pThis->pszName, nMsgs); int bDone = 0; do { @@ -1854,10 +1854,10 @@ actionWriteToAction(action_t * const pAction, smsg_t *pMsg, wti_t * const pWti) * of view. -- rgerhards, 2008-08-07. */ if(pAction->iExecEveryNthOccur > 1) { - /* we need to care about multiple occurences */ + /* we need to care about multiple occurrences */ if( pAction->iExecEveryNthOccurTO > 0 && (getActNow(pAction) - pAction->tLastOccur) > pAction->iExecEveryNthOccurTO) { - DBGPRINTF("n-th occurence handling timed out (%d sec), restarting from 0\n", + DBGPRINTF("n-th occurrence handling timed out (%d sec), restarting from 0\n", (int) (getActNow(pAction) - pAction->tLastOccur)); pAction->iNbrNoExec = 0; pAction->tLastOccur = getActNow(pAction); diff --git a/action.h b/action.h index 00de1cd6b9..dd2bb9f862 100644 --- a/action.h +++ b/action.h @@ -53,9 +53,9 @@ struct action_s { int iResumeIntervalMax;/* maximum resume interval for this action --> -1: unbounded */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ int iNbrNoExec; /* number of matches that did not yet yield to an exec */ - int iExecEveryNthOccur;/* execute this action only every n-th occurence (with n=0,1 -> always) */ - int iExecEveryNthOccurTO;/* timeout for n-th occurence feature */ - time_t tLastOccur; /* time last occurence was seen (for timing them out) */ + int iExecEveryNthOccur;/* execute this action only every n-th occurrence (with n=0,1 -> always) */ + int iExecEveryNthOccurTO;/* timeout for n-th occurrence feature */ + time_t tLastOccur; /* time last occurrence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ diff --git a/contrib/imbatchreport/imbatchreport.c b/contrib/imbatchreport/imbatchreport.c index e26375ef12..902b1a8a4f 100644 --- a/contrib/imbatchreport/imbatchreport.c +++ b/contrib/imbatchreport/imbatchreport.c @@ -13,7 +13,7 @@ * limit produce a message which references it as "too large" and its new location * The "too large" files are also renamed to keep them available. * - * This modules allows to centralize batch reports with other standard logs and + * This modules allows one to centralize batch reports with other standard logs and * performance monitoring in a single repository (ElasticSearch, HDFS, ...). This * centralization helps to identify cause of global performance issues. * @@ -512,7 +512,7 @@ static void addInstance(instanceConf_t *inst) { } } -/* create input instance, set default paramters, and +/* create input instance, set default parameters, and * add it to the list of instances. */ static rsRetVal diff --git a/contrib/imkmsg/kmsg.c b/contrib/imkmsg/kmsg.c index 5a3f45e49c..9566ff631b 100644 --- a/contrib/imkmsg/kmsg.c +++ b/contrib/imkmsg/kmsg.c @@ -221,7 +221,7 @@ readkmsg(void) } else { /* something went wrong - error or zero length message */ if (i < 0 && errno != EINTR && errno != EAGAIN) { - /* error occured */ + /* error occurred */ imkmsgLogIntMsg(LOG_ERR, "imkmsg: error reading kernel log - shutting down: %s", rs_strerror_r(errno, errmsg, sizeof(errmsg))); diff --git a/contrib/mmgrok/mmgrok.c b/contrib/mmgrok/mmgrok.c index 7bfbcb2737..340c7baae1 100644 --- a/contrib/mmgrok/mmgrok.c +++ b/contrib/mmgrok/mmgrok.c @@ -60,7 +60,7 @@ struct modConfData_s{ static modConfData_t *loadModConf = NULL; static modConfData_t *runModConf = NULL; -/* action (instance) paramters */ +/* action (instance) parameters */ static struct cnfparamdescr actpdescr[]={ {"patterndir",eCmdHdlrString,0}, {"match",eCmdHdlrString,0}, diff --git a/contrib/omfile-hardened/omfile-hardened.c b/contrib/omfile-hardened/omfile-hardened.c index 8a93225cb7..83686afc72 100644 --- a/contrib/omfile-hardened/omfile-hardened.c +++ b/contrib/omfile-hardened/omfile-hardened.c @@ -680,7 +680,7 @@ fsCheck(instanceData *__restrict__ const pData, const uchar *__restrict__ const } /* check if we have space available for all buffers to be flushed and for - * a maximum lenght message, perhaps current msg size would be enough */ + * a maximum length message, perhaps current msg size would be enough */ if (stat.f_bsize * stat.f_bavail < pData->iIOBufSize * pData->iDynaFileCacheSize + (uint)(glbl.GetMaxLine(runModConf->pConf))) { diff --git a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c index dc28611adf..f27a236dba 100644 --- a/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c +++ b/contrib/pmaixforwardedfrom/pmaixforwardedfrom.c @@ -143,7 +143,7 @@ really an AIX log, but has a similar preamble */ pMsg->iLenRawMsg -=1; pMsg->iLenMSG -=1; /* now, claim to abort so that something else can parse the now modified message */ - DBGPRINTF("pmaixforwardedfrom: new mesage: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); + DBGPRINTF("pmaixforwardedfrom: new message: [%d]'%s'\n", lenMsg, pMsg->pszRawMsg + pMsg->offAfterPRI); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: diff --git a/contrib/pmcisconames/pmcisconames.c b/contrib/pmcisconames/pmcisconames.c index 1220bfdd0e..0d7cb5193d 100644 --- a/contrib/pmcisconames/pmcisconames.c +++ b/contrib/pmcisconames/pmcisconames.c @@ -143,7 +143,7 @@ CODESTARTparse pMsg->iLenRawMsg -=2; pMsg->iLenMSG -=2; /* now, claim to abort so that something else can parse the now modified message */ - DBGPRINTF("pmcisconames: new mesage: [%d]'%s'\n", lenMsg, p2parse); + DBGPRINTF("pmcisconames: new message: [%d]'%s'\n", lenMsg, p2parse); ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); finalize_it: diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 492eac8f10..35f08d6e70 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -732,7 +732,7 @@ nvlstChkDupes(struct nvlst *lst) if(nvlstFindName(lst->next, lst->name) != NULL) { cstr = es_str2cstr(lst->name, NULL); parser_errmsg("duplicate parameter '%s' -- " - "interpretation is ambigious, one value " + "interpretation is ambiguous, one value " "will be randomly selected. Fix this problem.", cstr); free(cstr); @@ -2865,7 +2865,7 @@ doFunct_num2ipv4(struct cnffunc *__restrict__ const func, numip[0], numip[1], numip[2], numip[3]); len = snprintf(str, 16, "%d.%d.%d.%d", numip[3], numip[2], numip[1], numip[0]); done: - DBGPRINTF("rainerscript: (num2ipv4) ipv4-Address: %s, lengh: %zu\n", str, len); + DBGPRINTF("rainerscript: (num2ipv4) ipv4-Address: %s, length: %zu\n", str, len); ret->d.estr = es_newStrFromCStr(str, len); ret->datatype = 'S'; } @@ -4566,10 +4566,10 @@ cnfstmtNewAct(struct nvlst *lst) } localRet = actionNewInst(lst, &cnfstmt->d.act); if(localRet == RS_RET_OK_WARN) { - parser_errmsg("warnings occured in file '%s' around line %d", + parser_errmsg("warnings occurred in file '%s' around line %d", cnfcurrfn, yylineno); } else if(localRet != RS_RET_OK) { - parser_errmsg("errors occured in file '%s' around line %d", + parser_errmsg("errors occurred in file '%s' around line %d", cnfcurrfn, yylineno); cnfstmt->nodetype = S_NOP; /* disable action! */ goto done; @@ -4593,7 +4593,7 @@ cnfstmtNewLegaAct(char *actline) cnfstmt->printable = (uchar*)strdup((char*)actline); localRet = cflineDoAction(loadConf, (uchar**)&actline, &cnfstmt->d.act); if(localRet != RS_RET_OK) { - parser_errmsg("%s occured in file '%s' around line %d", + parser_errmsg("%s occurred in file '%s' around line %d", (localRet == RS_RET_OK_WARN) ? "warnings" : "errors", cnfcurrfn, yylineno); if(localRet != RS_RET_OK_WARN) { @@ -5475,7 +5475,7 @@ includeProcessCnf(struct nvlst *const lst) } else if(!strcmp(md, "optional")) { optional = 1; } else { - parser_errmsg("invalid 'mode' paramter: '%s' - ignored", md); + parser_errmsg("invalid 'mode' parameter: '%s' - ignored", md); } free((void*)md); } else { diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index f79c10fb4c..b9aac7dcdd 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -985,7 +985,7 @@ poll_timeouts(fs_edge_t *const edge) act_obj_t *act; for(act = edge->active ; act != NULL ; act = act->next) { if(act->pStrm && strmReadMultiLine_isTimedOut(act->pStrm)) { - DBGPRINTF("timeout occured on %s\n", act->name); + DBGPRINTF("timeout occurred on %s\n", act->name); pollFile(act); } } diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c index 9b227a1793..9b590cb8bf 100644 --- a/plugins/imjournal/imjournal.c +++ b/plugins/imjournal/imjournal.c @@ -815,7 +815,7 @@ CODESTARTrunInput free(cs.usePid); cs.usePid = strdup("system"); LogError(0, RS_RET_DEPRECATED, - "\"usepidfromsystem\" is depricated, use \"usepid\" instead"); + "\"usepidfromsystem\" is deprecated, use \"usepid\" instead"); } if (cs.usePid && (strcmp(cs.usePid, "system") == 0)) { diff --git a/plugins/impstats/statslog-analyzer.py b/plugins/impstats/statslog-analyzer.py index f0a7d2d39b..aca99b780f 100644 --- a/plugins/impstats/statslog-analyzer.py +++ b/plugins/impstats/statslog-analyzer.py @@ -261,7 +261,7 @@ aProblemFound[logDataID]['endline'] = iLineNum aProblemFound[logDataID]['enddate'] = szLineDate elif aProblemFound[logDataID]['value'] > iLogData: - # Logdata was resetted + # Logdata was reset aProblemFound[logDataID]['endline'] = iLineNum aProblemFound[logDataID]['enddate'] = szLineDate @@ -293,7 +293,7 @@ aProblemFound['omfile']['endline'] = iLineNum aProblemFound['omfile']['enddate'] = szLineDate elif aProblemFound['omfile'][logDataID] > iLogData: - # Logdata was resetted + # Logdata was reset aProblemFound['omfile']['endline'] = iLineNum aProblemFound['omfile']['enddate'] = szLineDate diff --git a/plugins/impstats/svg.jquery.js b/plugins/impstats/svg.jquery.js index 30fb6dc2d6..57dae554ce 100644 --- a/plugins/impstats/svg.jquery.js +++ b/plugins/impstats/svg.jquery.js @@ -8011,7 +8011,7 @@ if ( jQuery.support.ajax ) { xml; // Firefox throws exceptions when accessing properties - // of an xhr when a network error occured + // of an xhr when a network error occurred // http://helpful.knobs-dials.com/index.php/Component_returned_failure_code:_0x80040111_(NS_ERROR_NOT_AVAILABLE) try { diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 33a575c3d4..2df46a236c 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1845,7 +1845,7 @@ static void stopWorkerPool(void) { int i; - DBGPRINTF("imptcp: stoping worker pool\n"); + DBGPRINTF("imptcp: stopping worker pool\n"); pthread_mutex_lock(&io_q.mut); pthread_cond_broadcast(&io_q.wakeup_worker); /* awake wrkr if not running */ pthread_mutex_unlock(&io_q.mut); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 1b6c5d24ea..982f6c4b1a 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -909,7 +909,7 @@ rcvMainLoop(struct wrkrInfo_s *const __restrict__ pWrkr) if(nfds < 0) { if(errno == EINTR) { - DBGPRINTF("imudp: EINTR occured\n"); + DBGPRINTF("imudp: EINTR occurred\n"); } else { LogMsg(errno, RS_RET_POLL_ERR, LOG_WARNING, "imudp: poll " "system call failed, may cause further troubles"); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 7129535f19..1ae550e87d 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -1224,7 +1224,7 @@ activateListeners(void) if(actSocks == 0) { LogError(0, RS_RET_ERR, "imuxsock does not run because we could not " - "aquire any socket\n"); + "acquire any socket\n"); ABORT_FINALIZE(RS_RET_ERR); } @@ -1533,7 +1533,7 @@ CODESTARTrunInput if(nfds < 0) { if(errno == EINTR) { - DBGPRINTF("imuxsock: EINTR occured\n"); + DBGPRINTF("imuxsock: EINTR occurred\n"); } else { LogMsg(errno, RS_RET_POLL_ERR, LOG_WARNING, "imuxsock: poll " "system call failed, may cause further troubles"); diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 8f3edcc9a5..9d9d12e127 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -534,7 +534,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) pData->compressionLevel = iLevel; } else { LogError(0, NO_ERRCODE, "Invalid compression level '%c' specified in " - "forwardig action - NOT turning on compression.", + "forwarding action - NOT turning on compression.", *p); } } else if(*p == 'o') { /* octet-couting based TCP framing? */ diff --git a/plugins/omkafka/omkafka.c b/plugins/omkafka/omkafka.c index 850dfc06be..44bffdd4df 100644 --- a/plugins/omkafka/omkafka.c +++ b/plugins/omkafka/omkafka.c @@ -1130,7 +1130,7 @@ do_rd_kafka_destroy(instanceData *const __restrict__ pData) } else /* TODO: Handle unsend messages here! */ { /* timeout = RD_KAFKA_RESP_ERR__TIMED_OUT */ LogError(0, RS_RET_KAFKA_ERROR, "omkafka: onDestroy " - "Failed to send remaing '%d' messages to " + "Failed to send remaining '%d' messages to " "topic '%s' on shutdown with error: '%s'", queuedCount, (pData->pTopic == NULL ? "NULL" : rd_kafka_topic_name(pData->pTopic)), diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 60fd172ed9..4ad8fcbd0b 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -225,7 +225,7 @@ reportDBError(instanceData *pData, int bSilent) /* output log message */ errno = 0; if(pData->conn == NULL) { - LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain connection handle"); + LogError(0, NO_ERRCODE, "unknown DB error occurred - could not obtain connection handle"); } else { /* we can ask dbi for the error description... */ uDBErrno = dbi_conn_error(pData->conn, &pszDbiErr); snprintf(errMsg, sizeof(errMsg), "db error (%d): %s\n", uDBErrno, pszDbiErr); @@ -330,7 +330,7 @@ writeDB(const uchar *psz, instanceData *const __restrict__ pData) /* try insert */ if((dbiRes = dbi_conn_query(pData->conn, (const char*)psz)) == NULL) { - /* error occured, try to re-init connection and retry */ + /* error occurred, try to re-init connection and retry */ closeConn(pData); /* close the current handle */ CHKiRet(initConn(pData, 0)); /* try to re-open */ if((dbiRes = dbi_conn_query(pData->conn, (const char*)psz)) == NULL) { /* re-try insert */ diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index a8b6ac96fd..6470637ed9 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -176,7 +176,7 @@ CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo -/* report error that occured during *last* operation +/* report error that occurred during *last* operation */ static void reportMongoError(instanceData *pData) diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index 51ae704118..5bdab5b961 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -183,7 +183,7 @@ static void reportDBError(wrkrInstanceData_t *pWrkrData, int bSilent) /* output log message */ errno = 0; if(pWrkrData->hmysql == NULL) { - LogError(0, NO_ERRCODE, "ommysql: unknown DB error occured - could not obtain MySQL handle"); + LogError(0, NO_ERRCODE, "ommysql: unknown DB error occurred - could not obtain MySQL handle"); } else { /* we can ask mysql for the error description... */ uMySQLErrno = mysql_errno(pWrkrData->hmysql); snprintf(errMsg, sizeof(errMsg), "db error (%u): %s\n", uMySQLErrno, @@ -289,7 +289,7 @@ static rsRetVal writeMySQL(wrkrInstanceData_t *pWrkrData, const uchar *const psz LogError(0, RS_RET_DATAFAIL, "The error statement was: %s", psz); ABORT_FINALIZE(RS_RET_DATAFAIL); } - /* potentially recoverable error occured, try to re-init connection and retry */ + /* potentially recoverable error occurred, try to re-init connection and retry */ closeMySQL(pWrkrData); /* close the current handle */ CHKiRet(initMySQL(pWrkrData, 0)); /* try to re-open */ if(mysql_query(pWrkrData->hmysql, (char*)psz)) { /* re-try insert */ @@ -575,7 +575,7 @@ CODEmodInit_QueryRegCFSLineHdlr mysql_server_init(0, NULL, NULL) # endif ) { - LogError(0, NO_ERRCODE, "ommysql: intializing mysql client failed, plugin " + LogError(0, NO_ERRCODE, "ommysql: initializing mysql client failed, plugin " "can not run"); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 0dd449c578..27248ffe5c 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -170,7 +170,7 @@ static void reportDBError(wrkrInstanceData_t *pWrkrData, int bSilent) /* output log message */ errno = 0; if (pWrkrData->f_hpgsql == NULL) { - LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain PgSQL handle"); + LogError(0, NO_ERRCODE, "unknown DB error occurred - could not obtain PgSQL handle"); } else { /* we can ask pgsql for the error description... */ ePgSQLStatus = PQstatus(pWrkrData->f_hpgsql); snprintf(errMsg, sizeof(errMsg), "db error (%d): %s\n", ePgSQLStatus, @@ -286,7 +286,7 @@ writePgSQL(uchar *psz, wrkrInstanceData_t *pWrkrData) if (bHadError || (PQstatus(pWrkrData->f_hpgsql) != CONNECTION_OK)) { #if 0 /* re-enable once we have transaction support */ - /* error occured, try to re-init connection and retry */ + /* error occurred, try to re-init connection and retry */ int inTransaction = 0; if(pData->f_hpgsql != NULL) { PGTransactionStatusType xactStatus = PQtransactionStatus(pData->f_hpgsql); diff --git a/plugins/omsnmp/omsnmp.h b/plugins/omsnmp/omsnmp.h index 0983d9a37e..dcb15375cd 100644 --- a/plugins/omsnmp/omsnmp.h +++ b/plugins/omsnmp/omsnmp.h @@ -23,9 +23,9 @@ #define OMSNMP_H_INCLUDED 1 #define OMSNMP_MAXTRANSPORLENGTH 10 -#define OMSNMP_MAXPORTLENGHT 5 -#define OMSNMP_MAXCOMMUNITYLENGHT 255 -#define OMSNMP_MAXOIDLENGHT 255 +#define OMSNMP_MAXPORTLENGTH 5 +#define OMSNMP_MAXCOMMUNITYLENGTH 255 +#define OMSNMP_MAXOIDLENGTH 255 #endif /* #ifndef OMMYSQL_H_INCLUDED */ diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index 1808d7a11f..34b1cd7a38 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -1048,7 +1048,7 @@ void dbgPrintCfSysLineHandlers(void) linkedListCookie_t llCookieCmdHdlr; uchar *pKey; - dbgprintf("Sytem Line Configuration Commands:\n"); + dbgprintf("System Line Configuration Commands:\n"); llCookieCmd = NULL; while(llGetNextElt(&llCmdList, &llCookieCmd, (void*)&pCmd) == RS_RET_OK) { llGetKey(llCookieCmd, (void*) &pKey); /* TODO: using the cookie is NOT clean! */ diff --git a/runtime/conf.c b/runtime/conf.c index fc376a2718..50b1b71ee0 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -27,7 +27,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define CFGLNSIZ 64*1024 /* the maximum size of a configuraton file line, after re-combination */ +#define CFGLNSIZ 64*1024 /* the maximum size of a configuration file line, after re-combination */ #include "config.h" #include #include @@ -530,7 +530,7 @@ rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction) break; } else if(iRet != RS_RET_CONFLINE_UNPROCESSED) { /* In this case, the module would have handled the config - * line, but some error occured while doing so. This error should + * line, but some error occurred while doing so. This error should * already by reported by the module. We do not try any other * modules on this line, because we found the right one. * rgerhards, 2007-07-24 diff --git a/runtime/datetime.c b/runtime/datetime.c index 29f5415237..9f13b27981 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -440,7 +440,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS, int *pLenStr) * only partial information and only that partial information is updated. * So the "output timestamp" is a valid timestamp only if the "input * timestamp" was valid, too. The is actually an optimization, as it - * permits us to use a pre-aquired timestamp and thus avoids to do + * permits us to use a pre-acquired timestamp and thus avoids to do * a (costly) time() call. Thanks to David Lang for insisting on * time() call reduction ;). * This method now also checks the maximum string length it is passed. @@ -821,7 +821,7 @@ applyDfltTZ(struct syslogTime *pTime, char *tz) * The caller must provide the timestamp as well as a character * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without - * the string terminator). If 0 is returend, an error occured. + * the string terminator). If 0 is returend, an error occurred. */ static int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) @@ -890,7 +890,7 @@ formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) * The caller must provide the timestamp as well as a character * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without - * the string terminator). If 0 is returend, an error occured. + * the string terminator). If 0 is returend, an error occurred. * The buffer must be at least 7 bytes large. * rgerhards, 2008-06-06 */ @@ -931,7 +931,7 @@ formatTimestampSecFrac(struct syslogTime *ts, char* pBuf) * The caller must provide the timestamp as well as a character * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without - * the string terminator). If 0 is returend, an error occured. + * the string terminator). If 0 is returend, an error occurred. */ static int formatTimestamp3339(struct syslogTime *ts, char* pBuf) @@ -1006,7 +1006,7 @@ formatTimestamp3339(struct syslogTime *ts, char* pBuf) * The caller must provide the timestamp as well as a character * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without - * the string termnator). If 0 is returend, an error occured. + * the string termnator). If 0 is returend, an error occurred. * rgerhards, 2010-03-05: Added support to for buggy 3164 dates, * where a zero-digit is written instead of a space for the first * day character if day < 10. syslog-ng seems to do that, and some diff --git a/runtime/debug.c b/runtime/debug.c index 80d6b5de8c..a655bc2e4e 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -392,7 +392,7 @@ dbgGetRuntimeOptions(void) /* Enables debugging, but turns off debug output */ Debug = DEBUG_ONDEMAND; debugging_on = 1; - dbgprintf("Note: debug on demand turned on via configuraton file, " + dbgprintf("Note: debug on demand turned on via configuration file, " "use USR1 signal to activate.\n"); debugging_on = 0; } else if(!strcasecmp((char*)optname, "nologtimestamp")) { diff --git a/runtime/glbl.c b/runtime/glbl.c index e46d8695d7..871cbbc927 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -311,7 +311,7 @@ storeLocalHostIPIF(uchar *myIP) /* This function is used to set the IP address that is to be * reported for the local host. Note that in order to ease things - * for the v6 config interface, we do not allow to set this more + * for the v6 config interface, we do not allow this to be set more * than once. * rgerhards, 2012-03-21 */ diff --git a/runtime/lookup.c b/runtime/lookup.c index dba7d63e5a..66fb09d51a 100644 --- a/runtime/lookup.c +++ b/runtime/lookup.c @@ -883,7 +883,7 @@ lookupPendingReloadCount(void) /* returns either a pointer to the value (read only!) or NULL - * if either the key could not be found or an error occured. + * if either the key could not be found or an error occurred. * Note that an estr_t object is returned. The caller is * responsible for freeing it. */ diff --git a/runtime/modules.c b/runtime/modules.c index ca337e9aa4..810b2e9b52 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -281,7 +281,7 @@ static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat) * Modules may call it to get access to core interface functions. Please note * that utility functions can be accessed via shared libraries - at least this * is my current shool of thinking. - * Please note that the implementation as a query interface allows to take + * Please note that the implementation as a query interface allows one to take * care of plug-in interface version differences. -- rgerhards, 2007-07-31 * ... but often it better not to use a new interface. So we now add core * functions here that a plugin may request. -- rgerhards, 2009-04-22 diff --git a/runtime/msg.c b/runtime/msg.c index 87e7e6703a..2513744380 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -491,7 +491,7 @@ MsgSetRulesetByName(smsg_t * const pMsg, cstr_t *const rulesetName) if(localRet != RS_RET_OK) { LogError(0, localRet, "msg: ruleset '%s' could not be found and could not " - "be assgined to message object. This possibly leads to the message " + "be assigned to message object. This possibly leads to the message " "being processed incorrectly. We cannot do anything against this, but " "wanted to let you know.", rs_name); } @@ -1481,7 +1481,7 @@ smsg_t *MsgAddRef(smsg_t * const pM) } -/* This functions tries to aquire the PROCID from TAG. Its primary use is +/* This functions tries to acquire the PROCID from TAG. Its primary use is * when a legacy syslog message has been received and should be forwarded as * syslog-protocol (or the PROCID is requested for any other reason). * In legacy syslog, the PROCID is considered to be the character sequence @@ -1492,7 +1492,7 @@ smsg_t *MsgAddRef(smsg_t * const pM) * rgerhards, 2005-11-24 * THIS MUST be called with the message lock locked. */ -static rsRetVal aquirePROCIDFromTAG(smsg_t * const pM) +static rsRetVal acquirePROCIDFromTAG(smsg_t * const pM) { register int i; uchar *pszTag; @@ -1557,7 +1557,7 @@ static rsRetVal aquirePROCIDFromTAG(smsg_t * const pM) * rgerhards, 2005-10-19 */ static rsRetVal -aquireProgramName(smsg_t * const pM) +acquireProgramName(smsg_t * const pM) { int i; uchar *pszTag, *pszProgName; @@ -2243,7 +2243,7 @@ rsRetVal MsgSetPROCID(smsg_t *__restrict__ const pMsg, const char* pszPROCID) } -/* check if we have a procid, and, if not, try to aquire/emulate it. +/* check if we have a procid, and, if not, try to acquire/emulate it. * This must be called WITHOUT the message lock being held. * rgerhards, 2009-06-26 */ @@ -2254,7 +2254,7 @@ static void preparePROCID(smsg_t * const pM, sbool bLockMutex) MsgLock(pM); /* re-query, things may have changed in the mean time... */ if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); + acquirePROCIDFromTAG(pM); if(bLockMutex == LOCK_MUTEX) MsgUnlock(pM); } @@ -2505,7 +2505,7 @@ tryEmulateTAG(smsg_t *const pM, const sbool bLockMutex) bufTAG[sizeof(bufTAG)-1] = '\0'; /* just to make sure... */ MsgSetTAG(pM, bufTAG, lenTAG); } - /* Signal change in TAG for aquireProgramName */ + /* Signal change in TAG for acquireProgramName */ pM->iLenPROGNAME = -1; } if(bLockMutex == LOCK_MUTEX) @@ -2633,7 +2633,7 @@ getProgramName(smsg_t *const pM, const sbool bLockMutex) rs_size_t bufLen = -1; getTAG(pM, &pRes, &bufLen, MUTEX_ALREADY_LOCKED); } - aquireProgramName(pM); + acquireProgramName(pM); } if(bLockMutex == LOCK_MUTEX) { @@ -2645,7 +2645,7 @@ getProgramName(smsg_t *const pM, const sbool bLockMutex) -/* check if we have a APPNAME, and, if not, try to aquire/emulate it. +/* check if we have a APPNAME, and, if not, try to acquire/emulate it. * rgerhards, 2009-06-26 */ static void ATTR_NONNULL(1) diff --git a/runtime/net.c b/runtime/net.c index 2818193cc1..8611125862 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1315,7 +1315,7 @@ create_single_udp_socket(int *const s, /* socket */ /* We need to enable BSD compatibility. Otherwise an attacker * could flood our log files by sending us tons of ICMP errors. */ - /* AIXPORT : SO_BSDCOMPAT socket option is depricated, and its usage + /* AIXPORT : SO_BSDCOMPAT socket option is deprecated, and its usage * has been discontinued on most unixes, AIX does not support this option, * hence avoid the call. */ diff --git a/runtime/netstrm.c b/runtime/netstrm.c index f7551d6edc..0e9500d100 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -162,7 +162,7 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), * the number of octets read (or -1 in case of error) on exit. This function * never blocks, not even when called on a blocking socket. That is important * for client sockets, which are set to block during send, but should not - * block when trying to read data. If *pLenBuf is -1, an error occured and + * block when trying to read data. If *pLenBuf is -1, an error occurred and * oserr holds the exact error cause. * rgerhards, 2008-03-17 */ diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 434389a302..4f605bef6a 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -481,7 +481,7 @@ osslRecordRecv(nsd_ossl_t *pThis) osslLastSSLErrorMsg(lenRcvd, pThis->ssl, LOG_ERR, "osslRecordRecv"); /* Check for underlaying socket errors **/ if (local_errno == ECONNRESET) { - DBGPRINTF("osslRecordRecv: Errno %d, connection resetted by peer\n", local_errno); + DBGPRINTF("osslRecordRecv: Errno %d, connection reset by peer\n", local_errno); ABORT_FINALIZE(RS_RET_CLOSED); } else { DBGPRINTF("osslRecordRecv: Errno %d\n", local_errno); @@ -1518,7 +1518,7 @@ osslHandshakeCheck(nsd_ossl_t *pNsd) free(fromHostIP); } if(iRet == RS_RET_OK) { - /* If no error occured, set socket to SSL mode */ + /* If no error occurred, set socket to SSL mode */ pNsd->iMode = 1; } diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index c5861ddd17..6ddaf41060 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -758,7 +758,7 @@ LstnInit(netstrms_t *const pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t* * the number of octets read (or -1 in case of error) on exit. This function * never blocks, not even when called on a blocking socket. That is important * for client sockets, which are set to block during send, but should not - * block when trying to read data. If *pLenBuf is -1, an error occured and + * block when trying to read data. If *pLenBuf is -1, an error occurred and * oserr holds the exact error cause. * rgerhards, 2008-03-17 */ diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 3d0120370a..21fb89bbe1 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -284,7 +284,7 @@ rsRetVal objName##ClassExit(void) \ * The object is always freed, even if there is some error while * Cancellation is blocked during destructors, as this could have fatal * side-effects. However, this also means the upper-level object should - * not perform any lenghty processing. + * not perform any lengthy processing. * IMPORTANT: if the upper level object requires some situations where the * object shall not be destructed (e.g. via reference counting), then * it shall set pThis to NULL, which prevents destruction of the diff --git a/runtime/obj.c b/runtime/obj.c index ad76ba9ee0..c78b1d27ab 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -16,7 +16,7 @@ * should request object interface only once in the object Init function and * free them when they exit. The only exception is when a caller needs to * access an object only conditional, in which case a pointer to its interface - * shall be aquired as need first arises but still be released only on exit + * shall be acquired as need first arises but still be released only on exit * or when there definitely is no further need. The whole idea is to limit * the very performance-intense act of dynamically loading an objects library. * Of course, it is possible to violate this suggestion, but than you should @@ -428,7 +428,7 @@ EndSerialize(strm_t *pStrm) /* de-serialize an embedded, non-octect-counted string. This is useful * for deserializing the object name inside the header. The string is - * terminated by the first occurence of the ':' character. + * terminated by the first occurrence of the ':' character. * rgerhards, 2008-02-29 */ static rsRetVal diff --git a/runtime/queue.c b/runtime/queue.c index 78a4618fb5..cf86325a70 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2169,7 +2169,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) DBGPRINTF("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); - /* now we are done, but potentially need to re-aquire the mutex */ + /* now we are done, but potentially need to re-acquire the mutex */ if(bNeedReLock) d_pthread_mutex_lock(pThis->mut); @@ -2212,7 +2212,7 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti) iRet = qqueueEnqMsg(pThis->pqDA, eFLOWCTL_NO_DELAY, MsgAddRef(pWti->batch.pElem[i].pMsg)); if(iRet != RS_RET_OK) { if(iRet == RS_RET_ERR_QUEUE_EMERGENCY) { - /* Queue emergency error occured */ + /* Queue emergency error occurred */ DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg caught RS_RET_ERR_QUEUE_EMERGENCY," "aborting loop.\n"); FINALIZE; @@ -2256,7 +2256,7 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti) DBGOPRINT((obj_t*) pThis, "ConsumerDA:qqueueEnqMsg returns with iRet %d\n", iRet); } - /* now we are done, but potentially need to re-aquire the mutex */ + /* now we are done, but potentially need to re-acquire the mutex */ if(bNeedReLock) d_pthread_mutex_lock(pThis->mut); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index c736fa89d8..9b4c4c201b 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -126,7 +126,7 @@ rsrtSetErrLogger(void (*errLogger)(const int, const int, const uchar*)) * NOTE: this is NOT thread safe and must not be called concurrently. If that * ever poses a problem, we may use proper mutex calls - not considered needed yet. * If ppErrObj is provided, it receives a char pointer to the name of the object that - * caused the problem (if one occured). The caller must never free this pointer. If + * caused the problem (if one occurred). The caller must never free this pointer. If * ppErrObj is NULL, no such information will be provided. pObjIF is the pointer to * the "obj" object interface, which may be used to query any other rsyslog objects. * rgerhards, 2008-04-16 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index d50175eae0..b641fc9952 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -337,7 +337,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_QTYPE_MISMATCH = -2038, /**< different qType when reading back a property type */ RS_RET_NO_FILE_ACCESS = -2039, /**< covers EACCES error on file open() */ RS_RET_FILE_NOT_FOUND = -2040, /**< file not found */ - RS_RET_TIMED_OUT = -2041, /**< timeout occured (not necessarily an error) */ + RS_RET_TIMED_OUT = -2041, /**< timeout occurred (not necessarily an error) */ RS_RET_QSIZE_ZERO = -2042, /**< queue size is zero where this is not supported */ RS_RET_ALREADY_STARTING = -2043, /**< something (a thread?) is already starting - not necessarily an error */ RS_RET_NO_MORE_THREADS = -2044, /**< no more threads available, not necessarily an error */ @@ -390,13 +390,13 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_CERT_INVALID_DN = -2091, /**< distinguised name in x509 certificate is invalid (e.g. wrong escaping) */ RS_RET_CERT_EXPIRED = -2092, /**< we are past a x.509 cert's expiration time */ RS_RET_CERT_NOT_YET_ACTIVE = -2094, /**< x.509 cert's activation time not yet reached */ - RS_RET_SYS_ERR = -2095, /**< system error occured (e.g. time() returned -1, quite unexpected) */ + RS_RET_SYS_ERR = -2095, /**< system error occurred (e.g. time() returned -1, quite unexpected) */ RS_RET_FILE_NO_STAT = -2096, /**< can not stat() a file */ RS_RET_FILE_TOO_LARGE = -2097, /**< a file is larger than permitted */ RS_RET_INVALID_WILDCARD = -2098, /**< a wildcard entry is invalid */ RS_RET_CLOSED = -2099, /**< connection was closed */ RS_RET_RETRY = -2100, /**< call should be retried (e.g. EGAIN on recv) */ - RS_RET_GSS_ERR = -2101, /**< generic error occured in GSSAPI subsystem */ + RS_RET_GSS_ERR = -2101, /**< generic error occurred in GSSAPI subsystem */ RS_RET_CERTLESS = -2102, /**< state: we run without machine cert (this may be OK) */ RS_RET_NO_ACTIONS = -2103, /**< no active actions are configured (no output will be created) */ RS_RET_CONF_FILE_NOT_FOUND = -2104, /**< config file or directory not found */ @@ -444,11 +444,11 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ - RS_RET_EINTR = -2161, /**< EINTR occured during a system call (not necessarily an error) */ + RS_RET_EINTR = -2161, /**< EINTR occurred during a system call (not necessarily an error) */ RS_RET_ERR_EPOLL = -2162, /**< epoll() returned with an unexpected error code */ RS_RET_ERR_EPOLL_CTL = -2163, /**< epol_ctll() returned with an unexpected error code */ - RS_RET_TIMEOUT = -2164, /**< timeout occured during operation */ - RS_RET_RCV_ERR = -2165, /**< error occured during socket rcv operation */ + RS_RET_TIMEOUT = -2164, /**< timeout occurred during operation */ + RS_RET_RCV_ERR = -2165, /**< error occurred during socket rcv operation */ RS_RET_NO_SOCK_CONFIGURED = -2166, /**< no socket (name) was configured where one is required */ RS_RET_CONF_NOT_GLBL = -2167, /**< $Begin not in global scope */ RS_RET_CONF_IN_GLBL = -2168, /**< $End when in global scope */ @@ -536,7 +536,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RELP_AUTH_FAIL = -2353,/**< RELP peer authentication failed */ RS_RET_ERR_UDPSEND = -2354,/**< sending msg via UDP failed */ RS_RET_LAST_ERRREPORT = -2355,/**< module does not emit more error messages as limit is reached */ - RS_RET_READ_ERR = -2356,/**< read error occured (file i/o) */ + RS_RET_READ_ERR = -2356,/**< read error occurred (file i/o) */ RS_RET_CONF_PARSE_WARNING = -2357,/**< warning parsing config file */ RS_RET_CONF_WRN_FULLDLY_BELOW_HIGHWTR = -2358,/**< warning queue full delay mark below high wtr mark */ RS_RET_RESUMED = -2359,/**< status: action was resumed (used for reporting) */ @@ -564,7 +564,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_SENDER_APPEARED = -2430,/**< info: new sender appeared */ RS_RET_FILE_ALREADY_IN_TABLE = -2431,/**< in imfile: table already contains to be added file */ RS_RET_ERR_DROP_PRIV = -2432,/**< error droping privileges */ - RS_RET_FILE_OPEN_ERROR = -2433, /**< error other than "not found" occured during open() */ + RS_RET_FILE_OPEN_ERROR = -2433, /**< error other than "not found" occurred during open() */ RS_RET_RENAME_TMP_QI_ERROR = -2435, /**< renaming temporary .qi file failed */ RS_RET_ERR_SETENV = -2436, /**< error setting an environment variable */ RS_RET_DIR_CHOWN_ERROR = -2437, /**< error during chown() */ diff --git a/runtime/srutils.c b/runtime/srutils.c index b2107f1cb7..33699751fd 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -620,7 +620,7 @@ int decodeSyslogName(uchar *name, syslogName_t *codetab) will be stored. * \param DstSize Maximum numbers of characters to store. * \param cSep Separator char. - * \ret int Returns 0 if no error occured. + * \ret int Returns 0 if no error occurred. * * rgerhards, 2008-02-12: some notes are due... I will once again fix this function, this time * so that it treats ' ' as a request for whitespace. But in general, the function and its callers diff --git a/runtime/stream.c b/runtime/stream.c index 27010aba53..064fe5ded6 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1426,7 +1426,7 @@ tryTTYRecover(strm_t *pThis, int err) /* issue write() api calls until either the buffer is completely - * written or an error occured (it may happen that multiple writes + * written or an error occurred (it may happen that multiple writes * are required, what is perfectly legal. On exit, *pLenBuf contains * the number of bytes actually written. * rgerhards, 2009-06-08 diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 8626eda44f..b7e7c75976 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -650,7 +650,7 @@ void rsCStrRegexDestruct(void *rc) * I have opted for performance and thus duplicated the code. I hope * this is a good, or at least acceptable, compromise. * rgerhards, 2005-09-26 - * This function also has an offset-pointer which allows to + * This function also has an offset-pointer which allows one to * specify *where* the compare operation should begin in * the CStr. If everything is to be compared, it must be set * to 0. If some leading bytes are to be skipped, it must be set @@ -711,7 +711,7 @@ int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz) } -/* Locate the first occurence of this rsCStr object inside a standard sz string. +/* Locate the first occurrence of this rsCStr object inside a standard sz string. * Returns the offset (0-bound) of this first occurrence. If not found, -1 is * returned. Both parameters MUST be given (NULL is not allowed). * rgerhards 2005-09-19 diff --git a/runtime/var.c b/runtime/var.c index 65e8ce66a3..c636fbb741 100644 --- a/runtime/var.c +++ b/runtime/var.c @@ -87,7 +87,7 @@ CODESTARTobjDebugPrint(var) case VARTYPE_SYSLOGTIME: case VARTYPE_NONE: default: - dbgoprint((obj_t*) pThis, "type %d currently not suppored in debug output\n", pThis->varType); + dbgoprint((obj_t*) pThis, "type %d currently not supported in debug output\n", pThis->varType); break; } ENDobjDebugPrint(var) diff --git a/runtime/wti.c b/runtime/wti.c index 290bf96166..23c295a436 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -314,9 +314,9 @@ wtiWorkerCancelCleanup(void *arg) pWtp = pThis->pWtp; ISOBJ_TYPE_assert(pWtp, wtp); - DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: cancellation cleanup handler called.\n", wtiGetDbgHdr(pThis)); pWtp->pfObjProcessed(pWtp->pUsr, pThis); - DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: done cancellation cleanup handler.\n", wtiGetDbgHdr(pThis)); } @@ -350,7 +350,7 @@ wtiWaitNonEmpty(wti_t *const pThis, const struct timespec timeout) * rgerhards, 2009-05-20 */ static void ATTR_NONNULL() -doIdleProcessing(wti_t *const pThis, wtp_t *const pWtp, int *const pbInactivityTOOccured) +doIdleProcessing(wti_t *const pThis, wtp_t *const pWtp, int *const pbInactivityTOOccurred) { struct timespec t; @@ -363,7 +363,7 @@ doIdleProcessing(wti_t *const pThis, wtp_t *const pWtp, int *const pbInactivityT timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ if(d_pthread_cond_timedwait(&pThis->pcondBusy, pWtp->pmutUsr, &t) != 0) { DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); - *pbInactivityTOOccured = 1; /* indicate we had a timeout */ + *pbInactivityTOOccurred = 1; /* indicate we had a timeout */ } } DBGOPRINT((obj_t*) pThis, "worker awoke from idle processing\n"); @@ -392,7 +392,7 @@ wtiWorker(wti_t *__restrict__ const pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - int bInactivityTOOccured = 0; + int bInactivityTOOccurred = 0; pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); DBGPRINTF("wti %p: worker starting\n", pThis); /* now we have our identity, on to real processing */ @@ -431,16 +431,16 @@ wtiWorker(wti_t *__restrict__ const pThis) if(localRet == RS_RET_ERR_QUEUE_EMERGENCY) { break; /* end of loop */ } else if(localRet == RS_RET_IDLE) { - if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { + if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccurred) { DBGOPRINT((obj_t*) pThis, "terminating worker terminateRet=%d, " - "bInactivityTOOccured=%d\n", terminateRet, bInactivityTOOccured); + "bInactivityTOOccurred=%d\n", terminateRet, bInactivityTOOccurred); break; /* end of loop */ } - doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); + doIdleProcessing(pThis, pWtp, &bInactivityTOOccurred); continue; /* request next iteration */ } - bInactivityTOOccured = 0; /* reset for next run */ + bInactivityTOOccurred = 0; /* reset for next run */ } d_pthread_mutex_unlock(pWtp->pmutUsr); diff --git a/runtime/wti.h b/runtime/wti.h index 609bf033d9..71eed56794 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -33,7 +33,7 @@ #define ACT_STATE_RDY 0 /* action ready, waiting for new transaction */ #define ACT_STATE_ITX 1 /* transaction active, waiting for new data or commit */ /* 2 currently not being used */ -#define ACT_STATE_RTRY 3 /* failure occured, trying to restablish ready state */ +#define ACT_STATE_RTRY 3 /* failure occurred, trying to restablish ready state */ #define ACT_STATE_SUSP 4 /* suspended due to failure (return fail until timeout expired) */ #define ACT_STATE_DATAFAIL 5 /* suspended due to failure in data, which means the message in questions needs to be dropped as it will always fail. The diff --git a/runtime/wtp.c b/runtime/wtp.c index 5570664a91..3bce02a318 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -508,7 +508,7 @@ if(dbgTimeoutToStderr) { ATOMIC_FETCH_32BIT(&pThis->iCurNumWrkThrd, &pThis->mutCurNumWrkThrd)); /* wait for the new thread to initialize its signal mask and - * cancelation cleanup handler before proceeding + * cancellation cleanup handler before proceeding */ do { d_pthread_cond_wait(&pThis->condThrdInitDone, &pThis->mutWtp); diff --git a/sample.conf b/sample.conf index c7088bf96d..e895149c12 100644 --- a/sample.conf +++ b/sample.conf @@ -29,7 +29,7 @@ $AllowedSender TCP, 10.0.0.1 # for TCP, we allow only 10.0.0.1 # remove the AllowedSender directives if you do not want to limit # who can send rsyslogd messages (not recommended) -# Templates are a key feature of rsyslog. They allow to specify any +# Templates are a key feature of rsyslog. They allow one to specify any # format a user might want. Every output in rsyslog uses templates - this # holds true for files, user messages and so on. The database writer # expects its template to be a proper SQL statement - so this is highly diff --git a/template.c b/template.c index b8655adad5..edb95f25af 100644 --- a/template.c +++ b/template.c @@ -544,7 +544,7 @@ tplConstruct(rsconf_t *conf) /* helper to tplAddLine. Parses a constant and generates * the necessary structure. - * Paramter "bDoEscapes" is to support legacy vs. v6+ config system. In + * Parameter "bDoEscapes" is to support legacy vs. v6+ config system. In * legacy, we must do escapes ourselves, whereas v6+ passes in already * escaped strings (which we are NOT permitted to further escape, this would * cause invalid result strings!). Note: if escapes are not permitted, @@ -2397,7 +2397,7 @@ void tplPrintList(rsconf_t *conf) dbgprintf("[drop last LF in msg] "); } if(pTpe->data.field.has_fields == 1) { - dbgprintf("[substring, field #%d only (delemiter %d)] ", + dbgprintf("[substring, field #%d only (delimiter %d)] ", pTpe->data.field.iFieldNr, pTpe->data.field.field_delim); } if(pTpe->data.field.iFromPos != 0 || pTpe->data.field.iToPos != 0) { diff --git a/template.h b/template.h index b387b16f53..436439f164 100644 --- a/template.h +++ b/template.h @@ -107,7 +107,7 @@ struct templateEntry { #endif unsigned has_fields; /* support for field-counting: field to extract */ - unsigned char field_delim; /* support for field-counting: field delemiter char */ + unsigned char field_delim; /* support for field-counting: field delimiter char */ #ifdef STRICT_GPLV3 int field_expand; /* use multiple instances of the field delimiter as a single one? */ #endif diff --git a/tests/asynwr_deadlock2.sh b/tests/asynwr_deadlock2.sh index 29c37f85be..f656721c68 100755 --- a/tests/asynwr_deadlock2.sh +++ b/tests/asynwr_deadlock2.sh @@ -17,7 +17,7 @@ # was done (in which case it would receive a wakeup). As such, it went into an eternal # wait. However, the invalid signaling did not take into account that it did not # signal the async writer to shut down. So the main thread went into a condition -# wait - and thus we had a deadlock. That situation occured only under very specific +# wait - and thus we had a deadlock. That situation occurred only under very specific # circumstances. As far as the analysis goes, the following need to happen: # 1. buffers on that file are being flushed # 2. no new data arrives diff --git a/tests/cfg1.cfgtest b/tests/cfg1.cfgtest index 8195244810..d49d2078ab 100644 --- a/tests/cfg1.cfgtest +++ b/tests/cfg1.cfgtest @@ -1,3 +1,3 @@ rsyslogd: invalid or yet-unknown config file command - have you forgotten to load a module? [try https://www.rsyslog.com/e/3003 ] -rsyslogd: the last error occured in ./cfg1.testin, line 2 +rsyslogd: the last error occurred in ./cfg1.testin, line 2 rsyslogd: End of config validation run. Bye. diff --git a/tests/cfg2.cfgtest b/tests/cfg2.cfgtest index 0e8dc69a60..8a98eb510a 100644 --- a/tests/cfg2.cfgtest +++ b/tests/cfg2.cfgtest @@ -1,3 +1,3 @@ rsyslogd: invalid or yet-unknown config file command - have you forgotten to load a module? [try https://www.rsyslog.com/e/3003 ] -rsyslogd: the last error occured in cfg1.testin, line 2 +rsyslogd: the last error occurred in cfg1.testin, line 2 rsyslogd: End of config validation run. Bye. diff --git a/tests/cfg3.cfgtest b/tests/cfg3.cfgtest index 3e7dde9e53..5edd4ef382 100644 --- a/tests/cfg3.cfgtest +++ b/tests/cfg3.cfgtest @@ -1,5 +1,5 @@ rsyslogd: error accessing config file or directory 'file-does-not-exist': No such file or directory [try https://www.rsyslog.com/e/2040 ] -rsyslogd: the last error occured in ./cfg3.testin, line 1 +rsyslogd: the last error occurred in ./cfg3.testin, line 1 rsyslogd: CONFIG ERROR: there are no active actions configured. Inputs will run, but no output whatsoever is created. [try https://www.rsyslog.com/e/2103 ] rsyslogd: EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file! rsyslogd: End of config validation run. Bye. diff --git a/tests/diag.sh b/tests/diag.sh index 6ee1fdf7a7..6dfd987ecd 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -767,7 +767,7 @@ content_check_with_count() { count=$(grep -c -F -- "$1" <${RSYSLOG_OUT_LOG}) fi if [ ${count:=0} -eq $2 ]; then - echo content_check_with_count SUCCESS, \"$1\" occured $2 times + echo content_check_with_count SUCCESS, \"$1\" occurred $2 times break else if [ "$timecounter" == "$timeoutend" ]; then diff --git a/tests/historical/DiagTalker.java b/tests/historical/DiagTalker.java index 147cec34dd..606f58f3d0 100644 --- a/tests/historical/DiagTalker.java +++ b/tests/historical/DiagTalker.java @@ -38,7 +38,7 @@ public static void main(String[] args) throws IOException { try { diagSocket = new Socket(host, port); - diagSocket.setSoTimeout(0); /* wait for lenghty operations */ + diagSocket.setSoTimeout(0); /* wait for lengthy operations */ out = new PrintWriter(diagSocket.getOutputStream(), true); in = new BufferedReader(new InputStreamReader( diagSocket.getInputStream())); diff --git a/tests/imdocker-new-logs-from-start.sh b/tests/imdocker-new-logs-from-start.sh index 6d68b3afa8..aaaf07d041 100755 --- a/tests/imdocker-new-logs-from-start.sh +++ b/tests/imdocker-new-logs-from-start.sh @@ -46,7 +46,7 @@ shutdown_when_empty wait_shutdown echo "file name: $RSYSLOG_OUT_LOG" -echo "\"tailed item\" occured: $(grep -c 'tailed item ' $RSYSLOG_OUT_LOG)/100 (expect less)." +echo "\"tailed item\" occurred: $(grep -c 'tailed item ' $RSYSLOG_OUT_LOG)/100 (expect less)." docker container rm $COOKIE exit_test diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 4ad2a5c583..278c3d6f00 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -36,7 +36,7 @@ * -B The specified file (-I) is binary. No data processing is done by * tcpflood. If multiple connections are specified, data is read in * chunks and spread across the connections without taking any record - * delemiters into account. + * delimiters into account. * -C when input from a file is read, this file is transmitted -C times * (C like cycle, running out of meaningful option switches ;)) * -D randomly drop and re-establish connections. Useful for stress-testing diff --git a/tests/testbench.h b/tests/testbench.h index ae70b52c3b..bdd17b2741 100644 --- a/tests/testbench.h +++ b/tests/testbench.h @@ -66,7 +66,7 @@ static rsRetVal doInit(void) \ #define ENDInit \ finalize_it: \ if(iRet != RS_RET_OK) { \ - printf("failure occured during init of object '%s'\n", pErrObj); \ + printf("failure occurred during init of object '%s'\n", pErrObj); \ } \ \ RETiRet; \ diff --git a/tools/omfwd.c b/tools/omfwd.c index d1729b52f3..1d4ac37d63 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -1412,7 +1412,7 @@ CODESTARTnewActInst pData->compressionMode = COMPRESS_SINGLE_MSG; } else { LogError(0, NO_ERRCODE, "Invalid ziplevel %d specified in " - "forwardig action - NOT turning on compression.", + "forwarding action - NOT turning on compression.", complevel); } } else if(!strcmp(actpblk.descr[i].name, "tcp_framedelimiter")) { @@ -1568,7 +1568,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) pData->compressionMode = COMPRESS_SINGLE_MSG; } else { LogError(0, NO_ERRCODE, "Invalid compression level '%c' specified in " - "forwardig action - NOT turning on compression.", + "forwarding action - NOT turning on compression.", *p); } } else if(*p == 'o') { /* octet-couting based TCP framing? */ diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c index 11515f378f..a1e136be53 100644 --- a/tools/pmrfc3164.c +++ b/tools/pmrfc3164.c @@ -369,7 +369,7 @@ CODESTARTparse * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 */ if(!(pMsg->msgFlags & INTERNAL_MSG)) { - DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); + DBGPRINTF("HOSTNAME and TAG not parsed by user configuration.\n"); } } diff --git a/tools/rsyslog.conf.5 b/tools/rsyslog.conf.5 index 8e8c0c94fd..a968109b4b 100644 --- a/tools/rsyslog.conf.5 +++ b/tools/rsyslog.conf.5 @@ -364,7 +364,7 @@ need to filter based on priority and facility, you should do this with selector not second-class citizens in rsyslog and offer the best performance for this job. .SS Property-Based Filters -Property-based filters are unique to rsyslogd. They allow to filter on any property, like HOSTNAME, +Property-based filters are unique to rsyslogd. They allow one to filter on any property, like HOSTNAME, syslogtag and msg. A property-based filter must start with a colon in column 0. This tells rsyslogd that it is the new diff --git a/tools/rsyslogd.8 b/tools/rsyslogd.8 index e6defc7a16..1f6f1a0571 100644 --- a/tools/rsyslogd.8 +++ b/tools/rsyslogd.8 @@ -168,7 +168,7 @@ is started with the debug option. .TP .B CHLD -Wait for childs if some were born, because of wall'ing messages. +Wait for children if some were born, because of wall'ing messages. .LP .SH SECURITY THREATS There is the potential for the rsyslogd daemon to be diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index c635842f71..649ca40cea 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -623,7 +623,7 @@ rsyslogd_InitGlobalClasses(void) * messages to stderr. -- rgerhards, 2008-04-02 */ fprintf(stderr, "Error during class init for object '%s' - failing...\n", pErrObj); - fprintf(stderr, "rsyslogd initializiation failed - global classes could not be initialized.\n" + fprintf(stderr, "rsyslogd initialization failed - global classes could not be initialized.\n" "Did you do a \"make install\"?\n" "Suggested action: run rsyslogd with -d -n options to see what exactly " "fails.\n"); From eb4011ab3d8b255a6d2ffab9b35622aeadfe755a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 17 Feb 2022 10:56:03 +0100 Subject: [PATCH 034/134] prepare for new scheduled stable release schedule --- ChangeLog | 2 ++ configure.ac | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2d6062c818..d79c4abdec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ ---------------------------------------------------------------------------------------- +Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-15 - 2022-02-14: imfile bugfix: remove cause for "internal error message" (not causing harm) When any message is output into a renamed input file, rsyslogd output the following: diff --git a/configure.ac b/configure.ac index 462213ff04..42baee2413 100644 --- a/configure.ac +++ b/configure.ac @@ -2,9 +2,9 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2202.0],[rsyslog@lists.adiscon.com]) # UPDATE on release -AC_DEFINE(VERSION_YEAR, 21, [year part of real rsyslog version]) # UPDATE on release -AC_DEFINE(VERSION_MONTH, 12, [month part of real rsyslog version]) # UPDATE on release +AC_INIT([rsyslog],[8.2204.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release +AC_DEFINE(VERSION_MONTH, 04, [month part of real rsyslog version]) # UPDATE on release AM_INIT_AUTOMAKE([subdir-objects]) From b07b45bc17522674a1b2e8f294765b22ff91f0c9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 18 Feb 2022 15:01:11 +0100 Subject: [PATCH 035/134] regression fix: script string comparison did not work correctly In rscript, comparison operations on strings did not work correctly and returned false results. This is cause by a regression in commit 5cec5dd634e0. While it fixed number comparisons, it introduced new problems in string comparisons, which were not present before. Note that most items in rsyslog are strings, so this can actually cause some problems. --- grammar/rainerscript.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 35f08d6e70..bd30ef2218 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -3071,7 +3071,7 @@ static int eval_strcmp_like(const struct cnfexpr *__restrict__ const expr, } else { estr_l = var2String(&l, &bMustFree); estr_r = var2String(&r, &bMustFree2); - ret = es_strcmp(estr_l, estr_r) < 0; /*CMP*/ + ret = es_strcmp(estr_l, estr_r); if(bMustFree) es_deleteStr(estr_l); if(bMustFree2) es_deleteStr(estr_r); } From 8c4380bdc619ef11272aec79bf30abfe89215526 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 18 Feb 2022 17:17:15 +0100 Subject: [PATCH 036/134] maintain ChangeLog --- ChangeLog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index d79c4abdec..203d14a634 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,12 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- regression fix: script string comparison did not work correctly + In rscript, comparison operations on strings did not work correctly + and returned false results. This is cause by a regression in commit + 5cec5dd634e0. While it fixed number comparisons, it introduced new + problems in string comparisons, which were not present before. Note + that most items in rsyslog are strings, so this can actually cause + some problems. ---------------------------------------------------------------------------------------- Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-15 - 2022-02-14: imfile bugfix: remove cause for "internal error message" (not causing harm) From 452b62b4a43b526189cfa81d1a930b60fed51e9d Mon Sep 17 00:00:00 2001 From: alakatos Date: Mon, 31 Jan 2022 15:20:37 +0100 Subject: [PATCH 037/134] Make the main message queue part of the config --- action.c | 3 +- dirty.h | 3 +- runtime/queue.c | 279 +++++++++++++++++++++++---------------- runtime/queue.h | 5 +- runtime/rsconf.c | 63 +++++++-- runtime/rsconf.h | 1 + runtime/ruleset.c | 14 +- runtime/unicode-helper.h | 5 + tests/runtime-dummy.c | 1 - tools/rsyslogd.c | 17 +-- 10 files changed, 247 insertions(+), 144 deletions(-) diff --git a/action.c b/action.c index 695967d46a..549108684c 100644 --- a/action.c +++ b/action.c @@ -560,6 +560,7 @@ actionConstructFinalize(action_t *__restrict__ const pThis, struct nvlst *lst) qqueueSetDefaultsActionQueue(pThis->pQueue); qqueueApplyCnfParam(pThis->pQueue, lst); } + qqueueCorrectParams(pThis->pQueue); # undef setQPROP # undef setQPROPstr @@ -1947,7 +1948,7 @@ DEFFUNC_llExecFunc(doActivateActions) { rsRetVal localRet; action_t * const pThis = (action_t*) pData; - localRet = qqueueStart(pThis->pQueue); + localRet = qqueueStart(runConf, pThis->pQueue); if(localRet != RS_RET_OK) { LogError(0, localRet, "error starting up action queue"); if(localRet == RS_RET_FILE_PREFIX_MISSING) { diff --git a/dirty.h b/dirty.h index 2d06fae4aa..9a74465de6 100644 --- a/dirty.h +++ b/dirty.h @@ -38,10 +38,9 @@ rsRetVal __attribute__((deprecated)) parseAndSubmitMessage(const uchar *hname, prop_t *pInputName, const struct syslogTime *stTime, const time_t ttGenTime, ruleset_t *pRuleset); rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst *lst); -rsRetVal startMainQueue(qqueue_t *pQueue); +rsRetVal startMainQueue(rsconf_t *cnf, qqueue_t *pQueue); extern int MarkInterval; -extern qqueue_t *pMsgQueue; /* the main message queue */ #define CONF_VERIFY_PARTIAL_CONF 0x02 /* bit: partial configuration to be checked */ extern int iConfigVerify; /* is this just a config verify run? */ extern int bHaveMainQueue; diff --git a/runtime/queue.c b/runtime/queue.c index 78a4618fb5..e3aac9b4c5 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -498,7 +498,7 @@ StartDA(qqueue_t *pThis) pThis->cryprovNameFull = NULL; } - iRet = qqueueStart(pThis->pqDA); + iRet = qqueueStart(runConf, pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) { errno = 0; /* else an errno is shown in errmsg! */ @@ -1502,6 +1502,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */ pThis->iMinDeqBatchSize = 0; /* conservative default, should still provide good performance */ + pThis->isRunning = 0; pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -2323,25 +2324,30 @@ GetDeqBatchSize(qqueue_t *pThis, int *pVal) * before. */ rsRetVal -qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ +qqueueStart(rsconf_t *cnf, qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; uchar pszBuf[64]; uchar pszQIFNam[MAXFNAME]; int wrk; - int goodval; /* a "good value" to use for comparisons (different objects) */ uchar *qName; size_t lenBuf; assert(pThis != NULL); + /* do not modify the queue if it's already running(happens when dynamic config reload is invoked + * and the queue is used in the new config as well) + */ + if (pThis->isRunning) + FINALIZE; + dbgoprint((obj_t*) pThis, "starting queue\n"); if(pThis->pszSpoolDir == NULL) { /* note: we need to pick the path so late as we do not have * the workdir during early config load */ - if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir(runConf))) == NULL) + if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir(cnf))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); } @@ -2392,116 +2398,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ break; } - if(pThis->iMaxQueueSize < 100 - && (pThis->qType == QUEUETYPE_LINKEDLIST || pThis->qType == QUEUETYPE_FIXED_ARRAY)) { - LogMsg(0, RS_RET_OK_WARN, LOG_WARNING, "Note: queue.size=\"%d\" is very " - "low and can lead to unpredictable results. See also " - "https://www.rsyslog.com/lower-bound-for-queue-sizes/", - pThis->iMaxQueueSize); - } - - /* we need to do a quick check if our water marks are set plausible. If not, - * we correct the most important shortcomings. - */ - goodval = (pThis->iMaxQueueSize / 100) * 60; - if(pThis->iHighWtrMrk != -1 && pThis->iHighWtrMrk < goodval) { - LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, "queue \"%s\": high water mark " - "is set quite low at %d. You should only set it below " - "60%% (%d) if you have a good reason for this.", - obj.GetName((obj_t*) pThis), pThis->iHighWtrMrk, goodval); - } - - if(pThis->iNumWorkerThreads > 1) { - goodval = (pThis->iMaxQueueSize / 100) * 10; - if(pThis->iMinMsgsPerWrkr != -1 && pThis->iMinMsgsPerWrkr < goodval) { - LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, "queue \"%s\": " - "queue.workerThreadMinimumMessage " - "is set quite low at %d. You should only set it below " - "10%% (%d) if you have a good reason for this.", - obj.GetName((obj_t*) pThis), pThis->iMinMsgsPerWrkr, goodval); - } - } - - if(pThis->iDiscardMrk > pThis->iMaxQueueSize) { - LogError(0, RS_RET_PARAM_ERROR, "error: queue \"%s\": " - "queue.discardMark %d is set larger than queue.size", - obj.GetName((obj_t*) pThis), pThis->iDiscardMrk); - } - - goodval = (pThis->iMaxQueueSize / 100) * 80; - if(pThis->iDiscardMrk != -1 && pThis->iDiscardMrk < goodval) { - LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, - "queue \"%s\": queue.discardMark " - "is set quite low at %d. You should only set it below " - "80%% (%d) if you have a good reason for this.", - obj.GetName((obj_t*) pThis), pThis->iDiscardMrk, goodval); - } - - if(pThis->pszFilePrefix != NULL) { /* This means we have a potential DA queue */ - if(pThis->iFullDlyMrk != -1 && pThis->iFullDlyMrk < pThis->iHighWtrMrk) { - LogMsg(0, RS_RET_CONF_WRN_FULLDLY_BELOW_HIGHWTR, LOG_WARNING, - "queue \"%s\": queue.fullDelayMark " - "is set below high water mark. This will result in DA mode " - " NOT being activated for full delayable messages: In many " - "cases this is a configuration error, please check if this " - "is really what you want", - obj.GetName((obj_t*) pThis)); - } - } - - /* now come parameter corrections and defaults */ - if(pThis->iHighWtrMrk < 2 || pThis->iHighWtrMrk > pThis->iMaxQueueSize) { - pThis->iHighWtrMrk = (pThis->iMaxQueueSize / 100) * 90; - if(pThis->iHighWtrMrk == 0) { /* guard against very low max queue sizes! */ - pThis->iHighWtrMrk = pThis->iMaxQueueSize; - } - } - if( pThis->iLowWtrMrk < 2 - || pThis->iLowWtrMrk > pThis->iMaxQueueSize - || pThis->iLowWtrMrk > pThis->iHighWtrMrk ) { - pThis->iLowWtrMrk = (pThis->iMaxQueueSize / 100) * 70; - if(pThis->iLowWtrMrk == 0) { - pThis->iLowWtrMrk = 1; - } - } - - if((pThis->iMinMsgsPerWrkr < 1 - || pThis->iMinMsgsPerWrkr > pThis->iMaxQueueSize) ) { - pThis->iMinMsgsPerWrkr = pThis->iMaxQueueSize / pThis->iNumWorkerThreads; - } - - if(pThis->iFullDlyMrk == -1 || pThis->iFullDlyMrk > pThis->iMaxQueueSize) { - pThis->iFullDlyMrk = (pThis->iMaxQueueSize / 100) * 97; - if(pThis->iFullDlyMrk == 0) { - pThis->iFullDlyMrk = - (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; - } - } - - if(pThis->iLightDlyMrk == 0) { - pThis->iLightDlyMrk = pThis->iMaxQueueSize; - } - - if(pThis->iLightDlyMrk == -1 || pThis->iLightDlyMrk > pThis->iMaxQueueSize) { - pThis->iLightDlyMrk = (pThis->iMaxQueueSize / 100) * 70; - if(pThis->iLightDlyMrk == 0) { - pThis->iLightDlyMrk = - (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; - } - } - - if(pThis->iDiscardMrk < 1 || pThis->iDiscardMrk > pThis->iMaxQueueSize) { - pThis->iDiscardMrk = (pThis->iMaxQueueSize / 100) * 98; - if(pThis->iDiscardMrk == 0) { - /* for very small queues, we disable this by default */ - pThis->iDiscardMrk = pThis->iMaxQueueSize; - } - } - - if(pThis->iMaxQueueSize > 0 && pThis->iDeqBatchSize > pThis->iMaxQueueSize) { - pThis->iDeqBatchSize = pThis->iMaxQueueSize; - } - /* finalize some initializations that could not yet be done because it is * influenced by properties which might have been set after queueConstruct () */ @@ -2621,6 +2517,8 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* note: a child uses it's parent mutex, so do not delete it! */ if(pThis->pqParent == NULL && pThis->mut != NULL) free(pThis->mut); + } else { + pThis->isRunning = 1; } RETiRet; } @@ -3326,6 +3224,122 @@ checkUniqueDiskFile(qqueue_t *const pThis) RETiRet; } +void +qqueueCorrectParams(qqueue_t *pThis) +{ + int goodval; /* a "good value" to use for comparisons (different objects) */ + + if(pThis->iMaxQueueSize < 100 + && (pThis->qType == QUEUETYPE_LINKEDLIST || pThis->qType == QUEUETYPE_FIXED_ARRAY)) { + LogMsg(0, RS_RET_OK_WARN, LOG_WARNING, "Note: queue.size=\"%d\" is very " + "low and can lead to unpredictable results. See also " + "https://www.rsyslog.com/lower-bound-for-queue-sizes/", + pThis->iMaxQueueSize); + } + + /* we need to do a quick check if our water marks are set plausible. If not, + * we correct the most important shortcomings. + */ + goodval = (pThis->iMaxQueueSize / 100) * 60; + if(pThis->iHighWtrMrk != -1 && pThis->iHighWtrMrk < goodval) { + LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, "queue \"%s\": high water mark " + "is set quite low at %d. You should only set it below " + "60%% (%d) if you have a good reason for this.", + obj.GetName((obj_t*) pThis), pThis->iHighWtrMrk, goodval); + } + + if(pThis->iNumWorkerThreads > 1) { + goodval = (pThis->iMaxQueueSize / 100) * 10; + if(pThis->iMinMsgsPerWrkr != -1 && pThis->iMinMsgsPerWrkr < goodval) { + LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, "queue \"%s\": " + "queue.workerThreadMinimumMessage " + "is set quite low at %d. You should only set it below " + "10%% (%d) if you have a good reason for this.", + obj.GetName((obj_t*) pThis), pThis->iMinMsgsPerWrkr, goodval); + } + } + + if(pThis->iDiscardMrk > pThis->iMaxQueueSize) { + LogError(0, RS_RET_PARAM_ERROR, "error: queue \"%s\": " + "queue.discardMark %d is set larger than queue.size", + obj.GetName((obj_t*) pThis), pThis->iDiscardMrk); + } + + goodval = (pThis->iMaxQueueSize / 100) * 80; + if(pThis->iDiscardMrk != -1 && pThis->iDiscardMrk < goodval) { + LogMsg(0, RS_RET_CONF_PARSE_WARNING, LOG_WARNING, + "queue \"%s\": queue.discardMark " + "is set quite low at %d. You should only set it below " + "80%% (%d) if you have a good reason for this.", + obj.GetName((obj_t*) pThis), pThis->iDiscardMrk, goodval); + } + + if(pThis->pszFilePrefix != NULL) { /* This means we have a potential DA queue */ + if(pThis->iFullDlyMrk != -1 && pThis->iFullDlyMrk < pThis->iHighWtrMrk) { + LogMsg(0, RS_RET_CONF_WRN_FULLDLY_BELOW_HIGHWTR, LOG_WARNING, + "queue \"%s\": queue.fullDelayMark " + "is set below high water mark. This will result in DA mode " + " NOT being activated for full delayable messages: In many " + "cases this is a configuration error, please check if this " + "is really what you want", + obj.GetName((obj_t*) pThis)); + } + } + + /* now come parameter corrections and defaults */ + if(pThis->iHighWtrMrk < 2 || pThis->iHighWtrMrk > pThis->iMaxQueueSize) { + pThis->iHighWtrMrk = (pThis->iMaxQueueSize / 100) * 90; + if(pThis->iHighWtrMrk == 0) { /* guard against very low max queue sizes! */ + pThis->iHighWtrMrk = pThis->iMaxQueueSize; + } + } + if( pThis->iLowWtrMrk < 2 + || pThis->iLowWtrMrk > pThis->iMaxQueueSize + || pThis->iLowWtrMrk > pThis->iHighWtrMrk ) { + pThis->iLowWtrMrk = (pThis->iMaxQueueSize / 100) * 70; + if(pThis->iLowWtrMrk == 0) { + pThis->iLowWtrMrk = 1; + } + } + + if((pThis->iMinMsgsPerWrkr < 1 + || pThis->iMinMsgsPerWrkr > pThis->iMaxQueueSize) ) { + pThis->iMinMsgsPerWrkr = pThis->iMaxQueueSize / pThis->iNumWorkerThreads; + } + + if(pThis->iFullDlyMrk == -1 || pThis->iFullDlyMrk > pThis->iMaxQueueSize) { + pThis->iFullDlyMrk = (pThis->iMaxQueueSize / 100) * 97; + if(pThis->iFullDlyMrk == 0) { + pThis->iFullDlyMrk = + (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; + } + } + + if(pThis->iLightDlyMrk == 0) { + pThis->iLightDlyMrk = pThis->iMaxQueueSize; + } + + if(pThis->iLightDlyMrk == -1 || pThis->iLightDlyMrk > pThis->iMaxQueueSize) { + pThis->iLightDlyMrk = (pThis->iMaxQueueSize / 100) * 70; + if(pThis->iLightDlyMrk == 0) { + pThis->iLightDlyMrk = + (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; + } + } + + if(pThis->iDiscardMrk < 1 || pThis->iDiscardMrk > pThis->iMaxQueueSize) { + pThis->iDiscardMrk = (pThis->iMaxQueueSize / 100) * 98; + if(pThis->iDiscardMrk == 0) { + /* for very small queues, we disable this by default */ + pThis->iDiscardMrk = pThis->iMaxQueueSize; + } + } + + if(pThis->iMaxQueueSize > 0 && pThis->iDeqBatchSize > pThis->iMaxQueueSize) { + pThis->iDeqBatchSize = pThis->iMaxQueueSize; + } +} + /* apply all params from param block to queue. Must be called before * finalizing. This supports the v6 config system. Defaults were already * set during queue creation. The pvals object is destructed by this @@ -3480,6 +3494,43 @@ qqueueApplyCnfParam(qqueue_t *pThis, struct nvlst *lst) RETiRet; } +/* return 1 if the content of two qqueue_t structs equal */ +int +queuesEqual(qqueue_t *pOld, qqueue_t *pNew) +{ + return ( + NUM_EQUALS(qType) && + NUM_EQUALS(iMaxQueueSize) && + NUM_EQUALS(iDeqBatchSize) && + NUM_EQUALS(iMinDeqBatchSize) && + NUM_EQUALS(toMinDeqBatchSize) && + NUM_EQUALS(sizeOnDiskMax) && + NUM_EQUALS(iHighWtrMrk) && + NUM_EQUALS(iLowWtrMrk) && + NUM_EQUALS(iFullDlyMrk) && + NUM_EQUALS(iLightDlyMrk) && + NUM_EQUALS(iDiscardMrk) && + NUM_EQUALS(iDiscardSeverity) && + NUM_EQUALS(iPersistUpdCnt) && + NUM_EQUALS(bSyncQueueFiles) && + NUM_EQUALS(iNumWorkerThreads) && + NUM_EQUALS(toQShutdown) && + NUM_EQUALS(toActShutdown) && + NUM_EQUALS(toEnq) && + NUM_EQUALS(toWrkShutdown) && + NUM_EQUALS(iMinMsgsPerWrkr) && + NUM_EQUALS(iMaxFileSize) && + NUM_EQUALS(bSaveOnShutdown) && + NUM_EQUALS(iDeqSlowdown) && + NUM_EQUALS(iDeqtWinFromHr) && + NUM_EQUALS(iDeqtWinToHr) && + NUM_EQUALS(iSmpInterval) && + NUM_EQUALS(takeFlowCtlFromMsg) && + USTR_EQUALS(pszFilePrefix) && + USTR_EQUALS(cryprovName) + ); +} + /* some simple object access methods */ DEFpropSetMeth(qqueue, bSyncQueueFiles, int) diff --git a/runtime/queue.h b/runtime/queue.h index 2367c34c9c..dd989bd280 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -188,6 +188,7 @@ struct queue_s { STATSCOUNTER_DEF(ctrNFDscrd, mutCtrNFDscrd) int ctrMaxqsize; /* NOT guarded by a mutex */ int iSmpInterval; /* line interval of sampling logs */ + int isRunning; }; @@ -201,7 +202,7 @@ struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); rsRetVal qqueueEnqMsg(qqueue_t *pThis, flowControl_t flwCtlType, smsg_t *pMsg); -rsRetVal qqueueStart(qqueue_t *pThis); +rsRetVal qqueueStart(rsconf_t *cnf, qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, @@ -213,6 +214,8 @@ void qqueueSetDefaultsActionQueue(qqueue_t *pThis); void qqueueDbgPrint(qqueue_t *pThis); rsRetVal qqueueShutdownWorkers(qqueue_t *pThis); void qqueueDoneLoadCnf(void); +int queuesEqual(qqueue_t *pOld, qqueue_t *pNew); +void qqueueCorrectParams(qqueue_t *pThis); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 6504708efc..7aa5372fd5 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -249,6 +249,7 @@ static void cnfSetDefaults(rsconf_t *pThis) pThis->globals.mainQ.bMainMsgQSaveOnShutdown = 1; pThis->globals.mainQ.iMainMsgQueueDeqtWinFromHr = 0; pThis->globals.mainQ.iMainMsgQueueDeqtWinToHr = 25; + pThis->pMsgQueue = NULL; pThis->globals.parser.cCCEscapeChar = '#'; pThis->globals.parser.bDropTrailingLF = 1; @@ -894,36 +895,61 @@ startInputModules(void) return RS_RET_OK; /* intentional: we do not care about module errors */ } - -/* activate the main queue */ +/* load the main queue */ static rsRetVal -activateMainQueue(void) +loadMainQueue(void) { - struct cnfobj *mainqCnfObj; DEFiRet; + struct cnfobj *mainqCnfObj; mainqCnfObj = glbl.GetmainqCnfObj(); - DBGPRINTF("activateMainQueue: mainq cnf obj ptr is %p\n", mainqCnfObj); + DBGPRINTF("loadMainQueue: mainq cnf obj ptr is %p\n", mainqCnfObj); /* create message queue */ - iRet = createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"), + iRet = createMainQueue(&loadConf->pMsgQueue, UCHAR_CONSTANT("main Q"), (mainqCnfObj == NULL) ? NULL : mainqCnfObj->nvlst); - if(iRet == RS_RET_OK) { - iRet = startMainQueue(pMsgQueue); + if (iRet == RS_RET_OK) { + if (runConf != NULL) { /* dynamic config reload */ + int areEqual = queuesEqual(loadConf->pMsgQueue, runConf->pMsgQueue); + DBGPRINTF("Comparison of old and new main queues: %d\n", areEqual); + if (areEqual) { /* content of the new main queue is the same as it was in previous conf */ + qqueueDestruct(&loadConf->pMsgQueue); + loadConf->pMsgQueue = runConf->pMsgQueue; + } + } } + + if(iRet != RS_RET_OK) { + /* no queue is fatal, we need to give up in that case... */ + fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); + FINALIZE; + } +finalize_it: + glblDestructMainqCnfObj(); + RETiRet; +} + +/* activate the main queue */ +static rsRetVal +activateMainQueue(void) +{ + DEFiRet; + + DBGPRINTF("activateMainQueue: will try to activate main queue %p\n", runConf->pMsgQueue); + + iRet = startMainQueue(runConf, runConf->pMsgQueue); if(iRet != RS_RET_OK) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); FINALIZE; } - if(runConf->globals.mainQ.MainMsgQueType == QUEUETYPE_DIRECT) { // ourConf -> runConf + if(runConf->globals.mainQ.MainMsgQueType == QUEUETYPE_DIRECT) { PREFER_STORE_0_TO_INT(&bHaveMainQueue); } else { PREFER_STORE_1_TO_INT(&bHaveMainQueue); } DBGPRINTF("Main processing queue is initialized and running\n"); finalize_it: - glblDestructMainqCnfObj(); RETiRet; } @@ -940,6 +966,20 @@ setUmask(int iUmask) return RS_RET_OK; } +/* Remove resources from previous config */ +static void +cleanupOldCnf(rsconf_t *cnf) +{ + if (cnf == NULL) + FINALIZE; + + if (runConf->pMsgQueue != cnf->pMsgQueue) + qqueueDestruct(&cnf->pMsgQueue); + +finalize_it: + return; +} + /* Activate an already-loaded configuration. The configuration will become * the new running conf (if successful). Note that in theory this method may @@ -951,6 +991,7 @@ static rsRetVal activate(rsconf_t *cnf) { DEFiRet; + rsconf_t *runCnfOld = runConf; /* at this point, we "switch" over to the running conf */ runConf = cnf; @@ -985,6 +1026,7 @@ activate(rsconf_t *cnf) qqueueDoneLoadCnf(); /* we no longer need config-load-only data structures */ dbgprintf("configuration %p activated\n", cnf); + cleanupOldCnf(runCnfOld); finalize_it: RETiRet; @@ -1455,6 +1497,7 @@ load(rsconf_t **cnf, uchar *confFile) tellModulesCheckConfig(); CHKiRet(validateConf(loadConf)); + CHKiRet(loadMainQueue()); /* we are done checking the config - now validate if we should actually run or not. * If not, terminate. -- rgerhards, 2008-07-25 diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 7d0fd85619..44b4c3f9a5 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -248,6 +248,7 @@ struct rsconf_s { * Of course, we need to debate if we shall change that some time... */ timezones_t timezones; + qqueue_t *pMsgQueue; /* the main message queue */ }; diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 64d75f0a48..0e281f1d6b 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -184,7 +184,7 @@ DEFFUNC_llExecFunc(doActivateRulesetQueues) dbgprintf("Activating Ruleset Queue[%p] for Ruleset %s\n", pThis->pQueue, pThis->pszName); if(pThis->pQueue != NULL) - startMainQueue(pThis->pQueue); + startMainQueue(runConf, pThis->pQueue); RETiRet; } /* activate all ruleset queues */ @@ -385,7 +385,7 @@ callForeachObject(struct cnfstmt *stmt, json_object *arr, smsg_t *pMsg, wti_t *p * we put an assertion in its place. */ assert(key == NULL); - + RETiRet; } @@ -397,7 +397,7 @@ execForeach(struct cnfstmt *const stmt, smsg_t *const pMsg, wti_t *const pWti) /* arr can either be an array or an associative-array (obj) */ arr = cnfexprEvalCollection(stmt->d.s_foreach.iter->collection, pMsg, pWti); - + if (arr == NULL) { DBGPRINTF("foreach loop skipped, as object to iterate upon is empty\n"); FINALIZE; @@ -558,12 +558,12 @@ execReloadLookupTable(struct cnfstmt *stmt) if (t == NULL) { ABORT_FINALIZE(RS_RET_NONE); } - + iRet = lookupReload(t, stmt->d.s_reload_lookup_table.stub_value); /* Note that reload dispatched above is performed asynchronously, on a different thread. So rsRetVal it returns means it was triggered successfully, and not that it was reloaded successfully. */ - + finalize_it: RETiRet; } @@ -737,7 +737,7 @@ static qqueue_t* GetRulesetQueue(ruleset_t *pThis) { ISOBJ_TYPE_assert(pThis, ruleset); - return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue; + return (pThis->pQueue == NULL) ? runConf->pMsgQueue : pThis->pQueue; } @@ -1078,7 +1078,7 @@ rulesetProcessCnf(struct cnfobj *o) cnfparamsPrint(&rspblk, pvals); nameIdx = cnfparamGetIdx(&rspblk, "name"); rsName = (uchar*)es_str2cstr(pvals[nameIdx].val.d.estr, NULL); - + localRet = rulesetGetRuleset(loadConf, &pRuleset, rsName); if(localRet == RS_RET_OK) { LogError(0, RS_RET_RULESET_EXISTS, diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 2c5ba6ccb0..61dc2e0a91 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -36,4 +36,9 @@ #define UCHAR_CONSTANT(x) ((uchar*) (x)) #define CHAR_CONVERT(x) ((char*) (x)) +/* Compare values of two instances/configs/queues especially during dynamic config reload */ +#define USTR_EQUALS(var) \ + ((pOld->var == NULL) ? (pNew->var == NULL) : (pNew->var != NULL && !ustrcmp(pOld->var, pNew->var))) +#define NUM_EQUALS(var) (pOld->var == pNew->var) + #endif /* multi-include protection */ diff --git a/tests/runtime-dummy.c b/tests/runtime-dummy.c index 05776453a0..a3827dd8ac 100644 --- a/tests/runtime-dummy.c +++ b/tests/runtime-dummy.c @@ -33,7 +33,6 @@ int bReduceRepeatMsgs = 0; int bActExecWhenPrevSusp = 0; int iActExecOnceInterval = 1; int MarkInterval = 30; -void *pMsgQueue = NULL; void cflineClassic(void) {}; void selectorAddList(void) {}; diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index c635842f71..2fc9f67c0d 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -196,7 +196,6 @@ int bHaveMainQueue = 0;/* set to 1 if the main queue - in queueing mode - is ava * If the main queue is either not yet ready or not running in * queueing mode (mode DIRECT!), then this is set to 0. */ -qqueue_t *pMsgQueue = NULL; /* default main message queue */ prop_t *pInternalInputName = NULL; /* there is only one global inputName for all internally-generated messages */ ratelimit_t *internalMsg_ratelimiter = NULL; /* ratelimiter for rsyslog-own messages */ int send_to_all = 0; /* send message to all IPv4/IPv6 addresses */ @@ -805,18 +804,20 @@ rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst * qqueueSetDefaultsRulesetQueue(*ppQueue); qqueueApplyCnfParam(*ppQueue, lst); } + qqueueCorrectParams(*ppQueue); + RETiRet; } rsRetVal -startMainQueue(qqueue_t *const pQueue) +startMainQueue(rsconf_t *cnf, qqueue_t *const pQueue) { DEFiRet; - CHKiRet_Hdlr(qqueueStart(pQueue)) { + CHKiRet_Hdlr(qqueueStart(cnf, pQueue)) { /* no queue is fatal, we need to give up in that case... */ LogError(0, iRet, "could not start (ruleset) main message queue"); pQueue->qType = QUEUETYPE_DIRECT; - CHKiRet_Hdlr(qqueueStart(pQueue)) { + CHKiRet_Hdlr(qqueueStart(cnf, pQueue)) { /* no queue is fatal, we need to give up in that case... */ LogError(0, iRet, "fatal error: could not even start queue in direct mode"); } @@ -1078,7 +1079,7 @@ submitMsg2(smsg_t *pMsg) } pRuleset = MsgGetRuleset(pMsg); - pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); + pQueue = (pRuleset == NULL) ? runConf->pMsgQueue : ruleset.GetRulesetQueue(pRuleset); /* if a plugin logs a message during shutdown, the queue may no longer exist */ if(pQueue == NULL) { @@ -1108,7 +1109,7 @@ multiSubmitMsg2(multi_submit_t *const pMultiSub) FINALIZE; pRuleset = MsgGetRuleset(pMultiSub->ppMsgs[0]); - pQueue = (pRuleset == NULL) ? pMsgQueue : ruleset.GetRulesetQueue(pRuleset); + pQueue = (pRuleset == NULL) ? runConf->pMsgQueue : ruleset.GetRulesetQueue(pRuleset); /* if a plugin logs a message during shutdown, the queue may no longer exist */ if(pQueue == NULL) { @@ -2032,8 +2033,8 @@ deinitAll(void) /* drain queue (if configured so) and stop main queue worker thread pool */ DBGPRINTF("Terminating main queue...\n"); - qqueueDestruct(&pMsgQueue); - pMsgQueue = NULL; + qqueueDestruct(&runConf->pMsgQueue); + runConf->pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining * repeated msgs. From 5caa989ce2d4e9b91a11780eae7a8848c211c55b Mon Sep 17 00:00:00 2001 From: alakatos Date: Tue, 1 Mar 2022 14:36:16 +0100 Subject: [PATCH 038/134] glblGetMaxLine() fix when config not yet exists, fixes #4810 --- runtime/glbl.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runtime/glbl.c b/runtime/glbl.c index 871cbbc927..958a522c01 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -85,6 +85,7 @@ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup, e static uchar *LocalHostNameOverride = NULL;/* user-overridden hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup, except HUP */ static uchar *LocalDomain = NULL;/* our local domain name - read-only after startup, except HUP */ +static int iMaxLine = 8096; int bTerminateInputs = 0; /* global switch that inputs shall terminate ASAP (1=> terminate) */ int glblUnloadModules = 1; char** glblDbgFiles = NULL; @@ -191,8 +192,8 @@ static struct cnfparamvals *cnfparamvals = NULL; int glblGetMaxLine(rsconf_t *cnf) { - assert(cnf != NULL); - return(cnf->globals.iMaxLine); + /* glblGetMaxLine might be invoked before our configuration exists */ + return((cnf != NULL) ? cnf->globals.iMaxLine : iMaxLine); } From 0b830970c2d8a31293448b7fdd4570d46b1a81d4 Mon Sep 17 00:00:00 2001 From: MIZUTA Takeshi Date: Thu, 3 Mar 2022 10:34:54 +0900 Subject: [PATCH 039/134] Don't refer to errno when the pthread library fails When the pthread library fails, errno is referenced even though errno is not set. Fix to refer to the return code of the pthread library instead of errno. --- plugins/imtcp/imtcp.c | 2 +- plugins/omelasticsearch/omelasticsearch.c | 5 +++-- runtime/lib_ksils12.c | 9 +++++---- runtime/tcpsrv.c | 2 +- runtime/wti.c | 5 +++-- threads.c | 4 ++-- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index a6256acad8..b51184a152 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -976,7 +976,7 @@ startSrvWrkr(tcpsrv_etry_t *const etry) pthread_attr_setstacksize(&sessThrdAttr, 4096*1024); r = pthread_create(&etry->tid, &sessThrdAttr, RunServerThread, etry); if(r != 0) { - LogError(errno, NO_ERRCODE, "imtcp error creating server thread"); + LogError(r, NO_ERRCODE, "imtcp error creating server thread"); /* we do NOT abort, as other servers may run - after all, we logged an error */ } pthread_attr_destroy(&sessThrdAttr); diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index ed9359732c..b6fa620a01 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -231,9 +231,10 @@ static rsRetVal curlSetup(wrkrInstanceData_t *pWrkrData); BEGINcreateInstance CODESTARTcreateInstance + int r; pData->fdErrFile = -1; - if(pthread_mutex_init(&pData->mutErrFile, NULL) != 0) { - LogError(errno, RS_RET_ERR, "omelasticsearch: cannot create " + if((r = pthread_mutex_init(&pData->mutErrFile, NULL)) != 0) { + LogError(r, RS_RET_ERR, "omelasticsearch: cannot create " "error file mutex, failing this action"); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/runtime/lib_ksils12.c b/runtime/lib_ksils12.c index ee1131d2a1..da0cc2399c 100644 --- a/runtime/lib_ksils12.c +++ b/runtime/lib_ksils12.c @@ -821,14 +821,15 @@ seedIVKSI(ksifile ksi) static int create_signer_thread(rsksictx ctx) { + int r; if (ctx->signer_state != SIGNER_STARTED) { - if (pthread_mutex_init(&ctx->module_lock, 0)) - report(ctx, "pthread_mutex_init: %s", strerror(errno)); + if ((r = pthread_mutex_init(&ctx->module_lock, 0))) + report(ctx, "pthread_mutex_init: %s", strerror(r)); ctx->signer_queue = ProtectedQueue_new(10); ctx->signer_state = SIGNER_INIT; - if (pthread_create(&ctx->signer_thread, NULL, signer_thread, ctx)) { - report(ctx, "pthread_mutex_init: %s", strerror(errno)); + if ((r = pthread_create(&ctx->signer_thread, NULL, signer_thread, ctx))) { + report(ctx, "pthread_mutex_init: %s", strerror(r)); ctx->signer_state = SIGNER_IDLE; return RSGTE_INTERNAL; } diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c index 6b46510091..c8d4b7ee5c 100644 --- a/runtime/tcpsrv.c +++ b/runtime/tcpsrv.c @@ -1693,7 +1693,7 @@ startWorkerPool(void) if(r == 0) { wrkrInfo[i].enabled = 1; } else { - LogError(errno, NO_ERRCODE, "tcpsrv error creating thread"); + LogError(r, NO_ERRCODE, "tcpsrv error creating thread"); } } pthread_attr_destroy(&sessThrdAttr); diff --git a/runtime/wti.c b/runtime/wti.c index 23c295a436..96bae4d209 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -91,11 +91,12 @@ wtiGetState(wti_t *pThis) void ATTR_NONNULL() wtiJoinThrd(wti_t *const pThis) { + int r; ISOBJ_TYPE_assert(pThis, wti); if(wtiGetState(pThis) == WRKTHRD_WAIT_JOIN) { DBGPRINTF("%s: joining terminated worker\n", wtiGetDbgHdr(pThis)); - if(pthread_join(pThis->thrdID, NULL) != 0) { - LogMsg(errno, RS_RET_INTERNAL_ERROR, LOG_WARNING, + if((r = pthread_join(pThis->thrdID, NULL)) != 0) { + LogMsg(r, RS_RET_INTERNAL_ERROR, LOG_WARNING, "rsyslog bug? wti cannot join terminated wrkr"); } DBGPRINTF("%s: worker fully terminated\n", wtiGetDbgHdr(pThis)); diff --git a/threads.c b/threads.c index 0e4875d2a1..2fd4bbb86c 100644 --- a/threads.c +++ b/threads.c @@ -130,7 +130,7 @@ thrdTerminateNonCancel(thrdInfo_t *pThis) pThis->name, runConf->globals.inputTimeoutShutdown); const int r = pthread_kill(pThis->thrdID, SIGTTIN); if(r != 0) { - LogError(errno, RS_RET_INTERNAL_ERROR, "error terminating thread %s " + LogError(r, RS_RET_INTERNAL_ERROR, "error terminating thread %s " "this may cause shutdown issues", pThis->name); } ret = d_pthread_cond_timedwait(&pThis->condThrdTerm, &pThis->mutThrd, &tTimeout); @@ -146,7 +146,7 @@ thrdTerminateNonCancel(thrdInfo_t *pThis) break; } else if(ret != 0) { char errStr[1024]; - int err = errno; + int err = ret; rs_strerror_r(err, errStr, sizeof(errStr)); DBGPRINTF("input thread term: cond_wait returned with error %d: %s\n", err, errStr); From f64ef442d5af7e0dbe541f9d6e8baa9c01a10d16 Mon Sep 17 00:00:00 2001 From: Sergio Arroutbi Date: Tue, 1 Mar 2022 11:41:20 +0100 Subject: [PATCH 040/134] Add option to limit error file to configured size action.errorfile.maxsize has been added to enable option to limit the amount of bytes dumped to configured errorfile fixes #4733 Signed-off-by: Sergio Arroutbi --- action.c | 36 +++++++++++++++++++++--------- action.h | 2 ++ tests/Makefile.am | 3 +++ tests/action-tx-errfile-maxsize.sh | 35 +++++++++++++++++++++++++++++ tests/diag.sh | 11 +++++++++ tests/omfwd-errfile-maxsize.sh | 17 ++++++++++++++ 6 files changed, 94 insertions(+), 10 deletions(-) create mode 100755 tests/action-tx-errfile-maxsize.sh create mode 100755 tests/omfwd-errfile-maxsize.sh diff --git a/action.c b/action.c index 62b4daaaee..9ecfb69ffa 100644 --- a/action.c +++ b/action.c @@ -189,6 +189,7 @@ static struct cnfparamdescr cnfparamdescr[] = { { "name", eCmdHdlrGetWord, 0 }, /* legacy: actionname */ { "type", eCmdHdlrString, CNFPARAM_REQUIRED }, /* legacy: actionname */ { "action.errorfile", eCmdHdlrString, 0 }, + { "action.errorfile.maxsize", eCmdHdlrInt, 0 }, { "action.writeallmarkmessages", eCmdHdlrBinary, 0 }, /* legacy: actionwriteallmarkmessages */ { "action.execonlyeverynthtime", eCmdHdlrInt, 0 }, /* legacy: actionexeconlyeverynthtime */ { "action.execonlyeverynthtimetimeout", eCmdHdlrInt, 0 }, /* legacy: actionexeconlyeverynthtimetimeout */ @@ -391,6 +392,8 @@ rsRetVal actionConstruct(action_t **ppThis) pThis->iResumeRetryCount = 0; pThis->pszName = NULL; pThis->pszErrFile = NULL; + pThis->maxErrFileSize = 0; + pThis->errFileWritten = 0; pThis->pszExternalStateFile = NULL; pThis->fdErrFile = -1; pThis->bWriteAllMarkMsgs = 1; @@ -1445,16 +1448,27 @@ actionWriteErrorFile(action_t *__restrict__ const pThis, const rsRetVal ret, char *const rendered = strdup((char*)fjson_object_to_json_string(etry)); if(rendered == NULL) goto done; - const size_t toWrite = strlen(rendered) + 1; - /* note: we use the '\0' inside the string to store a LF - we do not - * otherwise need it and it safes us a copy/realloc. - */ - rendered[toWrite-1] = '\n'; /* NO LONGER A STRING! */ - const ssize_t wrRet = write(pThis->fdErrFile, rendered, toWrite); - if(wrRet != (ssize_t) toWrite) { - LogError(errno, RS_RET_IO_ERROR, - "action %s: error writing errorFile %s, write returned %lld", - pThis->pszName, pThis->pszErrFile, (long long) wrRet); + + size_t toWrite = strlen(rendered) + 1; + // Check if need to truncate the amount of bytes to write + if (pThis->maxErrFileSize > 0) { + if (pThis->errFileWritten + toWrite > pThis->maxErrFileSize) { + // Truncate to the pending available + toWrite = pThis->maxErrFileSize - pThis->errFileWritten; + } + pThis->errFileWritten += toWrite; + } + if(toWrite > 0) { + /* note: we use the '\0' inside the string to store a LF - we do not + * otherwise need it and it safes us a copy/realloc. + */ + rendered[toWrite-1] = '\n'; /* NO LONGER A STRING! */ + const ssize_t wrRet = write(pThis->fdErrFile, rendered, toWrite); + if(wrRet != (ssize_t) toWrite) { + LogError(errno, RS_RET_IO_ERROR, + "action %s: error writing errorFile %s, write returned %lld", + pThis->pszName, pThis->pszErrFile, (long long) wrRet); + } } free(rendered); @@ -2039,6 +2053,8 @@ actionApplyCnfParam(action_t * const pAction, struct cnfparamvals * const pvals) continue; /* this is handled seperately during module select! */ } else if(!strcmp(pblk.descr[i].name, "action.errorfile")) { pAction->pszErrFile = es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(pblk.descr[i].name, "action.errorfile.maxsize")) { + pAction->maxErrFileSize = pvals[i].val.d.n; } else if(!strcmp(pblk.descr[i].name, "action.externalstate.file")) { pAction->pszExternalStateFile = es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(pblk.descr[i].name, "action.writeallmarkmessages")) { diff --git a/action.h b/action.h index dd2bb9f862..f7cd5abceb 100644 --- a/action.h +++ b/action.h @@ -74,6 +74,8 @@ struct action_s { /* error file */ const char *pszErrFile; int fdErrFile; + size_t maxErrFileSize; + size_t errFileWritten; pthread_mutex_t mutErrFile; /* external stat file system */ const char *pszExternalStateFile; diff --git a/tests/Makefile.am b/tests/Makefile.am index 00e8685d10..11bfaac771 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -718,6 +718,7 @@ TESTS += \ mysql-actq-mt.sh \ mysql-actq-mt-withpause.sh \ action-tx-single-processing.sh \ + action-tx-errfile-maxsize.sh \ action-tx-errfile.sh mysql-basic.log: mysqld-start.log @@ -2333,6 +2334,8 @@ EXTRA_DIST= \ sndrcv_omudpspoof_nonstdpt.sh \ sndrcv_gzip.sh \ action-tx-single-processing.sh \ + omfwd-errfile-maxsize.sh \ + action-tx-errfile-maxsize.sh \ action-tx-errfile.sh \ testsuites/action-tx-errfile.result \ pipeaction.sh \ diff --git a/tests/action-tx-errfile-maxsize.sh b/tests/action-tx-errfile-maxsize.sh new file mode 100755 index 0000000000..eaff6d7595 --- /dev/null +++ b/tests/action-tx-errfile-maxsize.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# part of the rsyslog project, released under ASL 2.0 + +. ${srcdir:=.}/diag.sh init + +export NUMMESSAGES=50 # enough to generate big file +export MAX_ERROR_SIZE=100 + +generate_conf +add_conf ' +$ModLoad ../plugins/ommysql/.libs/ommysql +global(errormessagestostderr.maxnumber="5") + +template(type="string" name="tpl" string="insert into SystemEvents (Message, Facility) values (\"%msg%\", %$!facility%)" option.sql="on") + +if((not($msg contains "error")) and ($msg contains "msgnum:")) then { + set $.num = field($msg, 58, 2); + if $.num % 2 == 0 then { + set $!facility = $syslogfacility; + } else { + set $/cntr = 0; + } + action(type="ommysql" name="mysql_action_errfile_maxsize" server="127.0.0.1" template="tpl" + db="'$RSYSLOG_DYNNAME'" uid="rsyslog" pwd="testbench" action.errorfile="'$RSYSLOG2_OUT_LOG'" action.errorfile.maxsize="'$MAX_ERROR_SIZE'") +} +' +mysql_prep_for_test +startup +injectmsg +shutdown_when_empty +wait_shutdown +mysql_get_data +check_file_exists ${RSYSLOG2_OUT_LOG} +file_size_check ${RSYSLOG2_OUT_LOG} ${MAX_ERROR_SIZE} +exit_test diff --git a/tests/diag.sh b/tests/diag.sh index 6dfd987ecd..55874903be 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -2522,6 +2522,17 @@ wait_for_stats_flush() { echo "stats push registered" } +# Check file exists and is of a particular size +# $1 - file to check +# $2 - size to check +file_size_check() { + local size=$(ls -l $1 | awk {'print $5'}) + if [ "${size}" != "${2}" ]; then + printf 'File:[%s] has unexpected size. Expected:[%d], Size:[%d]\n', $1 $2 $size + error_exit 1 + fi + return 0 +} case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason diff --git a/tests/omfwd-errfile-maxsize.sh b/tests/omfwd-errfile-maxsize.sh new file mode 100755 index 0000000000..91756a8b75 --- /dev/null +++ b/tests/omfwd-errfile-maxsize.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init + +export MAX_ERROR_SIZE=1999 + +generate_conf +add_conf ' +action(type="omfwd" target="1.2.3.4" port="1234" Protocol="tcp" NetworkNamespace="doesNotExist" + action.errorfile="'$RSYSLOG2_OUT_LOG'" action.errorfile.maxsize="'$MAX_ERROR_SIZE'") +' +startup +shutdown_when_empty +wait_shutdown +check_file_exists ${RSYSLOG2_OUT_LOG} +file_size_check ${RSYSLOG2_OUT_LOG} ${MAX_ERROR_SIZE} +exit_test From cb73734c7803761a48ed0d5e94d86544f73fa81e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 7 Mar 2022 11:43:44 +0100 Subject: [PATCH 041/134] maintain ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 203d14a634..7868b6a03e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- regression bugfix: rsyslog may segfault during startup + glblGetMaxLine() might be called even before the main configuration file exists + resulting unexpected behavior, most probably segmentation fault. This is addressed + by re-introducing the old default of 8KiB. The problem was introduced earlier in + 2022. - regression fix: script string comparison did not work correctly In rscript, comparison operations on strings did not work correctly and returned false results. This is cause by a regression in commit From 53fb41a1ebd7d2d31209ef0003a0601eb1011927 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 16 Mar 2022 12:51:02 +0100 Subject: [PATCH 042/134] maintain ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7868b6a03e..e9a230c959 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-03-16: refactor: ake the main message queue part of the config + The intent of this patch is to make the main message queue part of the main config. + It will help us to proceed towards dynamic configuration reload. - regression bugfix: rsyslog may segfault during startup glblGetMaxLine() might be called even before the main configuration file exists resulting unexpected behavior, most probably segmentation fault. This is addressed From a7ffb1524fc894906bd0363788bd8c0165553502 Mon Sep 17 00:00:00 2001 From: Aleksey Deyneko Date: Sun, 20 Mar 2022 19:49:08 +0300 Subject: [PATCH 043/134] Fix: check ReportSuspension option before print message about action retry --- action.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/action.c b/action.c index 809d29249d..0962ec4672 100644 --- a/action.c +++ b/action.c @@ -763,11 +763,13 @@ static void ATTR_NONNULL() actionRetry(action_t * const pThis, wti_t * const pWt { setSuspendMessageConfVars(pThis); actionSetState(pThis, pWti, ACT_STATE_RTRY); - LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING, - "action '%s' suspended (module '%s'), retry %d. There should " - "be messages before this one giving the reason for suspension.", - pThis->pszName, pThis->pMod->pszName, - getActionNbrResRtry(pWti, pThis)); + if(pThis->bReportSuspension) { + LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING, + "action '%s' suspended (module '%s'), retry %d. There should " + "be messages before this one giving the reason for suspension.", + pThis->pszName, pThis->pMod->pszName, + getActionNbrResRtry(pWti, pThis)); + } incActionResumeInRow(pWti, pThis); } From 988c3c2a67dece47d7a68e2412bc3dcc2c8d6f73 Mon Sep 17 00:00:00 2001 From: MIZUTA Takeshi Date: Tue, 22 Mar 2022 16:59:24 +0900 Subject: [PATCH 044/134] Fix message typo when pthread_create fails in KSI-LS12 support library --- runtime/lib_ksils12.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/lib_ksils12.c b/runtime/lib_ksils12.c index da0cc2399c..3320c6acf3 100644 --- a/runtime/lib_ksils12.c +++ b/runtime/lib_ksils12.c @@ -829,7 +829,7 @@ create_signer_thread(rsksictx ctx) { ctx->signer_state = SIGNER_INIT; if ((r = pthread_create(&ctx->signer_thread, NULL, signer_thread, ctx))) { - report(ctx, "pthread_mutex_init: %s", strerror(r)); + report(ctx, "pthread_create: %s", strerror(r)); ctx->signer_state = SIGNER_IDLE; return RSGTE_INTERNAL; } From 452f55efa066b8a77284aa4a734d27b2c9fc3935 Mon Sep 17 00:00:00 2001 From: alakatos Date: Wed, 23 Mar 2022 10:00:26 +0100 Subject: [PATCH 045/134] Move the parser directive to the main config --- runtime/parser.c | 100 +++++++++++++++++---------------------------- runtime/parser.h | 9 ++-- runtime/rsconf.c | 7 +++- runtime/rsconf.h | 12 ++++++ runtime/ruleset.c | 2 +- runtime/typedefs.h | 1 + 6 files changed, 64 insertions(+), 67 deletions(-) diff --git a/runtime/parser.c b/runtime/parser.c index ea3514d3dd..a22ad72af6 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -58,27 +58,14 @@ static char hexdigit[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; -/* This is the list of all parsers known to us. - * This is also used to unload all modules on shutdown. - */ -parserList_t *pParsLstRoot = NULL; - -/* this is the list of the default parsers, to be used if no others - * are specified. - */ -parserList_t *pDfltParsLst = NULL; - -/* intialize (but NOT allocate) a parser list. Primarily meant as a hook - * which can be used to extend the list in the future. So far, just sets - * it to NULL. - */ -static rsRetVal -InitParserList(parserList_t **pListRoot) -{ - *pListRoot = NULL; - return RS_RET_OK; -} - +BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(parser) + DBGPRINTF("destructing parser '%s'\n", pThis->pName); + if(pThis->pInst != NULL) { + pThis->pModule->mod.pm.freeParserInst(pThis->pInst); + } + free(pThis->pName); +ENDobjDestruct(parser) /* destruct a parser list. The list elements are destroyed, but the parser objects * themselves are not modified. (That is done at a late stage during rsyslogd @@ -146,12 +133,12 @@ printParserList(parserList_t *pList) /* find a parser based on the provided name */ static rsRetVal -FindParser(parser_t **ppParser, uchar *pName) +FindParser(parserList_t *pParserListRoot, parser_t **ppParser, uchar *pName) { parserList_t *pThis; DEFiRet; - for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + for(pThis = pParserListRoot ; pThis != NULL ; pThis = pThis->pNext) { if(ustrcmp(pThis->pParser->pName, pName) == 0) { *ppParser = pThis->pParser; FINALIZE; /* found it, iRet still eq. OK! */ @@ -178,8 +165,8 @@ AddDfltParser(uchar *pName) parser_t *pParser; DEFiRet; - CHKiRet(FindParser(&pParser, pName)); - CHKiRet(AddParserToList(&pDfltParsLst, pParser)); + CHKiRet(FindParser(loadConf->parsers.pParsLstRoot, &pParser, pName)); + CHKiRet(AddParserToList(&loadConf->parsers.pDfltParsLst, pParser)); DBGPRINTF("Parser '%s' added to default parser set.\n", pName); finalize_it: @@ -250,7 +237,7 @@ static rsRetVal parserConstructFinalize(parser_t *pThis) DEFiRet; ISOBJ_TYPE_assert(pThis, parser); - CHKiRet(AddParserToList(&pParsLstRoot, pThis)); + CHKiRet(AddParserToList(&loadConf->parsers.pParsLstRoot, pThis)); DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName); finalize_it: @@ -293,15 +280,6 @@ parserConstructViaModAndName(modInfo_t *__restrict__ pMod, uchar *const __restri free(pParser); RETiRet; } -BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ -CODESTARTobjDestruct(parser) - DBGPRINTF("destructing parser '%s'\n", pThis->pName); - if(pThis->pInst != NULL) { - pThis->pModule->mod.pm.freeParserInst(pThis->pInst); - } - free(pThis->pName); -ENDobjDestruct(parser) - /* uncompress a received message if it is compressed. * pMsg->pszRawMsg buffer is updated. @@ -643,10 +621,10 @@ ParseMsg(smsg_t *pMsg) */ pParserList = ruleset.GetParserList(runConf, pMsg); if(pParserList == NULL) { - pParserList = pDfltParsLst; + pParserList = runConf->parsers.pDfltParsLst; } DBGPRINTF("parse using parser list %p%s.\n", pParserList, - (pParserList == pDfltParsLst) ? " (the default list)" : ""); + (pParserList == runConf->parsers.pDfltParsLst) ? " (the default list)" : ""); bIsSanitized = RSFALSE; bPRIisParsed = RSFALSE; @@ -691,6 +669,28 @@ ParseMsg(smsg_t *pMsg) finalize_it: RETiRet; } + +/* This destroys the master parserlist and all of its parser entries. + * Parser modules are NOT unloaded, rsyslog does that at a later stage + * for all dynamically loaded modules. + */ +static rsRetVal +destroyMasterParserList(parserList_t *pParserListRoot) +{ + DEFiRet; + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = pParserListRoot; + while(pParsLst != NULL) { + parserDestruct(&pParsLst->pParser); + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } + RETiRet; +} + /* queryInterface function-- rgerhards, 2009-11-03 */ BEGINobjQueryInterface(parser) @@ -712,39 +712,18 @@ CODESTARTobjQueryInterface(parser) pIf->SetDoPRIParsing = SetDoPRIParsing; pIf->ParseMsg = ParseMsg; pIf->SanitizeMsg = SanitizeMsg; - pIf->InitParserList = InitParserList; pIf->DestructParserList = DestructParserList; pIf->AddParserToList = AddParserToList; pIf->AddDfltParser = AddDfltParser; pIf->FindParser = FindParser; + pIf->destroyMasterParserList = destroyMasterParserList; finalize_it: ENDobjQueryInterface(parser) -/* This destroys the master parserlist and all of its parser entries. MUST only be - * done when the module is shut down. Parser modules are NOT unloaded, rsyslog - * does that at a later stage for all dynamically loaded modules. - */ -static void -destroyMasterParserList(void) -{ - parserList_t *pParsLst; - parserList_t *pParsLstDel; - - pParsLst = pParsLstRoot; - while(pParsLst != NULL) { - parserDestruct(&pParsLst->pParser); - pParsLstDel = pParsLst; - pParsLst = pParsLst->pNext; - free(pParsLstDel); - } -} - /* Exit our class. * rgerhards, 2009-11-04 */ BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */ - DestructParserList(&pDfltParsLst); - destroyMasterParserList(); objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); @@ -760,7 +739,4 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(ruleset, CORE_COMPONENT)); - - InitParserList(&pParsLstRoot); - InitParserList(&pDfltParsLst); ENDObjClassInit(parser) diff --git a/runtime/parser.h b/runtime/parser.h index 166a483e1f..d028b1a5ba 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -50,18 +50,21 @@ BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetName)(parser_t *pThis, uchar *name); rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod); rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int); - rsRetVal (*FindParser)(parser_t **ppThis, uchar*name); - rsRetVal (*InitParserList)(parserList_t **pListRoot); + rsRetVal (*FindParser)(parserList_t *pParserListRoot, parser_t **ppThis, uchar*name); rsRetVal (*DestructParserList)(parserList_t **pListRoot); rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser); + rsRetVal (*destroyMasterParserList)(parserList_t *pParserListRoot); /* static functions */ rsRetVal (*ParseMsg)(smsg_t *pMsg); rsRetVal (*SanitizeMsg)(smsg_t *pMsg); rsRetVal (*AddDfltParser)(uchar *); ENDinterface(parser) -#define parserCURR_IF_VERSION 2 /* increment whenever you change the interface above! */ +#define parserCURR_IF_VERSION 3 /* increment whenever you change the interface above! */ /* version changes 2 SetDoSanitization removed, no longer needed + 3 InitParserList removed, no longer needed + destroyMasterParserList added + findParser extended with new parameter specifying the parser list */ void printParserList(parserList_t *pList); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 7aa5372fd5..f9e2e02d63 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -260,6 +260,9 @@ static void cnfSetDefaults(rsconf_t *pThis) pThis->globals.parser.bParserEscapeCCCStyle = 0; pThis->globals.parser.bPermitSlashInProgramname = 0; pThis->globals.parser.bParseHOSTNAMEandTAG = 1; + + pThis->parsers.pDfltParsLst = NULL; + pThis->parsers.pParsLstRoot = NULL; } @@ -316,6 +319,8 @@ CODESTARTobjDestruct(rsconf) perctileBucketsDestruct(); ochDeleteAll(); freeTimezones(pThis); + parser.DestructParserList(&pThis->parsers.pDfltParsLst); + parser.destroyMasterParserList(pThis->parsers.pParsLstRoot); free(pThis->globals.mainQ.pszMainMsgQFName); free(pThis->globals.pszConfDAGFile); free(pThis->globals.pszWorkDir); @@ -413,7 +418,7 @@ parserProcessCnf(struct cnfobj *o) cnfparamsPrint(&parserpblk, pvals); paramIdx = cnfparamGetIdx(&parserpblk, "name"); parserName = (uchar*)es_str2cstr(pvals[paramIdx].val.d.estr, NULL); - if(parser.FindParser(&myparser, parserName) != RS_RET_PARSER_NOT_FOUND) { + if(parser.FindParser(loadConf->parsers.pParsLstRoot, &myparser, parserName) != RS_RET_PARSER_NOT_FOUND) { LogError(0, RS_RET_PARSER_NAME_EXISTS, "parser module name '%s' already exists", parserName); ABORT_FINALIZE(RS_RET_PARSER_NAME_EXISTS); diff --git a/runtime/rsconf.h b/runtime/rsconf.h index 44b4c3f9a5..5a734acff7 100644 --- a/runtime/rsconf.h +++ b/runtime/rsconf.h @@ -204,6 +204,17 @@ struct templates_s { struct template *lastStatic; /* last static element of the template list */ }; +struct parsers_s { + /* This is the list of all parsers known to us. + * This is also used to unload all modules on shutdown. + */ + parserList_t *pParsLstRoot; + + /* this is the list of the default parsers, to be used if no others + * are specified. + */ + parserList_t *pDfltParsLst; +}; struct actions_s { /* number of active actions */ @@ -235,6 +246,7 @@ struct rsconf_s { globals_t globals; defaults_t defaults; templates_t templates; + parsers_t parsers; lookup_tables_t lu_tabs; dynstats_buckets_t dynstats_buckets; perctile_buckets_t perctile_buckets; diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 0e281f1d6b..c137ee3b01 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -1027,7 +1027,7 @@ doRulesetAddParser(ruleset_t *pRuleset, uchar *pName) DEFiRet; CHKiRet(objUse(parser, CORE_COMPONENT)); - iRet = parser.FindParser(&pParser, pName); + iRet = parser.FindParser(loadConf->parsers.pParsLstRoot, &pParser, pName); if(iRet == RS_RET_PARSER_NOT_FOUND) { LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time " "(maybe defined too late in rsyslog.conf?)", pName); diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 706ab3b2da..6c199cfb3c 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -95,6 +95,7 @@ typedef struct statsobj_s statsobj_t; typedef void (*statsobj_read_notifier_t)(statsobj_t *, void *); typedef struct nsd_epworkset_s nsd_epworkset_t; typedef struct templates_s templates_t; +typedef struct parsers_s parsers_t; typedef struct queuecnf_s queuecnf_t; typedef struct parsercnf_s parsercnf_t; typedef struct rulesets_s rulesets_t; From aaad6623be1f7adde739a37880752c275bc4ec4d Mon Sep 17 00:00:00 2001 From: alakatos Date: Wed, 23 Mar 2022 13:15:38 +0100 Subject: [PATCH 046/134] Add missing include to fix compilation error, resolves #4827 --- contrib/omfile-hardened/omfile-hardened.c | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/omfile-hardened/omfile-hardened.c b/contrib/omfile-hardened/omfile-hardened.c index 83686afc72..2b14a2c5c1 100644 --- a/contrib/omfile-hardened/omfile-hardened.c +++ b/contrib/omfile-hardened/omfile-hardened.c @@ -73,6 +73,7 @@ #include "cryprov.h" #include "parserif.h" #include "janitor.h" +#include "rsconf.h" MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP From 857f77906f95681aa15c7ba3f88cbda8952f7e5f Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Thu, 24 Mar 2022 16:34:09 +0800 Subject: [PATCH 047/134] rsyslogd: adjust the order of doHUP() and processImInternal() After call doHUP(), probably there is a internal log in the list. However, it will not be wrote out immediately, because the mainloop will be blocked at pselect in wait_timeout() until a long timeout or next message occur. More deadly, the log may be lost if the deamon exits unexpectedly. We might as well put processImInternal() after doHUP(), so that the message will be flushed out immediately. Fixes: 723f6fdfa6(rsyslogd: Fix race between signals and main loop timeout) Signed-off-by: Yun Zhou --- tools/rsyslogd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index 6dbbb0313d..373324394d 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -1952,8 +1952,6 @@ mainloop(void) sigaddset(&sigblockset, SIGHUP); do { - processImInternal(); - pthread_sigmask(SIG_BLOCK, &sigblockset, &origmask); if(bChildDied) { reapChild(); @@ -1965,6 +1963,8 @@ mainloop(void) bHadHUP = 0; } + processImInternal(); + if(bFinished) break; /* exit as quickly as possible */ From 2387c248f397c887b2e4fa856781276073d2c09d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 25 Mar 2022 12:10:36 +0100 Subject: [PATCH 048/134] testbench: change git access protocol to cover github change github disabled anon access via git protocol, so we need to adjust this to https no functional change --- tests/solaris/prep-libfastjson.sh | 2 +- tests/solaris/prep-librelp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/solaris/prep-libfastjson.sh b/tests/solaris/prep-libfastjson.sh index cdf1043bbd..afafe9406a 100755 --- a/tests/solaris/prep-libfastjson.sh +++ b/tests/solaris/prep-libfastjson.sh @@ -5,7 +5,7 @@ mkdir local_env mkdir local_env/install cd local_env pwd -git clone git://github.com/rsyslog/libfastjson +git clone http://github.com/rsyslog/libfastjson cd libfastjson git log -2 autoreconf -fvi diff --git a/tests/solaris/prep-librelp.sh b/tests/solaris/prep-librelp.sh index 06b1a14e53..199e82b31f 100755 --- a/tests/solaris/prep-librelp.sh +++ b/tests/solaris/prep-librelp.sh @@ -5,7 +5,7 @@ mkdir local_env mkdir local_env/install cd local_env pwd -git clone git://github.com/rsyslog/librelp +git clone http://github.com/rsyslog/librelp cd librelp git log -2 env | grep FLAGS From 46c821bd2ac42670e3784ae4638f5025d0e71496 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 31 Mar 2022 09:32:02 +0200 Subject: [PATCH 049/134] maintain ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index e9a230c959..aef6af9731 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-03-20: refactor: Move the parser directive to the main config + Thanks to Attila Lakatos for the patch. - 2022-03-16: refactor: ake the main message queue part of the config The intent of this patch is to make the main message queue part of the main config. It will help us to proceed towards dynamic configuration reload. From 71b81626efb258cd4e519c2a5722dd6eb4a618c7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 18 Feb 2022 16:17:00 +0100 Subject: [PATCH 050/134] testbench: add tests for rscript comparison operations --- tests/Makefile.am | 41 +++++++++++++++++-- ...rison_all.sh => rscript_compare-common.sh} | 4 +- tests/rscript_compare_num-num-vg.sh | 5 +++ tests/rscript_compare_num-num.sh | 4 ++ tests/rscript_compare_num-numstr-vg.sh | 5 +++ tests/rscript_compare_num-numstr.sh | 4 ++ tests/rscript_compare_num-str-vg.sh | 5 +++ tests/rscript_compare_num-str.sh | 4 ++ tests/rscript_compare_numstr-num-vg.sh | 5 +++ tests/rscript_compare_numstr-num.sh | 4 ++ tests/rscript_compare_numstr-numstr-vg.sh | 5 +++ tests/rscript_compare_numstr-numstr.sh | 4 ++ tests/rscript_compare_numstr-str-vg.sh | 5 +++ tests/rscript_compare_numstr-str.sh | 4 ++ tests/rscript_compare_str-num-vg.sh | 5 +++ tests/rscript_compare_str-num.sh | 4 ++ tests/rscript_compare_str-numstr-vg.sh | 5 +++ tests/rscript_compare_str-numstr.sh | 4 ++ tests/rscript_compare_str-str-vg.sh | 5 +++ tests/rscript_compare_str-str.sh | 4 ++ 20 files changed, 120 insertions(+), 6 deletions(-) rename tests/{rscript_number_comparison_all.sh => rscript_compare-common.sh} (95%) create mode 100755 tests/rscript_compare_num-num-vg.sh create mode 100755 tests/rscript_compare_num-num.sh create mode 100755 tests/rscript_compare_num-numstr-vg.sh create mode 100755 tests/rscript_compare_num-numstr.sh create mode 100755 tests/rscript_compare_num-str-vg.sh create mode 100755 tests/rscript_compare_num-str.sh create mode 100755 tests/rscript_compare_numstr-num-vg.sh create mode 100755 tests/rscript_compare_numstr-num.sh create mode 100755 tests/rscript_compare_numstr-numstr-vg.sh create mode 100755 tests/rscript_compare_numstr-numstr.sh create mode 100755 tests/rscript_compare_numstr-str-vg.sh create mode 100755 tests/rscript_compare_numstr-str.sh create mode 100755 tests/rscript_compare_str-num-vg.sh create mode 100755 tests/rscript_compare_str-num.sh create mode 100755 tests/rscript_compare_str-numstr-vg.sh create mode 100755 tests/rscript_compare_str-numstr.sh create mode 100755 tests/rscript_compare_str-str-vg.sh create mode 100755 tests/rscript_compare_str-str.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 11bfaac771..60cc2f8687 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -428,9 +428,17 @@ TESTS += \ rscript_lt_var.sh \ rscript_ne.sh \ rscript_ne_var.sh \ - rscript_number_comparison_all.sh \ rscript_number_comparison_LE.sh \ rscript_number_comparison_LT.sh \ + rscript_compare_str-numstr.sh \ + rscript_compare_str-num.sh \ + rscript_compare_numstr-str.sh \ + rscript_compare_num-str.sh \ + rscript_compare_numstr-numstr.sh \ + rscript_compare_numstr-num.sh \ + rscript_compare_num-numstr.sh \ + rscript_compare_num-num.sh \ + rscript_compare_str-str.sh \ rscript_num2ipv4.sh \ rscript_int2Hex.sh \ rscript_trim.sh \ @@ -531,8 +539,16 @@ TESTS += \ udp-msgreduc-orgmsg-vg.sh \ tcp-msgreduc-vg.sh \ rscript_field-vg.sh \ - rscript_number_comparison_all-vg.sh \ rscript_number_comparison_LE-vg.sh \ + rscript_compare_str-str-vg.sh \ + rscript_compare_str-num-vg.sh \ + rscript_compare_str-numstr-vg.sh \ + rscript_compare_num-str-vg.sh \ + rscript_compare_numstr-str-vg.sh \ + rscript_compare_numstr-num-vg.sh \ + rscript_compare_numstr-numstr-vg.sh \ + rscript_compare_num-num-vg.sh \ + rscript_compare_num-numstr-vg.sh \ unused_lookup_table-vg.sh \ lookup_table-vg.sh \ lookup_table_no_hup_reload-vg.sh \ @@ -1844,11 +1860,28 @@ EXTRA_DIST= \ rscript_gt_var.sh \ rscript_ne.sh \ rscript_ne_var.sh \ - rscript_number_comparison_all.sh \ - rscript_number_comparison_all-vg.sh \ rscript_number_comparison_LE.sh \ rscript_number_comparison_LE-vg.sh \ rscript_number_comparison_LT.sh \ + rscript_compare_str-numstr.sh \ + rscript_compare_str-num.sh \ + rscript_compare_numstr-str.sh \ + rscript_compare_num-str.sh \ + rscript_compare_numstr-numstr.sh \ + rscript_compare_numstr-num.sh \ + rscript_compare_num-numstr.sh \ + rscript_compare_num-num.sh \ + rscript_compare_str-str.sh \ + rscript_compare_str-str-vg.sh \ + rscript_compare_str-num-vg.sh \ + rscript_compare_str-numstr-vg.sh \ + rscript_compare_num-str-vg.sh \ + rscript_compare_numstr-str-vg.sh \ + rscript_compare_numstr-num-vg.sh \ + rscript_compare_numstr-numstr-vg.sh \ + rscript_compare_num-num-vg.sh \ + rscript_compare_num-numstr-vg.sh \ + rscript_compare-common.sh \ rscript_num2ipv4.sh \ rscript_int2Hex.sh \ rscript_trim.sh \ diff --git a/tests/rscript_number_comparison_all.sh b/tests/rscript_compare-common.sh similarity index 95% rename from tests/rscript_number_comparison_all.sh rename to tests/rscript_compare-common.sh index 4b2a7def0f..71810c6931 100755 --- a/tests/rscript_number_comparison_all.sh +++ b/tests/rscript_compare-common.sh @@ -8,8 +8,8 @@ template(name="outfmt" type="list") { property(name="$!result") constant(value="\n") } -set $!lower_nr = 1111; -set $!higher_nr = 2222; +set $!lower_nr = '$LOWER_VAL'; +set $!higher_nr = '$HIGHER_VAL'; if $!lower_nr <= $!higher_nr then { set $!result = "<= RIGHT"; } diff --git a/tests/rscript_compare_num-num-vg.sh b/tests/rscript_compare_num-num-vg.sh new file mode 100755 index 0000000000..f08a92d55b --- /dev/null +++ b/tests/rscript_compare_num-num-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='1' +export HIGHER_VAL='2' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_num-num.sh b/tests/rscript_compare_num-num.sh new file mode 100755 index 0000000000..c3a75d8e53 --- /dev/null +++ b/tests/rscript_compare_num-num.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='1' +export HIGHER_VAL='2' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_num-numstr-vg.sh b/tests/rscript_compare_num-numstr-vg.sh new file mode 100755 index 0000000000..889f5359f0 --- /dev/null +++ b/tests/rscript_compare_num-numstr-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='1' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_num-numstr.sh b/tests/rscript_compare_num-numstr.sh new file mode 100755 index 0000000000..be510b12ab --- /dev/null +++ b/tests/rscript_compare_num-numstr.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='1' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_num-str-vg.sh b/tests/rscript_compare_num-str-vg.sh new file mode 100755 index 0000000000..334a6b20ed --- /dev/null +++ b/tests/rscript_compare_num-str-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='1' +export HIGHER_VAL='"b"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_num-str.sh b/tests/rscript_compare_num-str.sh new file mode 100755 index 0000000000..5f2bc87349 --- /dev/null +++ b/tests/rscript_compare_num-str.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='1' +export HIGHER_VAL='"b"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-num-vg.sh b/tests/rscript_compare_numstr-num-vg.sh new file mode 100755 index 0000000000..32f3686c2c --- /dev/null +++ b/tests/rscript_compare_numstr-num-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"1"' +export HIGHER_VAL='2' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-num.sh b/tests/rscript_compare_numstr-num.sh new file mode 100755 index 0000000000..31ec55e85b --- /dev/null +++ b/tests/rscript_compare_numstr-num.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"1"' +export HIGHER_VAL='2' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-numstr-vg.sh b/tests/rscript_compare_numstr-numstr-vg.sh new file mode 100755 index 0000000000..6663f21157 --- /dev/null +++ b/tests/rscript_compare_numstr-numstr-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"1"' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-numstr.sh b/tests/rscript_compare_numstr-numstr.sh new file mode 100755 index 0000000000..b58fe78c1c --- /dev/null +++ b/tests/rscript_compare_numstr-numstr.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"1"' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-str-vg.sh b/tests/rscript_compare_numstr-str-vg.sh new file mode 100755 index 0000000000..82d727bb57 --- /dev/null +++ b/tests/rscript_compare_numstr-str-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"1"' +export HIGHER_VAL='"abc"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_numstr-str.sh b/tests/rscript_compare_numstr-str.sh new file mode 100755 index 0000000000..509e36f561 --- /dev/null +++ b/tests/rscript_compare_numstr-str.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"1"' +export HIGHER_VAL='"abc"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-num-vg.sh b/tests/rscript_compare_str-num-vg.sh new file mode 100755 index 0000000000..fe5e8e69e7 --- /dev/null +++ b/tests/rscript_compare_str-num-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"-"' +export HIGHER_VAL='1' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-num.sh b/tests/rscript_compare_str-num.sh new file mode 100755 index 0000000000..5c1e1860ee --- /dev/null +++ b/tests/rscript_compare_str-num.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"-"' +export HIGHER_VAL='1' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-numstr-vg.sh b/tests/rscript_compare_str-numstr-vg.sh new file mode 100755 index 0000000000..3c83884109 --- /dev/null +++ b/tests/rscript_compare_str-numstr-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"-"' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-numstr.sh b/tests/rscript_compare_str-numstr.sh new file mode 100755 index 0000000000..abffeedcd0 --- /dev/null +++ b/tests/rscript_compare_str-numstr.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"-"' +export HIGHER_VAL='"2"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-str-vg.sh b/tests/rscript_compare_str-str-vg.sh new file mode 100755 index 0000000000..21a27a3f44 --- /dev/null +++ b/tests/rscript_compare_str-str-vg.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export LOWER_VAL='"a"' +export HIGHER_VAL='"b"' +source ${srcdir:-.}/rscript_compare-common.sh diff --git a/tests/rscript_compare_str-str.sh b/tests/rscript_compare_str-str.sh new file mode 100755 index 0000000000..a7d8baf4fe --- /dev/null +++ b/tests/rscript_compare_str-str.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export LOWER_VAL='"a"' +export HIGHER_VAL='"b"' +source ${srcdir:-.}/rscript_compare-common.sh From af02203dee93f1cdf313fc76882389e0bb817bd6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 31 Mar 2022 12:38:22 +0200 Subject: [PATCH 051/134] maintain ChangeLog --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index aef6af9731..dbac6d4729 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- core bugfix: make internal logs emitted during HUP procesing appear quicker + After call doHUP(), probably there is a internal log in the list. However, it + will not be wrote out immediately, because the mainloop will be blocked at + pselect in wait_timeout() until a long timeout or next message occur. + More deadly, the log may be lost if the deamon exits unexpectedly. + We might as well put processImInternal() after doHUP(), so that the message + will be flushed out immediately. + Fixes: 723f6fdfa6(rsyslogd: Fix race between signals and main loop timeout) + Thanks to Yun Zhou for the patch. - 2022-03-20: refactor: Move the parser directive to the main config Thanks to Attila Lakatos for the patch. - 2022-03-16: refactor: ake the main message queue part of the config From f20324f44790002272f200413c1546ae5c1445b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 31 Mar 2022 12:55:13 +0200 Subject: [PATCH 052/134] CI: move docker worker to github action This makes things a bit more reliable and save us much-needed time as we want to minimize our maintenance of the docker farm. --- .github/workflows/run_ubuntu_18_gtls_only.yml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/run_ubuntu_18_gtls_only.yml diff --git a/.github/workflows/run_ubuntu_18_gtls_only.yml b/.github/workflows/run_ubuntu_18_gtls_only.yml new file mode 100644 index 0000000000..b585ccbc0c --- /dev/null +++ b/.github/workflows/run_ubuntu_18_gtls_only.yml @@ -0,0 +1,64 @@ +# Copyright 2020 Rainer Gerhards and Others +# +# https://github.com/rsyslog/rsyslog-pkg-ubuntu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# References: +# +# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options +# https://github.com/settings/notifications +# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog + + +--- +name: check ubuntu 18 GnuTLS only + +on: + pull_request: + +jobs: + check_run: + runs-on: ubuntu-18.04 + timeout-minutes: 50 + continue-on-error: true + + steps: + - name: git checkout project + uses: actions/checkout@v1 + + - name: run container CI pipeline + run: | + chmod -R go+rw . + export RSYSLOG_CONTAINER_UID="" # use default + export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' + export CFLAGS='-g' + export CC='clang' + export USE_AUTO_DEBUG='off' + export CI_MAKE_OPT='-j20' + export CI_MAKE_CHECK_OPT='-j8' + export CI_CHECK_CMD='check' + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' + export CC='clang' + export CFLAGS="-g" + # note: we need pathes in container, thus /rsyslog vs. $(pwd) in TSAN_OPTIONS + export ABORT_ALL_ON_TEST_FAIL='NO' + # note: we OVERRIDE all of the container's default configure options to keep it very slim + export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE="-enable-testbench --enable-omstdout --enable-imdiag --disable-fmhttp --enable-valgrind --disable-default-tests --enable-gnutls --disable-imfile-tests --enable-extended-tests" + devtools/devcontainer.sh --rm devtools/run-ci.sh + + - name: show error logs (if we errored) + if: ${{ failure() || cancelled() }} + run: | + devtools/gather-check-logs.sh + cat failed-tests.log From 4b697a01e4d581b090a88bd750642c4c9d29762b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Apr 2022 09:23:27 +0200 Subject: [PATCH 053/134] maintain ChangeLog --- ChangeLog | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index dbac6d4729..9d71ac53ec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,11 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? -- core bugfix: make internal logs emitted during HUP procesing appear quicker +- 2022-04-01: core bugfix: "action suspended" message was emitted even when turned off + Most messages were diasabled, but there was one part of the code that ignored the + user configuration. + Thanks to Deyneko Aleksey for the patch. +- 2022-03-31: testbench: add more tests for rscript comparison operations +- 2022-03-31: core bugfix: make internal logs emitted during HUP procesing appear quicker After call doHUP(), probably there is a internal log in the list. However, it will not be wrote out immediately, because the mainloop will be blocked at pselect in wait_timeout() until a long timeout or next message occur. From 5734b5853a05163c13fc95e23a3d997a5b63578b Mon Sep 17 00:00:00 2001 From: Sergio Arroutbi Date: Tue, 8 Mar 2022 15:28:15 +0100 Subject: [PATCH 054/134] Ensure errorfile max. size correct handling When action.errorfile.maxsize configuration option is enabled and error file already has a certain size smaller than max size configured, it is increasing higher than configured max size as the error file is considered to be zero in code. This fix reads current error file size and limits the size to the maximum size configured fixes #4821 Signed-off-by: Sergio Arroutbi --- action.c | 16 +++++++--- action.h | 2 +- tests/Makefile.am | 1 + tests/errfile-basic.sh | 46 +++++++++++++++++++++++++++ tests/omfwd-errfile-maxsize-filled.sh | 18 +++++++++++ tests/testbench_test_failed_rsyslog | 1 + 6 files changed, 79 insertions(+), 5 deletions(-) create mode 100755 tests/errfile-basic.sh create mode 100755 tests/omfwd-errfile-maxsize-filled.sh create mode 100644 tests/testbench_test_failed_rsyslog diff --git a/action.c b/action.c index 9ecfb69ffa..7fb1ef255f 100644 --- a/action.c +++ b/action.c @@ -393,7 +393,7 @@ rsRetVal actionConstruct(action_t **ppThis) pThis->pszName = NULL; pThis->pszErrFile = NULL; pThis->maxErrFileSize = 0; - pThis->errFileWritten = 0; + pThis->currentErrFileSize = 0; pThis->pszExternalStateFile = NULL; pThis->fdErrFile = -1; pThis->bWriteAllMarkMsgs = 1; @@ -1430,6 +1430,14 @@ actionWriteErrorFile(action_t *__restrict__ const pThis, const rsRetVal ret, pThis->pszName, pThis->pszErrFile); goto done; } + if (pThis->maxErrFileSize > 0) { + struct stat statbuf; + if (fstat(pThis->fdErrFile, &statbuf) == -1) { + LogError(errno, RS_RET_ERR, "failed to fstat %s", pThis->pszErrFile); + goto done; + } + pThis->currentErrFileSize = statbuf.st_size; + } } for(int i = 0 ; i < nparams ; ++i) { @@ -1452,11 +1460,11 @@ actionWriteErrorFile(action_t *__restrict__ const pThis, const rsRetVal ret, size_t toWrite = strlen(rendered) + 1; // Check if need to truncate the amount of bytes to write if (pThis->maxErrFileSize > 0) { - if (pThis->errFileWritten + toWrite > pThis->maxErrFileSize) { + if (pThis->currentErrFileSize + toWrite > pThis->maxErrFileSize) { // Truncate to the pending available - toWrite = pThis->maxErrFileSize - pThis->errFileWritten; + toWrite = pThis->maxErrFileSize - pThis->currentErrFileSize; } - pThis->errFileWritten += toWrite; + pThis->currentErrFileSize += toWrite; } if(toWrite > 0) { /* note: we use the '\0' inside the string to store a LF - we do not diff --git a/action.h b/action.h index f7cd5abceb..16da900fc5 100644 --- a/action.h +++ b/action.h @@ -75,7 +75,7 @@ struct action_s { const char *pszErrFile; int fdErrFile; size_t maxErrFileSize; - size_t errFileWritten; + size_t currentErrFileSize; pthread_mutex_t mutErrFile; /* external stat file system */ const char *pszExternalStateFile; diff --git a/tests/Makefile.am b/tests/Makefile.am index 11bfaac771..0f4b6ac780 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -2335,6 +2335,7 @@ EXTRA_DIST= \ sndrcv_gzip.sh \ action-tx-single-processing.sh \ omfwd-errfile-maxsize.sh \ + omfwd-errfile-maxsize-filled.sh \ action-tx-errfile-maxsize.sh \ action-tx-errfile.sh \ testsuites/action-tx-errfile.result \ diff --git a/tests/errfile-basic.sh b/tests/errfile-basic.sh new file mode 100755 index 0000000000..831e4cc394 --- /dev/null +++ b/tests/errfile-basic.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# This file is part of the rsyslog project, released under ASL 2.0 + +# Starting actual testbench +. ${srcdir:=.}/diag.sh init + +export NUMMESSAGES=1000 + +port=80 +omhttp_start_server $port + +echo "============================> RSYSLOG_DYNNAME:$RSYSLOG_DYNNAME <============================" + +generate_conf +add_conf ' +template(name="tpl" type="string" + string="{\"msgnum\":\"%msg:F,58:2%\"}") + +module(load="../contrib/omhttp/.libs/omhttp") + +if $msg contains "msgnum:" then + action( + # Payload + name="my_http_action" + type="omhttp" + action.errorfile="'$RSYSLOG_DYNNAME/omhttp_sarroutb.error.log'" + action.errorfile.maxsize="2000" + template="tpl" + + server="localhost" + serverport="'$port'" + restpath="my/endpoint" + batch="off" + + # Auth + usehttps="off" + ) +' +startup +injectmsg +shutdown_when_empty +wait_shutdown +omhttp_get_data $port my/endpoint +omhttp_stop_server +seq_check +exit_test diff --git a/tests/omfwd-errfile-maxsize-filled.sh b/tests/omfwd-errfile-maxsize-filled.sh new file mode 100755 index 0000000000..25a08ece87 --- /dev/null +++ b/tests/omfwd-errfile-maxsize-filled.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +ERRFILE="$RSYSLOG_DYNNAME.err" +export MAX_ERROR_SIZE=1999 +export INITIAL_FILE_SIZE=$((MAX_ERROR_SIZE - 100)) +dd if=/dev/urandom of=${ERRFILE} bs=1 count=${INITIAL_FILE_SIZE} +generate_conf +add_conf ' +action(type="omfwd" target="1.2.3.4" port="1234" Protocol="tcp" NetworkNamespace="doesNotExist" + action.errorfile="'$ERRFILE'" action.errorfile.maxsize="'$MAX_ERROR_SIZE'") +' +startup +shutdown_when_empty +wait_shutdown +check_file_exists ${ERRFILE} +file_size_check ${ERRFILE} ${MAX_ERROR_SIZE} +exit_test diff --git a/tests/testbench_test_failed_rsyslog b/tests/testbench_test_failed_rsyslog new file mode 100644 index 0000000000..9565b809a4 --- /dev/null +++ b/tests/testbench_test_failed_rsyslog @@ -0,0 +1 @@ +./omfwd-errfile-maxsize-filled.sh From 68ad91fc28c476048b95a64d7b994e5f490ee4df Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Apr 2022 12:26:57 +0200 Subject: [PATCH 055/134] bugfix: cosmetic data races there was a more or less cosmetic data race which could happen when children processes died in quick sequence. Even then, no real harm happened, as all children were reaped eventually. A similar data race exists for HUP processing. However, these races polluted TSAN test runs, and so we fixed them. --- tools/rsyslogd.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tools/rsyslogd.c b/tools/rsyslogd.c index 373324394d..6b8aa93a99 100644 --- a/tools/rsyslogd.c +++ b/tools/rsyslogd.c @@ -178,7 +178,9 @@ void rsyslogdDoDie(int sig); #endif /* global data items */ -static int bChildDied; +static pthread_mutex_t mutChildDied; +static int bChildDied = 0; +static pthread_mutex_t mutHadHUP; static int bHadHUP; static int doFork = 1; /* fork - run in daemon mode - read-only after startup */ int bFinished = 0; /* used by termination signal handler, read-only except there @@ -1232,7 +1234,9 @@ hdlr_enable(int sig, void (*hdlr)()) static void hdlr_sighup(void) { + pthread_mutex_lock(&mutHadHUP); bHadHUP = 1; + pthread_mutex_unlock(&mutHadHUP); /* at least on FreeBSD we seem not to necessarily awake the main thread. * So let's do it explicitely. */ @@ -1243,7 +1247,9 @@ hdlr_sighup(void) static void hdlr_sigchld(void) { + pthread_mutex_lock(&mutChildDied); bChildDied = 1; + pthread_mutex_unlock(&mutChildDied); } static void @@ -1855,9 +1861,12 @@ wait_timeout(const sigset_t *sigmask) tvSelectTimeout.tv_sec = 0; tvSelectTimeout.tv_nsec = wait_period; do { + pthread_mutex_lock(&mutHadHUP); if(bFinished || bHadHUP) { + pthread_mutex_unlock(&mutHadHUP); break; } + pthread_mutex_unlock(&mutHadHUP); pselect(1, NULL, NULL, NULL, &tvSelectTimeout, sigmask); } while(--timeout > 0); } else { @@ -1945,6 +1954,7 @@ mainloop(void) time_t tTime; sigset_t origmask; sigset_t sigblockset; + int need_free_mutex; sigemptyset(&sigblockset); sigaddset(&sigblockset, SIGTERM); @@ -1953,14 +1963,28 @@ mainloop(void) do { pthread_sigmask(SIG_BLOCK, &sigblockset, &origmask); + pthread_mutex_lock(&mutChildDied); + need_free_mutex = 1; if(bChildDied) { - reapChild(); bChildDied = 0; + pthread_mutex_unlock(&mutChildDied); + need_free_mutex = 0; + reapChild(); + } + if(need_free_mutex) { + pthread_mutex_unlock(&mutChildDied); } + pthread_mutex_lock(&mutHadHUP); + need_free_mutex = 1; if(bHadHUP) { - doHUP(); bHadHUP = 0; + need_free_mutex = 0; + pthread_mutex_unlock(&mutHadHUP); + doHUP(); + } + if(need_free_mutex) { + pthread_mutex_unlock(&mutHadHUP); } processImInternal(); @@ -2095,6 +2119,8 @@ main(int argc, char **argv) * internally by rsyslogd. */ + pthread_mutex_init(&mutHadHUP, NULL); + pthread_mutex_init(&mutChildDied, NULL); strncpy(progname,argv[0], sizeof(progname)-1); addrsz = sizeof(srcaddr); if ((rc = getsockname(0, &srcaddr, &addrsz)) < 0) { @@ -2146,5 +2172,7 @@ main(int argc, char **argv) LogMsg(0, RS_RET_OK, LOG_DEBUG, "rsyslogd shutting down"); deinitAll(); osf_close(); + pthread_mutex_destroy(&mutChildDied); + pthread_mutex_destroy(&mutHadHUP); return 0; } From f5b7129652817b08d45c9e4af59cb189c97bbae8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 1 Apr 2022 13:00:11 +0200 Subject: [PATCH 056/134] maintain ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9d71ac53ec..f634b4314d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-04-01: add property options to support ISO week/year number + Thanks to Mattia Barbon for the patch. - 2022-04-01: core bugfix: "action suspended" message was emitted even when turned off Most messages were diasabled, but there was one part of the code that ignored the user configuration. From f2873d9cd6c9f3e74fd31656e8138a9a57b035e7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Apr 2022 09:18:08 +0200 Subject: [PATCH 057/134] maintain ChangeLog --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index f634b4314d..182431152a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-04-04: bugfix: cosmetic data races + there was a more or less cosmetic data race which could happen when children + processes died in quick sequence. Even then, no real harm happened, as all + children were reaped eventually. + A similar data race exists for HUP processing. + However, these races polluted TSAN test runs, and so we fixed them - 2022-04-01: add property options to support ISO week/year number Thanks to Mattia Barbon for the patch. - 2022-04-01: core bugfix: "action suspended" message was emitted even when turned off From 758a5e28f63231a244cd2f871c21c1423938aab8 Mon Sep 17 00:00:00 2001 From: Mikko Kortelainen Date: Mon, 19 Oct 2020 16:22:31 +0300 Subject: [PATCH 058/134] imfile: do_inotify() use only one blocking call do_inotify block only once for events with timeouts. There is possible scenario where no data is available and read() enters blocking while poll() is released. The change introduced here changes the racy reading to be same for the triggered file as for the rest of the files: current: - Inotify is triggered - Triggering file is read for new data - Rest of the files are read for timer expiration - more inotifys are processed these may be at the end of the file already as well then new: - Inotify is triggered - All files are read for timer expiration, which includes the triggering file - Triggering files is read for data, this may be at the end of the file already then - more inotifys are processed these may be at the end of the file already as well then Therefore the change introduced harmonises the way triggering and the future triggers are handled, by not making the triggering one as an exceptional one, so it is easier to test the changes as well. --- plugins/imfile/imfile.c | 106 +++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 45 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index b9aac7dcdd..78f1cba0f9 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -2444,10 +2444,11 @@ do_inotify(void) int rd; int currev; static int last_timeout = 0; + struct pollfd pollfd; DEFiRet; CHKiRet(wdmapInit()); - ino_fd = inotify_init(); + ino_fd = inotify_init1(IN_NONBLOCK); if(ino_fd < 0) { LogError(errno, RS_RET_INOTIFY_INIT_FAILED, "imfile: Init inotify " "instance failed "); @@ -2458,58 +2459,73 @@ do_inotify(void) do_initial_poll_run(); while(glbl.GetGlobalInputTermState() == 0) { - if(runModConf->haveReadTimeouts) { - int r; - struct pollfd pollfd; - pollfd.fd = ino_fd; - pollfd.events = POLLIN; - do { - r = poll(&pollfd, 1, runModConf->timeoutGranularity); - } while(r == -1 && errno == EINTR); - if(r == 0) { - DBGPRINTF("readTimeouts are configured, checking if some apply\n"); + int r; + + pollfd.fd = ino_fd; + pollfd.events = POLLIN; + + if (runModConf->haveReadTimeouts) + r = poll(&pollfd, 1, runModConf->timeoutGranularity); + else + r = poll(&pollfd, 1, -1); + + if (r == -1 && errno == EINTR) { + DBGPRINTF("do_inotify interrupted while polling on ino_fd\n"); + continue; + } + if(r == 0) { + DBGPRINTF("readTimeouts are configured, checking if some apply\n"); + if (runModConf->haveReadTimeouts) { fs_node_walk(runModConf->conf_tree, poll_timeouts); last_timeout = time(NULL); - continue; - } else if (r == -1) { - LogError(errno, RS_RET_INTERNAL_ERROR, + } + continue; + } else if (r == -1) { + LogError(errno, RS_RET_INTERNAL_ERROR, "%s:%d: unexpected error during poll timeout wait", __FILE__, __LINE__); - /* we do not abort, as this would render the whole input defunct */ - continue; - } else if(r != 1) { - LogError(errno, RS_RET_INTERNAL_ERROR, + /* we do not abort, as this would render the whole input defunct */ + continue; + } else if(r != 1) { + LogError(errno, RS_RET_INTERNAL_ERROR, "%s:%d: ERROR: poll returned more fds (%d) than given to it (1)", __FILE__, __LINE__, r); - /* we do not abort, as this would render the whole input defunct */ - continue; - } - } - rd = read(ino_fd, iobuf, sizeof(iobuf)); - if(rd == -1 && errno == EINTR) { - /* This might have been our termination signal! */ - DBGPRINTF("EINTR received during inotify, restarting poll\n"); + /* we do not abort, as this would render the whole input defunct */ continue; } - if(rd < 0) { - LogError(errno, RS_RET_IO_ERROR, "imfile: error during inotify - ignored"); - continue; - } - currev = 0; - while(currev < rd) { - union { - char *buf; - struct inotify_event *ev; - } savecast; - savecast.buf = iobuf+currev; - in_dbg_showEv(savecast.ev); - in_processEvent(savecast.ev); - currev += sizeof(struct inotify_event) + savecast.ev->len; - } - int now = time(NULL); - if(last_timeout + (runModConf->timeoutGranularity / 1000) > now) { - fs_node_walk(runModConf->conf_tree, poll_timeouts); - last_timeout = time(NULL); + else { + // process timeouts always, ino_fd may be too busy to ever have timeout occur from poll + if(runModConf->haveReadTimeouts) { + int now = time(NULL); + if(last_timeout + (runModConf->timeoutGranularity / 1000) > now) { + fs_node_walk(runModConf->conf_tree, poll_timeouts); + last_timeout = time(NULL); + } + } + rd = read(ino_fd, iobuf, sizeof(iobuf)); + if(rd == -1 && errno == EINTR) { + /* This might have been our termination signal! */ + DBGPRINTF("EINTR received during inotify, restarting poll\n"); + continue; + } + if (rd == -1 && errno == EWOULDBLOCK) { + continue; + } + if(rd < 0) { + LogError(errno, RS_RET_IO_ERROR, "imfile: error during inotify - ignored"); + continue; + } + currev = 0; + while(currev < rd) { + union { + char *buf; + struct inotify_event *ev; + } savecast; + savecast.buf = iobuf+currev; + in_dbg_showEv(savecast.ev); + in_processEvent(savecast.ev); + currev += sizeof(struct inotify_event) + savecast.ev->len; + } } } From f332bff979b67835922dcf18653acbe4364b17d0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Apr 2022 12:56:44 +0200 Subject: [PATCH 059/134] maintain ChangeLog --- ChangeLog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index 182431152a..d3097c6f10 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,12 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-04-04: imfile: potential processing delay + This was mentioned by Mikko Kortelainen without exact details on what exactly + this could cause in practice. But we were confident enough that it is worth + merging (though it does not look like something that brought real problems in + practice, as we do not know any related reports). + see also: https://github.com/rsyslog/rsyslog/pull/4445 + Thanks to Mikko Kortelainen for the patch. - 2022-04-04: bugfix: cosmetic data races there was a more or less cosmetic data race which could happen when children processes died in quick sequence. Even then, no real harm happened, as all From aefcfa4d0f6e213c9fac814c3e6bd53970b7e90e Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Wed, 9 Mar 2022 17:58:05 +0100 Subject: [PATCH 060/134] gnutls bugfix: Fix error handling in gtlsRecordRecv There was a rare possibility that the E_AGAIN/E_INTERRUPT handling could cause an infinite loop (100% CPU Usage), for example when a TLS handshake is interrupted at a certain stage. - After gnutls_record_recv is called, and E_AGAIN/E_INTERRUPT error occurs, we need to do additional read/write direction handling with gnutls_record_get_direction. - After the second call of gnutls_record_recv (Expand buffer) we needed to also check the eror codes for E_AGAIN/E_INTERRUPT to do propper errorhandling. - Add extra debug output based on ossl driver. - Potential fix for 100% CPU Loop Receiveloop after gtlsRecordRecv in doRetry call. see also: https://github.com/rsyslog/rsyslog/issues/4818 --- runtime/nsd_gtls.c | 40 ++++++++++++++++++++++++++++++---------- runtime/nsd_gtls.h | 5 +++++ runtime/nsdsel_gtls.c | 28 +++++++++++++++++++++------- runtime/tcpsrv.c | 11 +++++++---- tests/imtcp-tls-basic.sh | 3 +++ 5 files changed, 66 insertions(+), 21 deletions(-) diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index a9725a8e12..9e65d2943b 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -551,8 +551,9 @@ gtlsRecordRecv(nsd_gtls_t *pThis) DEFiRet; ISOBJ_TYPE_assert(pThis, nsd_gtls); - DBGPRINTF("gtlsRecordRecv: start\n"); - + DBGPRINTF("gtlsRecordRecv: start (Pending Data: %zd | Wanted Direction: %s)\n", + gnutls_record_check_pending(pThis->sess), + (gnutls_record_get_direction(pThis->sess) == gtlsDir_READ ? "READ" : "WRITE") ); lenRcvd = gnutls_record_recv(pThis->sess, pThis->pszRcvBuf, NSD_GTLS_MAX_RCVBUF); if(lenRcvd >= 0) { DBGPRINTF("gtlsRecordRecv: gnutls_record_recv received %zd bytes\n", lenRcvd); @@ -576,14 +577,30 @@ gtlsRecordRecv(nsd_gtls_t *pThis) (NSD_GTLS_MAX_RCVBUF+lenRcvd)); pThis->lenRcvBuf = NSD_GTLS_MAX_RCVBUF+lenRcvd; } else { - goto sslerr; + if (lenRcvd == GNUTLS_E_AGAIN || lenRcvd == GNUTLS_E_INTERRUPTED) { + goto sslerragain; /* Go to ERR AGAIN handling */ + } else { + /* Do all other error handling */ + int gnuRet = lenRcvd; + ABORTgnutls; + } } } } else if(lenRcvd == GNUTLS_E_AGAIN || lenRcvd == GNUTLS_E_INTERRUPTED) { -sslerr: - pThis->rtryCall = gtlsRtry_recv; - dbgprintf("GnuTLS receive requires a retry (this most probably is OK and no error condition)\n"); - ABORT_FINALIZE(RS_RET_RETRY); +sslerragain: + /* Check if the underlaying file descriptor needs to read or write data!*/ + if (gnutls_record_get_direction(pThis->sess) == gtlsDir_READ) { + pThis->rtryCall = gtlsRtry_recv; + dbgprintf("GnuTLS receive requires a retry, this most probably is OK and no error condition\n"); + ABORT_FINALIZE(RS_RET_RETRY); + } else { + uchar *pErr = gtlsStrerror(lenRcvd); + LogError(0, RS_RET_GNUTLS_ERR, "GnuTLS receive error %zd has wrong read direction(wants write) " + "- this could be caused by a broken connection. GnuTLS reports: %s\n", + lenRcvd, pErr); + free(pErr); + ABORT_FINALIZE(RS_RET_GNUTLS_ERR); + } } else { int gnuRet = lenRcvd; ABORTgnutls; @@ -2032,6 +2049,7 @@ static rsRetVal Send(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) { int iSent; + int wantsWriteData = 0; nsd_gtls_t *pThis = (nsd_gtls_t*) pNsd; DEFiRet; ISOBJ_TYPE_assert(pThis, nsd_gtls); @@ -2052,10 +2070,12 @@ Send(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) break; } if(iSent != GNUTLS_E_INTERRUPTED && iSent != GNUTLS_E_AGAIN) { + /* Check if the underlaying file descriptor needs to read or write data!*/ + wantsWriteData = gnutls_record_get_direction(pThis->sess); uchar *pErr = gtlsStrerror(iSent); - LogError(0, RS_RET_GNUTLS_ERR, "unexpected GnuTLS error %d - this " - "could be caused by a broken connection. GnuTLS reports: %s \n", - iSent, pErr); + LogError(0, RS_RET_GNUTLS_ERR, "unexpected GnuTLS error %d, wantsWriteData=%d - this " + "could be caused by a broken connection. GnuTLS reports: %s\n", + iSent, wantsWriteData, pErr); free(pErr); gnutls_perror(iSent); ABORT_FINALIZE(RS_RET_GNUTLS_ERR); diff --git a/runtime/nsd_gtls.h b/runtime/nsd_gtls.h index a3ef59f549..b9988aed50 100644 --- a/runtime/nsd_gtls.h +++ b/runtime/nsd_gtls.h @@ -33,6 +33,11 @@ typedef enum { gtlsRtry_recv = 2 } gtlsRtryCall_t; /**< IDs of calls that needs to be retried */ +typedef enum { + gtlsDir_READ = 0, /**< GNUTLS wants READ */ + gtlsDir_WRITE = 1 /**< GNUTLS wants WRITE */ +} gtlsDirection_t; + typedef nsd_if_t nsd_gtls_if_t; /* we just *implement* this interface */ /* the nsd_gtls object */ diff --git a/runtime/nsdsel_gtls.c b/runtime/nsdsel_gtls.c index 6ed7187e51..01cfb05177 100644 --- a/runtime/nsdsel_gtls.c +++ b/runtime/nsdsel_gtls.c @@ -81,6 +81,7 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) ISOBJ_TYPE_assert(pThis, nsdsel_gtls); ISOBJ_TYPE_assert(pNsdGTLS, nsd_gtls); + DBGPRINTF("Add on nsd %p:\n", pNsdGTLS); if(pNsdGTLS->iMode == 1) { if(waitOp == NSDSEL_RD && gtlsHasRcvInBuffer(pNsdGTLS)) { ++pThis->iBufferRcvReady; @@ -99,6 +100,7 @@ Add(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp) } } + dbgprintf("nsdsel_gtls: reached end on nsd %p, calling nsdsel_ptcp.Add with waitOp %d... \n", pNsdGTLS, waitOp); /* if we reach this point, we need no special handling */ CHKiRet(nsdsel_ptcp.Add(pThis->pTcp, pNsdGTLS->pTcp, waitOp)); @@ -120,7 +122,8 @@ Select(nsdsel_t *pNsdsel, int *piNumReady) if(pThis->iBufferRcvReady > 0) { /* we still have data ready! */ *piNumReady = pThis->iBufferRcvReady; - dbgprintf("nsdsel_gtls: doing dummy select, data present\n"); + dbgprintf("nsdsel_gtls: doing dummy select for %p->iBufferRcvReady=%d, data present\n", + pThis, pThis->iBufferRcvReady); } else { iRet = nsdsel_ptcp.Select(pThis->pTcp, piNumReady); } @@ -138,7 +141,7 @@ doRetry(nsd_gtls_t *pNsd) DEFiRet; int gnuRet; - dbgprintf("GnuTLS requested retry of %d operation - executing\n", pNsd->rtryCall); + dbgprintf("doRetry: GnuTLS requested retry of %d operation - executing\n", pNsd->rtryCall); /* We follow a common scheme here: first, we do the systen call and * then we check the result. So far, the result is checked after the @@ -151,7 +154,7 @@ doRetry(nsd_gtls_t *pNsd) case gtlsRtry_handshake: gnuRet = gnutls_handshake(pNsd->sess); if(gnuRet == GNUTLS_E_AGAIN || gnuRet == GNUTLS_E_INTERRUPTED) { - dbgprintf("GnuTLS handshake retry did not finish - " + dbgprintf("doRetry: GnuTLS handshake retry did not finish - " "setting to retry (this is OK and can happen)\n"); FINALIZE; } else if(gnuRet == 0) { @@ -167,9 +170,20 @@ doRetry(nsd_gtls_t *pNsd) } break; case gtlsRtry_recv: - dbgprintf("retrying gtls recv, nsd: %p\n", pNsd); - CHKiRet(gtlsRecordRecv(pNsd)); - pNsd->rtryCall = gtlsRtry_None; /* we are done */ + dbgprintf("doRetry: retrying gtls recv, nsd: %p\n", pNsd); + iRet = gtlsRecordRecv(pNsd); + if (iRet == RS_RET_RETRY) { + // Check if there is pending data + size_t stBytesLeft = gnutls_record_check_pending(pNsd->sess); + if (stBytesLeft > 0) { + // We are in retry and more data waiting, finalize it + goto finalize_it; + } else { + dbgprintf("doRetry: gtlsRecordRecv returned RETRY, but there is no pending" + "data on nsd: %p\n", pNsd); + } + } + pNsd->rtryCall = gtlsRtry_None; /* no more data, we are done */ gnuRet = 0; break; case gtlsRtry_None: @@ -241,7 +255,7 @@ IsReady(nsdsel_t *pNsdsel, nsd_t *pNsd, nsdsel_waitOp_t waitOp, int *pbIsReady) * socket. -- rgerhards, 2010-11-20 */ if(pThis->iBufferRcvReady) { - dbgprintf("nsd_gtls: dummy read, buffer not available for this FD\n"); + dbgprintf("nsd_gtls: dummy read, %p->buffer not available for this FD\n", pThis); *pbIsReady = 0; FINALIZE; } diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c index c8d4b7ee5c..b0310eedcb 100644 --- a/runtime/tcpsrv.c +++ b/runtime/tcpsrv.c @@ -596,14 +596,15 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) int oserr = 0; ISOBJ_TYPE_assert(pThis, tcpsrv); - DBGPRINTF("netstream %p with new data\n", (*ppSess)->pStrm); + prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + DBGPRINTF("netstream %p with new data from remote peer %s\n", (*ppSess)->pStrm, pszPeer); /* Receive message */ iRet = pThis->pRcvData(*ppSess, buf, sizeof(buf), &iRcvd, &oserr); switch(iRet) { case RS_RET_CLOSED: if(pThis->bEmitMsgOnClose) { errno = 0; - prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + // prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); LogError(0, RS_RET_PEER_CLOSED_CONN, "Netstream session %p closed by remote " "peer %s.\n", (*ppSess)->pStrm, pszPeer); } @@ -619,13 +620,13 @@ doReceive(tcpsrv_t *pThis, tcps_sess_t **ppSess, nspoll_t *pPoll) /* in this case, something went awfully wrong. * We are instructed to terminate the session. */ - prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + // prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); LogError(oserr, localRet, "Tearing down TCP Session from %s", pszPeer); CHKiRet(closeSess(pThis, ppSess, pPoll)); } break; default: - prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); + // prop.GetString((*ppSess)->fromHostIP, &pszPeer, &lenPeer); LogError(oserr, iRet, "netstream session %p from %s will be closed due to error", (*ppSess)->pStrm, pszPeer); CHKiRet(closeSess(pThis, ppSess, pPoll)); @@ -835,6 +836,8 @@ RunSelect(tcpsrv_t *pThis, nsd_epworkset_t workset[], size_t sizeWorkset) while(iTCPSess != -1) { /* TODO: access to pNsd is NOT really CLEAN, use method... */ CHKiRet(nssel.Add(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD)); + DBGPRINTF("tcpsrv process session %d:\n", iTCPSess); + /* now get next... */ iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); } diff --git a/tests/imtcp-tls-basic.sh b/tests/imtcp-tls-basic.sh index c36938fa23..b65e7c4113 100755 --- a/tests/imtcp-tls-basic.sh +++ b/tests/imtcp-tls-basic.sh @@ -5,6 +5,9 @@ export NUMMESSAGES=50000 export TB_TEST_MAX_RUNTIME=1500 export QUEUE_EMPTY_CHECK_FUNC=wait_seq_check +# uncomment for debugging support: +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debuglog" generate_conf add_conf ' global( defaultNetstreamDriverCAFile="'$srcdir'/tls-certs/ca.pem" From a5ba462da89a4fbd7a9d503dbb8994b8d91a38c9 Mon Sep 17 00:00:00 2001 From: Art O Cathain Date: Tue, 5 Apr 2022 14:05:47 +0100 Subject: [PATCH 061/134] Fixed a broken link in the PR failure message --- tests/CI/PR_validation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CI/PR_validation.sh b/tests/CI/PR_validation.sh index bc80845ab4..f2b5b859d8 100755 --- a/tests/CI/PR_validation.sh +++ b/tests/CI/PR_validation.sh @@ -25,7 +25,7 @@ if ! tests/CI/check_commit_text.py gitlog; then EOF cat <<- EOF For more info, please see - https://rainer.gerhards.net/posts#descriptive_commit_message + https://rainer.gerhards.net/2019/03/howto-great-pull-request.html#descriptive_commit_message EOF exitcode=1 From 93eab591054b218e6b8f9aea91ee19f54c5557cb Mon Sep 17 00:00:00 2001 From: Art O Cathain Date: Tue, 5 Apr 2022 14:18:32 +0100 Subject: [PATCH 062/134] Include programname in a new JSON template Without the program name, the logs intermingle output from multiple programs. StdJSONFmt is left as-is for backwards-compatibility. --- runtime/rsconf.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runtime/rsconf.c b/runtime/rsconf.c index f9e2e02d63..ea9d536872 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -115,6 +115,15 @@ static uchar template_StdJSONFmt[] = "\"{\\\"message\\\":\\\"%msg:::json%\\\",\\ "%HOSTNAME:::json%\\\",\\\"facility\\\":\\\"%syslogfacility-text%\\\",\\\"priority\\\":\\\"" "%syslogpriority-text%\\\",\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\",\\\"timegenerated\\\":\\\"" "%timegenerated:::date-rfc3339%\\\"}\""; +static uchar template_FullJSONFmt[] = "\"{\\\"message\\\":\\\"%msg:::json%\\\"," +"\\\"fromhost\\\":\\\"%HOSTNAME:::json%\\\"," +"\\\"programname\\\":\\\"%programname%\\\"," +"\\\"procid\\\":\\\"%PROCID%\\\"," +"\\\"msgid\\\":\\\"%MSGID%\\\"," +"\\\"facility\\\":\\\"%syslogfacility-text%\\\"," +"\\\"priority\\\":\\\"%syslogpriority-text%\\\"," +"\\\"timereported\\\":\\\"%timereported:::date-rfc3339%\\\"," +"\\\"timegenerated\\\":\\\"%timegenerated:::date-rfc3339%\\\"}\""; static uchar template_StdClickHouseFmt[] = "\"INSERT INTO rsyslog.SystemEvents (severity, facility, " "timestamp, hostname, tag, message) VALUES (%syslogseverity%, %syslogfacility%, " "'%timereported:::date-unixtimestamp%', '%hostname%', '%syslogtag%', '%msg%')\",STDSQL"; @@ -1405,6 +1414,8 @@ initLegacyConf(void) tplAddLine(ourConf, " StdPgSQLFmt", &pTmp); pTmp = template_StdJSONFmt; tplAddLine(ourConf, " StdJSONFmt", &pTmp); + pTmp = template_FullJSONFmt; + tplAddLine(ourConf, " FullJSONFmt", &pTmp); pTmp = template_StdClickHouseFmt; tplAddLine(ourConf, " StdClickHouseFmt", &pTmp); pTmp = template_spoofadr; From a854538eb66a47cf9b23d4a16e2ad6889d9e3ccc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 6 Apr 2022 09:19:33 +0200 Subject: [PATCH 063/134] maintain ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index d3097c6f10..c3bf4d9249 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +- 2022-04-06: added new "FullJSONFmt" standard template (with addtl fields) + This comes handy for a number of use cases, especially with ElasticSearch. + Thanks to Art O Cathain for the patch. - 2022-04-04: imfile: potential processing delay This was mentioned by Mikko Kortelainen without exact details on what exactly this could cause in practice. But we were confident enough that it is worth From 2c8c9db065cef3b5086c90c3782ac48da40c8b2f Mon Sep 17 00:00:00 2001 From: David Buckley Date: Tue, 12 Apr 2022 17:38:49 +0100 Subject: [PATCH 064/134] Fix non-null-terminated-string used with strlen The `failedmsg_entry` expects a null-terminated string in `key`, but here we allocate with malloc and copy a string-with-length-n into only the first n bytes. If the final byte is null, this is by coincidence only. We've observed this by means of seeing random binary data appended to keys submitted to kafka apparently at random, and this looks like a smoking gun. --- plugins/omkafka/omkafka.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omkafka/omkafka.c b/plugins/omkafka/omkafka.c index 44bffdd4df..e8eae08ddb 100644 --- a/plugins/omkafka/omkafka.c +++ b/plugins/omkafka/omkafka.c @@ -350,6 +350,7 @@ const size_t msglen, const char *const topicname) return NULL; } memcpy(etry->key, key, keylen); + etry->key[keylen] = '\0'; } else { etry->key=NULL; } From b427de5109c21affedcded15221bdc63a662987a Mon Sep 17 00:00:00 2001 From: frikilax Date: Thu, 14 Apr 2022 17:27:02 +0200 Subject: [PATCH 065/134] MMDBLOOKUP::FIXED:: Fix parsing of data received from mmdb lib --- plugins/mmdblookup/mmdblookup.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/mmdblookup/mmdblookup.c b/plugins/mmdblookup/mmdblookup.c index 3fe5172a95..affbc73578 100644 --- a/plugins/mmdblookup/mmdblookup.c +++ b/plugins/mmdblookup/mmdblookup.c @@ -324,16 +324,25 @@ str_split(char **membuf) memset(tempbuf, 0, strlen(buf)); while (*buf++ != '\0') { - if (*buf == '\n' || *buf == '\t' || (*buf == ' ' && !in_quotes)) - continue; - else { + if (in_quotes) { + if (*buf == '"' && *(buf - 1) != '\\') { + in_quotes = !in_quotes; + strncat(tempbuf, buf, 1); + } else { + strncat(tempbuf, buf, 1); + } + } else { + if (*buf == '\n' || *buf == '\t' || *buf == ' ') + continue; if (*buf == '<') { char *p = strchr(buf, '>'); buf = buf + (int)(p - buf); strcat(tempbuf, ","); } else if (*buf == '}') { strcat(tempbuf, "},"); - } else if (*buf == '"') { + } else if (*buf == ']') { + strcat(tempbuf, "],"); + } else if (*buf == '"' && *(buf - 1) != '\\') { in_quotes = !in_quotes; strncat(tempbuf, buf, 1); } else { @@ -342,8 +351,7 @@ str_split(char **membuf) } } - tempbuf[strlen(tempbuf) + 1] = '\n'; - memcpy(*membuf, tempbuf, strlen(tempbuf)); + memcpy(*membuf, tempbuf, strlen(tempbuf)+1); } From 2bc7ece976d3f59b0d70a839f3fdc2c4b8f882d9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 Apr 2022 11:11:57 +0200 Subject: [PATCH 066/134] maintain ChangeLog --- ChangeLog | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index c3bf4d9249..7b569782cb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,38 @@ ---------------------------------------------------------------------------------------- -Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-?? +Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-19 +- 2022-04-18: gnutls bugfix: possibility of infinite loop + There was a rare possibility that the E_AGAIN/E_INTERRUPT handling + could cause an infinite loop (100% CPU Usage), for example when a TLS + handshake is interrupted at a certain stage. + * After gnutls_record_recv is called, and E_AGAIN/E_INTERRUPT error + occurs, we need to do additional read/write direction handling + with gnutls_record_get_direction. + * After the second call of gnutls_record_recv (Expand buffer) + we needed to also check the eror codes for E_AGAIN/E_INTERRUPT + to do propper errorhandling. + * Add extra debug output based on ossl driver. + * Potential fix for 100% CPU Loop Receiveloop after gtlsRecordRecv + in doRetry call. + closes https://github.com/rsyslog/rsyslog/issues/4834 + closes https://github.com/rsyslog/rsyslog/issues/4818 + closes https://github.com/rsyslog/rsyslog/issues/4638 +- 2022-04-17: core/bugfix: errorfile could grow over max configures size + When action.errorfile.maxsize configuration option is enabled and error file + already has a certain size smaller than max size configured, it is increasing + higher than configured max size as the error file is considered to be zero in code. + This fix reads current error file size and limits the size to the maximum + size configured. + Thanks to Sergio Arroutbi for the patch. + fixes https://github.com/rsyslog/rsyslog/issues/4821 +- 2022-04-17: omkafka bugfix: potential misadressing + The `failedmsg_entry` expects a null-terminated string in `key`, but + here we allocate with malloc and copy a string-with-length-n into only + the first n bytes. If the final byte is null, this is by coincidence + only. + This was observed by means of seeing random binary data appended to + keys submitted to kafka apparently at random. This could also result + in more severe problems, inclusing a segfault. + Thanks to David Buckley for the patch. - 2022-04-06: added new "FullJSONFmt" standard template (with addtl fields) This comes handy for a number of use cases, especially with ElasticSearch. Thanks to Art O Cathain for the patch. From d886418e4dfc54b7d15587f1a21c0d24d13b533c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 Apr 2022 11:18:13 +0200 Subject: [PATCH 067/134] prepare for v8.2204.0 scheduled stable release --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 42baee2413..b4010c6407 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2204.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2204.0],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 04, [month part of real rsyslog version]) # UPDATE on release From 30cbfd2cc7d6c22f442f5ece7ebb00244b080ac0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Apr 2022 08:28:23 +0200 Subject: [PATCH 068/134] bump version number for next dev cycle also fix minor copy&paste date error in ChangeLog --- ChangeLog | 6 ++++-- configure.ac | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7b569782cb..ae48b8df97 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ ---------------------------------------------------------------------------------------- -Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-19 +Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +---------------------------------------------------------------------------------------- +Scheduled Release 8.2204.0 (aka 2022.04) 2022-04-19 - 2022-04-18: gnutls bugfix: possibility of infinite loop There was a rare possibility that the E_AGAIN/E_INTERRUPT handling could cause an infinite loop (100% CPU Usage), for example when a TLS @@ -83,7 +85,7 @@ Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-19 that most items in rsyslog are strings, so this can actually cause some problems. ---------------------------------------------------------------------------------------- -Scheduled Release 8.2202.0 (aka 2022.02) 2021-02-15 +Scheduled Release 8.2202.0 (aka 2022.02) 2022-02-15 - 2022-02-14: imfile bugfix: remove cause for "internal error message" (not causing harm) When any message is output into a renamed input file, rsyslogd output the following: message. diff --git a/configure.ac b/configure.ac index b4010c6407..a78f3093c9 100644 --- a/configure.ac +++ b/configure.ac @@ -2,9 +2,9 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2204.0],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2206.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release -AC_DEFINE(VERSION_MONTH, 04, [month part of real rsyslog version]) # UPDATE on release +AC_DEFINE(VERSION_MONTH, 06, [month part of real rsyslog version]) # UPDATE on release AM_INIT_AUTOMAKE([subdir-objects]) From 41df565b14f122a56e3433fc2adfe91a7b52b9ef Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Apr 2022 10:44:10 +0200 Subject: [PATCH 069/134] maintain ChangeLog --- ChangeLog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index ae48b8df97..0c5323c943 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,15 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-04-22: mmdblookup fix: wrong copy of buffer + ...following parse of libmaxminddb's return after a successful search sometimes + failed to return specific field from data. + Thanks to Théo Bertin for the patch. +- 2022-04-22: mmdblookup: several enhancements + - support arrays in MMDB entry + - support escaped quotes '"' in MMDB entry + - support '<' characters in MMDB entry, when in a field + - support '}' characters in MMDB entry, when in a field + Thanks to Théo Bertin for the patch. ---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2022-04-19 - 2022-04-18: gnutls bugfix: possibility of infinite loop From aaf3ffd36444c26be96c73946d06f8b2b3c55dec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Apr 2022 10:45:20 +0200 Subject: [PATCH 070/134] cleanup: remove file accidentally added to git --- tests/testbench_test_failed_rsyslog | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/testbench_test_failed_rsyslog diff --git a/tests/testbench_test_failed_rsyslog b/tests/testbench_test_failed_rsyslog deleted file mode 100644 index 9565b809a4..0000000000 --- a/tests/testbench_test_failed_rsyslog +++ /dev/null @@ -1 +0,0 @@ -./omfwd-errfile-maxsize-filled.sh From d59de97be98ab70ee38d9efa11d04d8015e23df6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Apr 2022 09:49:46 +0200 Subject: [PATCH 071/134] net bugfix: potential buffer overrun --- contrib/imhttp/imhttp.c | 4 +++- plugins/imptcp/imptcp.c | 4 +++- runtime/tcps_sess.c | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/imhttp/imhttp.c b/contrib/imhttp/imhttp.c index f09260b586..95704af985 100644 --- a/contrib/imhttp/imhttp.c +++ b/contrib/imhttp/imhttp.c @@ -487,7 +487,9 @@ processOctetMsgLen(const instanceConf_t *const inst, struct conn_wrkr_s *connWrk connWrkr->parseState.iOctetsRemain = connWrkr->parseState.iOctetsRemain * 10 + ch - '0'; } // temporarily save this character into the message buffer - connWrkr->pMsg[connWrkr->iMsg++] = ch; + if(connWrkr->iMsg + 1 < s_iMaxLine) { + connWrkr->pMsg[connWrkr->iMsg++] = ch; + } } else { const char *remoteAddr = ""; if (connWrkr->propRemoteAddr) { diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 2df46a236c..c32dec5851 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1107,7 +1107,9 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, if(pThis->iOctetsRemain <= 200000000) { pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; } - *(pThis->pMsg + pThis->iMsg++) = c; + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } } else { /* done with the octet count, so this must be the SP terminator */ DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); prop.GetString(pThis->peerName, &propPeerName, &lenPeerName); diff --git a/runtime/tcps_sess.c b/runtime/tcps_sess.c index 0efa2c23c4..c5442f7638 100644 --- a/runtime/tcps_sess.c +++ b/runtime/tcps_sess.c @@ -390,7 +390,9 @@ processDataRcvd(tcps_sess_t *pThis, if(pThis->iOctetsRemain <= 200000000) { pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; } - *(pThis->pMsg + pThis->iMsg++) = c; + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } } else { /* done with the octet count, so this must be the SP terminator */ DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); prop.GetString(pThis->fromHost, &propPeerName, &lenPeerName); From 30ccf7cd4c00bfc38c2e0b1461b799b1a412d7fb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 Apr 2022 10:18:46 +0200 Subject: [PATCH 072/134] testbench: new tests for potential buffer overrun --- tests/Makefile.am | 4 ++++ tests/imptcp-octet-framing-too-long-vg.sh | 23 +++++++++++++++++++++++ tests/imtcp-octet-framing-too-long-vg.sh | 23 +++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100755 tests/imptcp-octet-framing-too-long-vg.sh create mode 100755 tests/imtcp-octet-framing-too-long-vg.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 330546890e..805949ec8b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -283,6 +283,7 @@ TESTS += \ allowed-sender-tcp-fail.sh \ allowed-sender-tcp-hostname-ok.sh \ allowed-sender-tcp-hostname-fail.sh \ + imtcp-octet-framing-too-long-vg.sh \ imtcp-discard-truncated-msg.sh \ imtcp-basic.sh \ imtcp-basic-hup.sh \ @@ -1074,6 +1075,7 @@ if ENABLE_IMPTCP # need to be disabled if we do not have this module TESTS += \ manyptcp.sh \ + imptcp-octet-framing-too-long-vg.sh \ imptcp_framing_regex.sh \ imptcp_framing_regex-oversize.sh \ imptcp_large.sh \ @@ -2121,6 +2123,7 @@ EXTRA_DIST= \ mmjsonparse_simple.sh \ mmjsonparse-invalid-containerName.sh \ wtpShutdownAll-assertionFailure.sh \ + imptcp-octet-framing-too-long-vg.sh \ imptcp-oversize-message-display.sh \ imptcp-msg-truncation-on-number.sh \ imptcp-msg-truncation-on-number2.sh \ @@ -2199,6 +2202,7 @@ EXTRA_DIST= \ allowed-sender-tcp-fail.sh \ allowed-sender-tcp-hostname-ok.sh \ allowed-sender-tcp-hostname-fail.sh \ + imtcp-octet-framing-too-long-vg.sh \ imtcp-discard-truncated-msg.sh \ imtcp-basic.sh \ imtcp-basic-hup.sh \ diff --git a/tests/imptcp-octet-framing-too-long-vg.sh b/tests/imptcp-octet-framing-too-long-vg.sh new file mode 100755 index 0000000000..d5b2c9adce --- /dev/null +++ b/tests/imptcp-octet-framing-too-long-vg.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# added 2022-04-25 by RGerhards, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +$MaxMessageSize 128 +global(processInternalMessages="on" + oversizemsg.input.mode="accept") +module(load="../plugins/imptcp/.libs/imptcp") +input(type="imptcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port") + +action(type="omfile" file="'$RSYSLOG_OUT_LOG'") +' +startup_vg +echo "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000 <120> 2011-03-01T11:22:12Z host tag: this is a way too long message that has to be truncatedtest1 test2 test3 test4 test5 ab" > $RSYSLOG_DYNNAME.inputfile +tcpflood -I $RSYSLOG_DYNNAME.inputfile +shutdown_when_empty +wait_shutdown_vg +check_exit_vg + +# the prime objective is to see if valgrind check is ok, but we also do a quick content check (just in case) +content_check "received oversize message from peer" +exit_test diff --git a/tests/imtcp-octet-framing-too-long-vg.sh b/tests/imtcp-octet-framing-too-long-vg.sh new file mode 100755 index 0000000000..88e8a14fb9 --- /dev/null +++ b/tests/imtcp-octet-framing-too-long-vg.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# added 2022-04-25 by RGerhards, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +generate_conf +add_conf ' +$MaxMessageSize 128 +global(processInternalMessages="on" + oversizemsg.input.mode="accept") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port") + +action(type="omfile" file="'$RSYSLOG_OUT_LOG'") +' +startup_vg +echo "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000 <120> 2011-03-01T11:22:12Z host tag: this is a way too long message that has to be truncatedtest1 test2 test3 test4 test5 ab" > $RSYSLOG_DYNNAME.inputfile +tcpflood -I $RSYSLOG_DYNNAME.inputfile +shutdown_when_empty +wait_shutdown_vg +check_exit_vg + +# the prime objective is to see if valgrind check is ok, but we also do a quick content check (just in case) +content_check "received oversize message from peer" +exit_test From 03ba122e9006d5f56535a3d648dd9e216a7a3596 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 Apr 2022 14:08:02 +0200 Subject: [PATCH 073/134] mmanon bugfix: shortened IPv6 form not always anonymized If the IPv6 is in non-recommended form followed by a 5 digit port number, it is not anonymized. A reproducer for this is: 1a00:c820:1180:c84c::ad3f:d991:ec2e:49255 closes https://github.com/rsyslog/rsyslog/issues/4856 --- plugins/mmanon/mmanon.c | 17 ++++++++++++++--- tests/mmanon_ipv6_port.sh | 19 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/plugins/mmanon/mmanon.c b/plugins/mmanon/mmanon.c index a177075956..607787ca02 100644 --- a/plugins/mmanon/mmanon.c +++ b/plugins/mmanon/mmanon.c @@ -495,11 +495,10 @@ isValidHexNum(const uchar *const __restrict__ buf, case 'E': case 'F': cyc++; + (*nprocessed)++; if(cyc == 5) { - cyc = 0; goto done; } - (*nprocessed)++; break; case '.': if(handleDot && cyc == 0) { @@ -537,7 +536,7 @@ syntax_ipv6(const uchar *const __restrict__ buf, while(*nprocessed < buflen) { numLen = isValidHexNum(buf + *nprocessed, buflen - *nprocessed, nprocessed, 0); - if(numLen > 0) { //found a valid num + if(numLen > 0 && numLen < 5) { //found a valid num if((ipParts == 7 && hadAbbrev) || ipParts > 7) { isIP = 0; goto done; @@ -560,6 +559,18 @@ syntax_ipv6(const uchar *const __restrict__ buf, } } lastSep = 1; + } else if (numLen == 5) { // maybe truncated with port + if(hadAbbrev && ipParts >= 2) { + isIP = 1; + /* we need to go back 6 chars: + * 5 digits plus leading ":" which designates port! + */ + *nprocessed -= 6; + } else { + isIP = 0; + /* nprocessed need not be corrected - it's only used if isIP == 1 */ + } + goto done; } else { //no valid num if(lastSep) { if(lastAbbrev && ipParts < 8) { diff --git a/tests/mmanon_ipv6_port.sh b/tests/mmanon_ipv6_port.sh index 3de275dab5..8ebbd6cc48 100755 --- a/tests/mmanon_ipv6_port.sh +++ b/tests/mmanon_ipv6_port.sh @@ -19,13 +19,28 @@ startup tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag: asdfghjk <129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:c820:1180:c84c:ad3f:4024:d991:ec2e:4922 <129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:c820:1180:c84c:ad3f:4024:d991:ec2e -<129>Mar 10 01:00:00 172.20.245.8 tag: [1a00:c820:1180:c84c:ad3f:4024:d991:ec2e]:4922\"" +<129>Mar 10 01:00:00 172.20.245.8 tag: [1a00:c820:1180:c84c:ad3f:4024:d991:ec2e]:4922 +<129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:c820:1180:c84c:ad3f::d991:ec2e:4922 +<129>Mar 10 01:00:00 172.20.245.8 tag: [1a00:c820:1180:c84c:ad3f::d991:ec2e]:4922 +<129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:c820:1180:c84c:ad3f::d991:ec2e:49225 +<129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:4922:4922:c84c:ad3f::d991:ec2e:49225 +<129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:4922:1180:c84c:ad3f::d991:4922:49225 +<129>Mar 10 01:00:00 172.20.245.8 tag: 1a00:c820:49225:c84c:ad3f::d991:ec2e:49225\"" + +# see this github comment on limits of IP address detection: +# https://github.com/rsyslog/rsyslog/issues/4856#issuecomment-1108445473 shutdown_when_empty wait_shutdown export EXPECTED=' asdfghjk 1a00:c820:0:0:0:0:0:0:4922 1a00:c820:0:0:0:0:0:0 - [1a00:c820:0:0:0:0:0:0]:4922' + [1a00:c820:0:0:0:0:0:0]:4922 + 1a00:c820:1180:0:0:0:0:0:0 + [1a00:c820:0:0:0:0:0:0]:4922 + 1a00:c820:0:0:0:0:0:0:49225 + 1a00:4922:0:0:0:0:0:0:49225 + 1a00:4922:0:0:0:0:0:0:49225 + 1a00:c820:49225:c84c:0:0:0:0:0:0:49225' cmp_exact exit_test From e0df4925ad27a741d76ef4481885b0b8282019fe Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 26 Apr 2022 15:46:36 +0200 Subject: [PATCH 074/134] maintain ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0c5323c943..2c10c17141 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-04-26: mmanon bugfix: shortened IPv6 form not always anonymized + If the IPv6 is in non-recommended form followed by a 5 digit port number, it + is not anonymized. + A reproducer for this is: 1a00:c820:1180:c84c::ad3f:d991:ec2e:49255 + closes https://github.com/rsyslog/rsyslog/issues/4856 - 2022-04-22: mmdblookup fix: wrong copy of buffer ...following parse of libmaxminddb's return after a successful search sometimes failed to return specific field from data. From 6ffc14fd450b90872272871130d29ab3ecf85f6f Mon Sep 17 00:00:00 2001 From: Iwan Timmer Date: Fri, 29 Apr 2022 15:14:27 +0200 Subject: [PATCH 075/134] tcpsrv: do not decrease number of to be processed fds on error nfds should only be decreased for processed streams and not for streams returning an error code, like RS_RET_RETRY. --- runtime/tcpsrv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c index b0310eedcb..16668b7650 100644 --- a/runtime/tcpsrv.c +++ b/runtime/tcpsrv.c @@ -880,7 +880,8 @@ RunSelect(tcpsrv_t *pThis, nsd_epworkset_t workset[], size_t sizeWorkset) processWorkset(pThis, NULL, iWorkset, workset); iWorkset = 0; } - --nfds; /* indicate we have processed one */ + if(bIsReady) + --nfds; /* indicate we have processed one */ } iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); } From c49180abdd48718e26fa52ebb058b55eabab9ef6 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Mon, 2 May 2022 13:43:06 +0200 Subject: [PATCH 076/134] testbench: removed unused notworking tls tests Testcase sndrcv_tls_certless_ossl_client.sh was not finished and was not used by testbench. Newer tests replaced its test usage. See testcases sndrcv_tls_gtls_serveranon*.sh and sndrcv_tls_ossl_serveranon*.sh closes: https://github.com/rsyslog/rsyslog/issues/4853 --- tests/sndrcv_tls_certless_ossl_client.sh | 67 ------------------------ 1 file changed, 67 deletions(-) delete mode 100755 tests/sndrcv_tls_certless_ossl_client.sh diff --git a/tests/sndrcv_tls_certless_ossl_client.sh b/tests/sndrcv_tls_certless_ossl_client.sh deleted file mode 100755 index acafe75637..0000000000 --- a/tests/sndrcv_tls_certless_ossl_client.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# alorbach, 2019-01-16 -# This file is part of the rsyslog project, released under ASL 2.0 -. ${srcdir:=.}/diag.sh init -export NUMMESSAGES=1000 -# uncomment for debugging support: -#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" -export RSYSLOG_DEBUGLOG="log" -generate_conf -export PORT_RCVR="$(get_free_port)" -add_conf ' -global( - defaultNetstreamDriverCAFile="'$srcdir/tls-certs/ca.pem'" -) - -module( load="../plugins/imtcp/.libs/imtcp" - StreamDriver.Name="gtls" - StreamDriver.Mode="1" - StreamDriver.AuthMode="x509/certvalid" ) -# then SENDER sends to this port (not tcpflood!) -input( type="imtcp" port="'$PORT_RCVR'" ) - -$template outfmt,"%msg:F,58:2%\n" -$template dynfile,"'$RSYSLOG_OUT_LOG'" # trick to use relative path names! -:msg, contains, "msgnum:" ?dynfile;outfmt -' -startup -export RSYSLOG_DEBUGLOG="log2" -#valgrind="valgrind" -generate_conf 2 -export TCPFLOOD_PORT="$(get_free_port)" -add_conf ' -global( - defaultNetstreamDriverCAFile="'$srcdir/tls-certs/ca.pem'" - defaultNetstreamDriverCertFile="'$srcdir/tls-certs/cert.pem'" - defaultNetstreamDriverKeyFile="'$srcdir/tls-certs/key.pem'" -) - -# Note: no TLS for the listener, this is for tcpflood! -$ModLoad ../plugins/imtcp/.libs/imtcp -input( type="imtcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port" ) - -# set up the action -action( type="omfwd" - protocol="tcp" - target="127.0.0.1" - port="'$PORT_RCVR'" - StreamDriver="ossl" - StreamDriverMode="1" - StreamDriverAuthMode="anon" - ) -' 2 -startup 2 - -# now inject the messages into instance 2. It will connect to instance 1, -# and that instance will record the data. -tcpflood -m$NUMMESSAGES -i1 -wait_file_lines -# shut down sender when everything is sent, receiver continues to run concurrently -shutdown_when_empty 2 -wait_shutdown 2 -# now it is time to stop the receiver as well -shutdown_when_empty -wait_shutdown - -seq_check 1 $NUMMESSAGES -exit_test From 0f437fba5d2c183126f0b3143f0167c0d2a59aea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 3 May 2022 13:04:43 +0200 Subject: [PATCH 077/134] imptcp: set OS worker thread name We now set the worker thread names to "imptcp/" where is the numerical index (0, 1, ...) of the worker thread. This enables to distinguish individual worker threads in OS tools like htop. That is useful for performance testing and system monitoring. The choosen name format is consistant with other similar thread names inside rsyslog. For imptcp, worker threads were not yet given individual names. Note: "in:imptcp" is imptcp's "main" thread, which also is used as a worker in some scenarios. This name was not modified. --- plugins/imptcp/imptcp.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 2df46a236c..2ec504b525 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -59,6 +59,9 @@ #if HAVE_FCNTL_H #include #endif +#ifdef HAVE_SYS_PRCTL_H +# include +#endif #include "rsyslog.h" #include "cfsysline.h" #include "prop.h" @@ -346,6 +349,7 @@ struct ptcplstn_s { static struct wrkrInfo_s { pthread_t tid; /* the worker's thread ID */ long long unsigned numCalled; /* how often was this called */ + int wrkrIdx; /* index for this worker - shortcut for thread name */ } *wrkrInfo; static int wrkrRunning; @@ -1833,6 +1837,7 @@ startWorkerPool(void) } for(i = 0 ; i < runModConf->wrkrMax ; ++i) { /* init worker info structure! */ + wrkrInfo[i].wrkrIdx = i; wrkrInfo[i].numCalled = 0; pthread_create(&wrkrInfo[i].tid, &wrkrThrdAttr, wrkr, &(wrkrInfo[i])); } @@ -2121,6 +2126,15 @@ wrkr(void *myself) ++wrkrRunning; pthread_mutex_unlock(&io_q.mut); + uchar thrdName[32]; + snprintf((char*)thrdName, sizeof(thrdName), "imptcp/w%d", me->wrkrIdx); +# if defined(HAVE_PRCTL) && defined(PR_SET_NAME) + /* set thread name - we ignore if the call fails, has no harsh consequences... */ + if(prctl(PR_SET_NAME, thrdName, 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", thrdName); + } +# endif + io_req_t *n; while(1) { n = NULL; From 89955b0bcb1ff105e1374aad7e0e993faa6a038f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Apr 2022 09:49:46 +0200 Subject: [PATCH 078/134] net bugfix: potential buffer overrun --- contrib/imhttp/imhttp.c | 4 +++- plugins/imptcp/imptcp.c | 4 +++- runtime/tcps_sess.c | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/imhttp/imhttp.c b/contrib/imhttp/imhttp.c index f09260b586..95704af985 100644 --- a/contrib/imhttp/imhttp.c +++ b/contrib/imhttp/imhttp.c @@ -487,7 +487,9 @@ processOctetMsgLen(const instanceConf_t *const inst, struct conn_wrkr_s *connWrk connWrkr->parseState.iOctetsRemain = connWrkr->parseState.iOctetsRemain * 10 + ch - '0'; } // temporarily save this character into the message buffer - connWrkr->pMsg[connWrkr->iMsg++] = ch; + if(connWrkr->iMsg + 1 < s_iMaxLine) { + connWrkr->pMsg[connWrkr->iMsg++] = ch; + } } else { const char *remoteAddr = ""; if (connWrkr->propRemoteAddr) { diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 2df46a236c..c32dec5851 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -1107,7 +1107,9 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, if(pThis->iOctetsRemain <= 200000000) { pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; } - *(pThis->pMsg + pThis->iMsg++) = c; + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } } else { /* done with the octet count, so this must be the SP terminator */ DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); prop.GetString(pThis->peerName, &propPeerName, &lenPeerName); diff --git a/runtime/tcps_sess.c b/runtime/tcps_sess.c index 0efa2c23c4..c5442f7638 100644 --- a/runtime/tcps_sess.c +++ b/runtime/tcps_sess.c @@ -390,7 +390,9 @@ processDataRcvd(tcps_sess_t *pThis, if(pThis->iOctetsRemain <= 200000000) { pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; } - *(pThis->pMsg + pThis->iMsg++) = c; + if(pThis->iMsg < iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } } else { /* done with the octet count, so this must be the SP terminator */ DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); prop.GetString(pThis->fromHost, &propPeerName, &lenPeerName); From 75a8aafbb0a8ef2c8f292dc3611288546bda0d5b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 5 May 2022 10:04:39 +0200 Subject: [PATCH 079/134] prepare for interim scheduled stable release 8.2204.1 --- ChangeLog | 5 +++++ configure.ac | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 7b569782cb..6dae57dfc6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,9 @@ ---------------------------------------------------------------------------------------- +Scheduled Release 8.2204.1 (aka 2022.04) 2021-05-05 +- security bugfix: potential buffer overrun in imptcp, imtcp, imgssapi and others + This addresses CVE-2022-24903 + see also https://github.com/rsyslog/rsyslog/security/advisories/GHSA-ggw7-xr6h-mmr8 +---------------------------------------------------------------------------------------- Scheduled Release 8.2204.0 (aka 2022.04) 2021-04-19 - 2022-04-18: gnutls bugfix: possibility of infinite loop There was a rare possibility that the E_AGAIN/E_INTERRUPT handling diff --git a/configure.ac b/configure.ac index b4010c6407..247a155a15 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2204.0],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2204.1],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 04, [month part of real rsyslog version]) # UPDATE on release From b681478c343b53b72646a851788d04c9e946dc0f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 5 May 2022 14:01:00 +0200 Subject: [PATCH 080/134] maintain ChangeLog --- ChangeLog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2c10c17141..a805e733c0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,15 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-05: imptcp: set OS worker thread name + We now set the worker thread names to "imptcp/" where + is the numerical index (0, 1, ...) of the worker thread. + This enables to distinguish individual worker threads in OS tools like + htop. That is useful for performance testing and system monitoring. + The choosen name format is consistant with other similar thread + names inside rsyslog. For imptcp, worker threads were not yet + given individual names. + Note: "in:imptcp" is imptcp's "main" thread, which also is used + as a worker in some scenarios. This name was not modified. - 2022-04-26: mmanon bugfix: shortened IPv6 form not always anonymized If the IPv6 is in non-recommended form followed by a 5 digit port number, it is not anonymized. From b5eed38ebc1c7ac844c8f942b8c246d77633879e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 May 2022 09:11:02 +0200 Subject: [PATCH 081/134] maintain ChangeLog --- ChangeLog | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ChangeLog b/ChangeLog index 6a33cc6144..3448cc874b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,30 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-09: tcp receiver bugfix: delay/potential hang on some error conditions + Error were not correctly handled in some cases for imtcp and imgssapi. This could + lead to a temporary stall of some connections. For ultry-low traffic systems, this + stall could stay for a long period of time. In most cases, it was resolved very quickly. + Note that imptcp was not affected. + Thanks to Iwan Timmer for the fix. +- 2022-05-05: net bugfix: potential buffer overrun + there is heap buffer overflow vulnerability in rsyslog tcp reception components. + This can only happen in octet-counted mode, which is enabled by default. + Affected components: imtcp, imptcp, imhttp, imgssapi, imdiag when octet-counted + framing was enabled. + If the receiver ports are exposed to the public Internet AND are used + without authentication, this can lead to remote DoS and potentially to + remote code execution. It is unclear if remote code execution is + actually possible. If so, it needs a very sophisticated attack. + When syslog best practices with proper firewalling and authentication + is used, thean attack can only be carried out from within the Intranet + and authorized systems. This limits the severity of the vulnerability + considerably (it would obviously require an attacker already to be + present inside the internal network). + Credits to Peter Agten for initially reporting the issue and working + with us on the resolution. + fixes CVE-2022-24903 + Advisory: + https://github.com/rsyslog/rsyslog/security/advisories/GHSA-ggw7-xr6h-mmr8#advisory-comment-72243 - 2022-05-05: imptcp: set OS worker thread name We now set the worker thread names to "imptcp/" where is the numerical index (0, 1, ...) of the worker thread. From 76dc3a0e6716ba1aebb3ad74c8e171d3885a9a2f Mon Sep 17 00:00:00 2001 From: EHerzog76 Date: Thu, 24 Feb 2022 09:09:22 +0100 Subject: [PATCH 082/134] omelasticsearch: add support for Data Streams (ES 8) See online: https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html --- plugins/omelasticsearch/omelasticsearch.c | 60 +++++++++++++++++------ 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index ed9359732c..8200403eaf 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -86,12 +86,14 @@ STATSCOUNTER_DEF(rebinds, mutRebinds) static prop_t *pInputName = NULL; # define META_STRT "{\"index\":{\"_index\": \"" -# define META_STRT_CREATE "{\"create\":{\"_index\": \"" +# define META_STRT_CREATE "{\"create\":{" /* \"_index\": \" */ +# define META_IX "\"_index\": \"" # define META_TYPE "\",\"_type\":\"" # define META_PIPELINE "\",\"pipeline\":\"" # define META_PARENT "\",\"_parent\":\"" # define META_ID "\", \"_id\":\"" # define META_END "\"}}\n" +# define META_END_NOQUOTE " }}\n" typedef enum { ES_WRITE_INDEX, @@ -362,8 +364,8 @@ CODESTARTdbgPrintInstInfo dbgprintf("\tdefaultPort=%d\n", pData->defaultPort); dbgprintf("\tuid='%s'\n", pData->uid == NULL ? (uchar*)"(not configured)" : pData->uid); dbgprintf("\tpwd=(%sconfigured)\n", pData->pwd == NULL ? "not " : ""); - dbgprintf("\tsearch index='%s'\n", pData->searchIndex); - dbgprintf("\tsearch type='%s'\n", pData->searchType); + dbgprintf("\tsearch index='%s'\n", pData->searchIndex == NULL ? (uchar*)"(not configured)" : pData->searchIndex); + dbgprintf("\tsearch type='%s'\n", pData->searchType == NULL ? (uchar*)"(not configured)" : pData->searchType); dbgprintf("\tpipeline name='%s'\n", pData->pipelineName); dbgprintf("\tdynamic pipeline name=%d\n", pData->dynPipelineName); dbgprintf("\tskipPipelineIfEmpty=%d\n", pData->skipPipelineIfEmpty); @@ -596,8 +598,8 @@ getIndexTypeAndParent(const instanceData *const pData, uchar **const tpls, } done: - assert(srchIndex != NULL); - assert(srchType != NULL); + //assert(srchIndex != NULL); + //assert(srchType != NULL); return; } @@ -633,9 +635,14 @@ setPostURL(wrkrInstanceData_t *const pWrkrData, uchar **const tpls) parent = NULL; } else { getIndexTypeAndParent(pData, tpls, &searchIndex, &searchType, &parent, &bulkId, &pipelineName); - r = es_addBuf(&url, (char*)searchIndex, ustrlen(searchIndex)); - if(r == 0) r = es_addChar(&url, '/'); - if(r == 0) r = es_addBuf(&url, (char*)searchType, ustrlen(searchType)); + if(searchIndex != NULL) { + r = es_addBuf(&url, (char*)searchIndex, ustrlen(searchIndex)); + if(r == 0) r = es_addChar(&url, '/'); + if(searchType != NULL) { + if(r == 0) r = es_addBuf(&url, (char*)searchType, ustrlen(searchType)); + } + } else + r = 0; if(pipelineName != NULL && (!pData->skipPipelineIfEmpty || pipelineName[0] != '\0')) { if(r == 0) r = es_addChar(&url, separator); if(r == 0) r = es_addBuf(&url, "pipeline=", sizeof("pipeline=")-1); @@ -692,7 +699,11 @@ computeMessageSize(const wrkrInstanceData_t *const pWrkrData, uchar *pipelineName; getIndexTypeAndParent(pWrkrData->pData, tpls, &searchIndex, &searchType, &parent, &bulkId, &pipelineName); - r += ustrlen((char *)message) + ustrlen(searchIndex) + ustrlen(searchType); + r += ustrlen((char *)message); + if(searchIndex != NULL) + r += ustrlen(searchIndex); + if(searchType != NULL) + r += ustrlen(searchType); if(parent != NULL) { r += sizeof(META_PARENT)-1 + ustrlen(parent); @@ -717,6 +728,7 @@ buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) { int length = strlen((char *)message); int r; + int endQuote = 1; uchar *searchIndex = NULL; uchar *searchType; uchar *parent = NULL; @@ -725,28 +737,43 @@ buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) DEFiRet; getIndexTypeAndParent(pWrkrData->pData, tpls, &searchIndex, &searchType, &parent, &bulkId, &pipelineName); - if (pWrkrData->pData->writeOperation == ES_WRITE_CREATE) + if (pWrkrData->pData->writeOperation == ES_WRITE_CREATE) { r = es_addBuf(&pWrkrData->batch.data, META_STRT_CREATE, sizeof(META_STRT_CREATE)-1); - else + endQuote = 0; + } else r = es_addBuf(&pWrkrData->batch.data, META_STRT, sizeof(META_STRT)-1); - if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchIndex, + if(searchIndex != NULL) { + endQuote = 1; + if (pWrkrData->pData->writeOperation == ES_WRITE_CREATE) + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_IX, sizeof(META_IX)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchIndex, ustrlen(searchIndex)); - if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_TYPE, sizeof(META_TYPE)-1); - if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchType, + if(searchType != NULL) { + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_TYPE, sizeof(META_TYPE)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchType, ustrlen(searchType)); + } + } if(parent != NULL) { + endQuote = 1; if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_PARENT, sizeof(META_PARENT)-1); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)parent, ustrlen(parent)); } if(pipelineName != NULL && (!pWrkrData->pData->skipPipelineIfEmpty || pipelineName[0] != '\0')) { + endQuote = 1; if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_PIPELINE, sizeof(META_PIPELINE)-1); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)pipelineName, ustrlen(pipelineName)); } if(bulkId != NULL) { + endQuote = 1; if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_ID, sizeof(META_ID)-1); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)bulkId, ustrlen(bulkId)); } - if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_END, sizeof(META_END)-1); + if(endQuote == 0) { + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_END_NOQUOTE, sizeof(META_END_NOQUOTE)-1); + } else { + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_END, sizeof(META_END)-1); + } if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)message, length); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, "\n", sizeof("\n")-1); if(r != 0) { @@ -2094,6 +2121,8 @@ CODESTARTnewActInst CHKiRet(computeBaseUrl("localhost", pData->defaultPort, pData->useHttps, pData->serverBaseUrls)); } + //Only needed befor ES-Version 7.x + /* if(pData->searchIndex == NULL) pData->searchIndex = (uchar*) strdup("system"); if(pData->searchType == NULL) @@ -2104,6 +2133,7 @@ CODESTARTnewActInst "omelasticsearch: writeoperation '%d' requires bulkid", pData->writeOperation); ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } + */ if (pData->retryFailures) { CHKiRet(ratelimitNew(&pData->ratelimiter, "omelasticsearch", NULL)); From b564a1345154fdc416ce04528fd0b4eca658a346 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 May 2022 13:06:59 +0200 Subject: [PATCH 083/134] testbench bugfix: some valgrind tests were run when valgrind disabled This leads to test failures e.g. with LLVM sanitizers. --- tests/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 805949ec8b..f14cd19808 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -283,7 +283,6 @@ TESTS += \ allowed-sender-tcp-fail.sh \ allowed-sender-tcp-hostname-ok.sh \ allowed-sender-tcp-hostname-fail.sh \ - imtcp-octet-framing-too-long-vg.sh \ imtcp-discard-truncated-msg.sh \ imtcp-basic.sh \ imtcp-basic-hup.sh \ @@ -514,6 +513,7 @@ TESTS += \ omfile_hup-vg.sh \ gzipwr_hup-vg.sh \ tcpflood_wrong_option_output.sh \ + imtcp-octet-framing-too-long-vg.sh \ smtradfile-vg.sh \ dnscache-TTL-0-vg.sh \ include-obj-outside-control-flow-vg.sh \ @@ -1075,7 +1075,6 @@ if ENABLE_IMPTCP # need to be disabled if we do not have this module TESTS += \ manyptcp.sh \ - imptcp-octet-framing-too-long-vg.sh \ imptcp_framing_regex.sh \ imptcp_framing_regex-oversize.sh \ imptcp_large.sh \ @@ -1100,6 +1099,7 @@ TESTS += \ omfile-outchannel-many.sh if HAVE_VALGRIND TESTS += \ + imptcp-octet-framing-too-long-vg.sh \ imptcp_conndrop-vg.sh if ENABLE_FMHASH TESTS += \ From 3f55aa73372dce178d6ef03a9d506e56c04e35e6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 9 May 2022 09:35:27 +0200 Subject: [PATCH 084/134] elasticsearch: add ability to configure ES server version This is needed in order to support version-specific ES API changes. This patch also corrects some code style issues. --- plugins/omelasticsearch/omelasticsearch.c | 51 +++++++++++++---------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index 8200403eaf..8b74d610df 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -4,7 +4,7 @@ * NOTE: read comments in module-template.h for more specifics! * * Copyright 2011 Nathan Scott. - * Copyright 2009-2021 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2022 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -130,6 +130,7 @@ typedef struct instanceConf_s { uchar *timeout; uchar *bulkId; uchar *errorFile; + int esVersion; sbool errorOnly; sbool interleaved; sbool dynSrchIdx; @@ -221,7 +222,8 @@ static struct cnfparamdescr actpdescr[] = { { "ratelimit.interval", eCmdHdlrInt, 0 }, { "ratelimit.burst", eCmdHdlrInt, 0 }, { "retryruleset", eCmdHdlrString, 0 }, - { "rebindinterval", eCmdHdlrInt, 0 } + { "rebindinterval", eCmdHdlrInt, 0 }, + { "esversion.major", eCmdHdlrPositiveInt, 0 } }; static struct cnfparamblk actpblk = { CNFPARAMBLK_VERSION, @@ -246,6 +248,7 @@ CODESTARTcreateInstance pData->retryRulesetName = NULL; pData->retryRuleset = NULL; pData->rebindInterval = DEFAULT_REBIND_INTERVAL; + pData->esVersion = 0; finalize_it: ENDcreateInstance @@ -364,8 +367,10 @@ CODESTARTdbgPrintInstInfo dbgprintf("\tdefaultPort=%d\n", pData->defaultPort); dbgprintf("\tuid='%s'\n", pData->uid == NULL ? (uchar*)"(not configured)" : pData->uid); dbgprintf("\tpwd=(%sconfigured)\n", pData->pwd == NULL ? "not " : ""); - dbgprintf("\tsearch index='%s'\n", pData->searchIndex == NULL ? (uchar*)"(not configured)" : pData->searchIndex); - dbgprintf("\tsearch type='%s'\n", pData->searchType == NULL ? (uchar*)"(not configured)" : pData->searchType); + dbgprintf("\tsearch index='%s'\n", pData->searchIndex == NULL + ? (uchar*)"(not configured)" : pData->searchIndex); + dbgprintf("\tsearch type='%s'\n", pData->searchType == NULL + ? (uchar*)"(not configured)" : pData->searchType); dbgprintf("\tpipeline name='%s'\n", pData->pipelineName); dbgprintf("\tdynamic pipeline name=%d\n", pData->dynPipelineName); dbgprintf("\tskipPipelineIfEmpty=%d\n", pData->skipPipelineIfEmpty); @@ -598,8 +603,6 @@ getIndexTypeAndParent(const instanceData *const pData, uchar **const tpls, } done: - //assert(srchIndex != NULL); - //assert(srchType != NULL); return; } @@ -700,11 +703,12 @@ computeMessageSize(const wrkrInstanceData_t *const pWrkrData, getIndexTypeAndParent(pWrkrData->pData, tpls, &searchIndex, &searchType, &parent, &bulkId, &pipelineName); r += ustrlen((char *)message); - if(searchIndex != NULL) - r += ustrlen(searchIndex); - if(searchType != NULL) - r += ustrlen(searchType); - + if(searchIndex != NULL) { + r += ustrlen(searchIndex); + } + if(searchType != NULL) { + r += ustrlen(searchType); + } if(parent != NULL) { r += sizeof(META_PARENT)-1 + ustrlen(parent); } @@ -728,7 +732,7 @@ buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) { int length = strlen((char *)message); int r; - int endQuote = 1; + int endQuote = 1; uchar *searchIndex = NULL; uchar *searchType; uchar *parent = NULL; @@ -1990,6 +1994,8 @@ CODESTARTnewActInst pData->retryRulesetName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "rebindinterval")) { pData->rebindInterval = (int) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "esversion.major")) { + pData->esVersion = pvals[i].val.d.n; } else { LogError(0, RS_RET_INTERNAL_ERROR, "omelasticsearch: program error, " "non-handled param '%s'", actpblk.descr[i].name); @@ -2121,19 +2127,18 @@ CODESTARTnewActInst CHKiRet(computeBaseUrl("localhost", pData->defaultPort, pData->useHttps, pData->serverBaseUrls)); } - //Only needed befor ES-Version 7.x - /* - if(pData->searchIndex == NULL) - pData->searchIndex = (uchar*) strdup("system"); - if(pData->searchType == NULL) - pData->searchType = (uchar*) strdup("events"); + if(pData->esVersion < 8) { + if(pData->searchIndex == NULL) + pData->searchIndex = (uchar*) strdup("system"); + if(pData->searchType == NULL) + pData->searchType = (uchar*) strdup("events"); - if ((pData->writeOperation != ES_WRITE_INDEX) && (pData->bulkId == NULL)) { - LogError(0, RS_RET_CONFIG_ERROR, - "omelasticsearch: writeoperation '%d' requires bulkid", pData->writeOperation); - ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + if ((pData->writeOperation != ES_WRITE_INDEX) && (pData->bulkId == NULL)) { + LogError(0, RS_RET_CONFIG_ERROR, + "omelasticsearch: writeoperation '%d' requires bulkid", pData->writeOperation); + ABORT_FINALIZE(RS_RET_CONFIG_ERROR); + } } - */ if (pData->retryFailures) { CHKiRet(ratelimitNew(&pData->ratelimiter, "omelasticsearch", NULL)); From a4d42a66bbf0786ae7ba56445b6cdc974d4e2f01 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 May 2022 10:52:09 +0200 Subject: [PATCH 085/134] maintain ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3448cc874b..aa333f1d78 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-11: omelasticsearch: several support option for ElasticSearch 8 + - config params searchIndex and documentType can be empty + - support for Data Stream API + Thanks to github user EHerzog76 for these changes. + - new config param esVersion.major - 2022-05-09: tcp receiver bugfix: delay/potential hang on some error conditions Error were not correctly handled in some cases for imtcp and imgssapi. This could lead to a temporary stall of some connections. For ultry-low traffic systems, this From 34e72ced504458d804f8d1049be67ea8cc00a4b4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 4 Mar 2022 11:39:11 +0100 Subject: [PATCH 086/134] imptcp bugfix: worker thread starvation on extreme traffic When connectes were totally busy, without any pause, the assigened worker did never terminate its reading loop. As such, it could not service any other conenctions. If this happened multiple time and to all configured workers, all other connections could not be processed at all. This extreme scenario is very unlikely, as the whole issue is relatively unlikely. In practice, the issue could lead to somewhat degraded performance and resolved itself after some time (in practice no connection is 100% busy for an extended period of time). Note that this patch sets a fixed limit of 16 iterations for very busy connections. This sounds like a good compromise between non-starvation and performance. The exact number may be made configurable if there is really need to. --- plugins/imptcp/imptcp.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 157b87b873..1ec3946fa8 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -10,7 +10,7 @@ * * File begun on 2010-08-10 by RGerhards * - * Copyright 2007-2018 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2022 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -1395,7 +1395,7 @@ addEPollSock(epolld_type_t typ, void *ptr, int sock, epolld_t **pEpd) epd->ptr = ptr; epd->sock = sock; *pEpd = epd; - epd->ev.events = EPOLLIN|EPOLLET|EPOLLONESHOT; + epd->ev.events = EPOLLIN|EPOLLONESHOT; epd->ev.data.ptr = (void*) epd; if(epoll_ctl(epollfd, EPOLL_CTL_ADD, sock, &(epd->ev)) != 0) { @@ -1944,11 +1944,12 @@ sessActivity(ptcpsess_t *const pSess, int *const continue_polling) int remsock = 0; /* init just to keep compiler happy... :-( */ sbool bEmitOnClose = 0; char rcvBuf[128*1024]; + int runs = 0; DEFiRet; DBGPRINTF("imptcp: new activity on session socket %d\n", pSess->sock); - while(1) { + while(runs++ < 16) { lenBuf = sizeof(rcvBuf); lenRcv = recv(pSess->sock, rcvBuf, lenBuf, 0); From c74f5c8523efcc1c67e2eecd3ea3a766dd7c0fd5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 May 2022 13:34:35 +0200 Subject: [PATCH 087/134] maintain ChangeLog --- ChangeLog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ChangeLog b/ChangeLog index aa333f1d78..348a5b288d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,18 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-12: imptcp bugfix: worker thread starvation on extreme traffic + When connectes were totally busy, without any pause, the assigened worker + did never terminate its reading loop. As such, it could not service any + other conenctions. If this happened multiple time and to all configured + workers, all other connections could not be processed at all. This extreme + scenario is very unlikely, as the whole issue is relatively unlikely. + In practice, the issue could lead to somewhat degraded performance and + resolved itself after some time (in practice no connection is 100% busy + for an extended period of time). + Note that this patch sets a fixed limit of 16 iterations for very busy + connections. This sounds like a good compromise between non-starvation + and performance. The exact number may be made configurable if there + is really need to. - 2022-05-11: omelasticsearch: several support option for ElasticSearch 8 - config params searchIndex and documentType can be empty - support for Data Stream API From d1a98b8260a35ac36b744573d944898c0805ec63 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 May 2022 10:19:21 +0200 Subject: [PATCH 088/134] tcpsrv/imtcp: slight performance improvements This patch slightly improves performance for tcpsrv-based servers. This affects imtcp and imgssapi as well as some helpers. No other functional change is included in this patch. --- runtime/tcps_sess.c | 142 ++++++++++++++++++++++---------------------- runtime/tcps_sess.h | 3 +- tests/diag.sh | 1 + 3 files changed, 74 insertions(+), 72 deletions(-) diff --git a/runtime/tcps_sess.c b/runtime/tcps_sess.c index c5442f7638..b0832b1fce 100644 --- a/runtime/tcps_sess.c +++ b/runtime/tcps_sess.c @@ -67,10 +67,11 @@ static rsRetVal Close(tcps_sess_t *pThis); /* Standard-Constructor */ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END macro! */ pThis->iMsg = 0; /* just make sure... */ + pThis->iMaxLine = glbl.GetMaxLine(runConf); pThis->inputState = eAtStrtFram; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ /* now allocate the message reception buffer */ - CHKmalloc(pThis->pMsg = (uchar*) malloc(glbl.GetMaxLine(runConf) + 1)); + CHKmalloc(pThis->pMsg = (uchar*) malloc(pThis->iMaxLine + 1)); finalize_it: ENDobjConstruct(tcps_sess) @@ -361,18 +362,13 @@ processDataRcvd(tcps_sess_t *pThis, DEFiRet; const tcpLstnParams_t *const cnf_params = pThis->pLstnInfo->cnf_params; ISOBJ_TYPE_assert(pThis, tcps_sess); - int iMaxLine = glbl.GetMaxLine(runConf); - uchar *propPeerName = NULL; - int lenPeerName = 0; - uchar *propPeerIP = NULL; - int lenPeerIP = 0; if(pThis->inputState == eAtStrtFram) { - if(pThis->bSuppOctetFram && c >= '0' && c <= '9') { + if(c >= '0' && c <= '9' && pThis->bSuppOctetFram) { pThis->inputState = eInOctetCnt; pThis->iOctetsRemain = 0; pThis->eFraming = TCP_FRAMING_OCTET_COUNTING; - } else if(pThis->bSPFramingFix && c == ' ') { + } else if(c == ' ' && pThis->bSPFramingFix) { /* Cisco ASA very occasionally sends a SP after a LF, which * thrashes framing if not taken special care of. Here, * we permit space *in front of the next frame* and @@ -385,68 +381,7 @@ processDataRcvd(tcps_sess_t *pThis, } } - if(pThis->inputState == eInOctetCnt) { - if(c >= '0' && c <= '9') { /* isdigit() the faster way */ - if(pThis->iOctetsRemain <= 200000000) { - pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; - } - if(pThis->iMsg < iMaxLine) { - *(pThis->pMsg + pThis->iMsg++) = c; - } - } else { /* done with the octet count, so this must be the SP terminator */ - DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); - prop.GetString(pThis->fromHost, &propPeerName, &lenPeerName); - prop.GetString(pThis->fromHost, &propPeerIP, &lenPeerIP); - if(c != ' ') { - LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " - "peer: (hostname) %s, (ip) %s: delimiter is not SP but has " - "ASCII value %d.", cnf_params->pszInputName, propPeerName, propPeerIP, c); - } - if(pThis->iOctetsRemain < 1) { - /* TODO: handle the case where the octet count is 0! */ - LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " - "peer: (hostname) %s, (ip) %s: invalid octet count %d.", - cnf_params->pszInputName, propPeerName, propPeerIP, pThis->iOctetsRemain); - pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; - } else if(pThis->iOctetsRemain > iMaxLine) { - /* while we can not do anything against it, we can at least log an indication - * that something went wrong) -- rgerhards, 2008-03-14 - */ - LogError(0, NO_ERRCODE, "imtcp %s: received oversize message from peer: " - "(hostname) %s, (ip) %s: size is %d bytes, max msg size " - "is %d, truncating...", cnf_params->pszInputName, propPeerName, - propPeerIP, pThis->iOctetsRemain, iMaxLine); - } - if(pThis->iOctetsRemain > pThis->pSrv->maxFrameSize) { - LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " - "peer: (hostname) %s, (ip) %s: frame too large: %d, change " - "to octet stuffing", cnf_params->pszInputName, propPeerName, propPeerIP, - pThis->iOctetsRemain); - pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; - } else { - pThis->iMsg = 0; - } - pThis->inputState = eInMsg; - } - } else if(pThis->inputState == eInMsgTruncating) { - if(pThis->eFraming == TCP_FRAMING_OCTET_COUNTING) { - DBGPRINTF("DEBUG: TCP_FRAMING_OCTET_COUNTING eInMsgTruncating c=%c remain=%d\n", - c, pThis->iOctetsRemain); - - pThis->iOctetsRemain--; - if(pThis->iOctetsRemain < 1) { - pThis->inputState = eAtStrtFram; - } - } else { - if( ((c == '\n') && !pThis->pSrv->bDisableLFDelim) - || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) - && (c == pThis->pSrv->addtlFrameDelim)) - ) { - pThis->inputState = eAtStrtFram; - } - } - } else { - assert(pThis->inputState == eInMsg); + if(pThis->inputState == eInMsg) { #if 0 // set to 1 for ultra-verbose DBGPRINTF("DEBUG: processDataRcvd c=%c remain=%d\n", c, pThis->iOctetsRemain); @@ -464,7 +399,7 @@ processDataRcvd(tcps_sess_t *pThis, * If we have a message that is larger than the max msg size, we truncate it. This is the best * we can do in light of what the engine supports. -- rgerhards, 2008-03-14 */ - if(pThis->iMsg < iMaxLine) { + if(pThis->iMsg < pThis->iMaxLine) { *(pThis->pMsg + pThis->iMsg++) = c; } else { /* emergency, we now need to flush, no matter if we are at end of message or not... */ @@ -496,6 +431,71 @@ processDataRcvd(tcps_sess_t *pThis, pThis->inputState = eAtStrtFram; } } + } else if(pThis->inputState == eInMsgTruncating) { + if(pThis->eFraming == TCP_FRAMING_OCTET_COUNTING) { + DBGPRINTF("DEBUG: TCP_FRAMING_OCTET_COUNTING eInMsgTruncating c=%c remain=%d\n", + c, pThis->iOctetsRemain); + + pThis->iOctetsRemain--; + if(pThis->iOctetsRemain < 1) { + pThis->inputState = eAtStrtFram; + } + } else { + if( ((c == '\n') && !pThis->pSrv->bDisableLFDelim) + || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) + && (c == pThis->pSrv->addtlFrameDelim)) + ) { + pThis->inputState = eAtStrtFram; + } + } + } else { + assert(pThis->inputState == eInOctetCnt); + if(c >= '0' && c <= '9') { /* isdigit() the faster way */ + if(pThis->iOctetsRemain <= 200000000) { + pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; + } + if(pThis->iMsg < pThis->iMaxLine) { + *(pThis->pMsg + pThis->iMsg++) = c; + } + } else { /* done with the octet count, so this must be the SP terminator */ + uchar *propPeerName = NULL; + uchar *propPeerIP = NULL; + int lenPeerName = 0; + int lenPeerIP = 0; + DBGPRINTF("TCP Message with octet-counter, size %d.\n", pThis->iOctetsRemain); + prop.GetString(pThis->fromHost, &propPeerName, &lenPeerName); + prop.GetString(pThis->fromHost, &propPeerIP, &lenPeerIP); + if(c != ' ') { + LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " + "peer: (hostname) %s, (ip) %s: delimiter is not SP but has " + "ASCII value %d.", cnf_params->pszInputName, propPeerName, propPeerIP, c); + } + if(pThis->iOctetsRemain < 1) { + /* TODO: handle the case where the octet count is 0! */ + LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " + "peer: (hostname) %s, (ip) %s: invalid octet count %d.", + cnf_params->pszInputName, propPeerName, propPeerIP, pThis->iOctetsRemain); + pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; + } else if(pThis->iOctetsRemain > pThis->iMaxLine) { + /* while we can not do anything against it, we can at least log an indication + * that something went wrong) -- rgerhards, 2008-03-14 + */ + LogError(0, NO_ERRCODE, "imtcp %s: received oversize message from peer: " + "(hostname) %s, (ip) %s: size is %d bytes, max msg size " + "is %d, truncating...", cnf_params->pszInputName, propPeerName, + propPeerIP, pThis->iOctetsRemain, pThis->iMaxLine); + } + if(pThis->iOctetsRemain > pThis->pSrv->maxFrameSize) { + LogError(0, NO_ERRCODE, "imtcp %s: Framing Error in received TCP message from " + "peer: (hostname) %s, (ip) %s: frame too large: %d, change " + "to octet stuffing", cnf_params->pszInputName, propPeerName, propPeerIP, + pThis->iOctetsRemain); + pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; + } else { + pThis->iMsg = 0; + } + pThis->inputState = eInMsg; + } } finalize_it: diff --git a/runtime/tcps_sess.h b/runtime/tcps_sess.h index 46998d4b14..578b55942e 100644 --- a/runtime/tcps_sess.h +++ b/runtime/tcps_sess.h @@ -1,7 +1,7 @@ /* Definitions for tcps_sess class. This implements a session of the * plain TCP server. * - * Copyright 2008-2015 Adiscon GmbH. + * Copyright 2008-2022 Adiscon GmbH. * * This file is part of rsyslog. * @@ -50,6 +50,7 @@ struct tcps_sess_s { prop_t *fromHostIP; void *pUsr; /* a user-pointer */ rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ + int iMaxLine; /* fast lookup buffer for config property */ }; diff --git a/tests/diag.sh b/tests/diag.sh index 55874903be..94c5a3d03b 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -68,6 +68,7 @@ export TB_ERR_TIMEOUT=101 export ZOOPIDFILE="$(pwd)/zookeeper.pid" #valgrind="valgrind --malloc-fill=ff --free-fill=fe --log-fd=1" +#valgrind="valgrind --tool=callgrind" # for kcachegrind profiling # **** use the line below for very hard to find leaks! ***** #valgrind="valgrind --leak-check=full --show-leak-kinds=all --malloc-fill=ff --free-fill=fe --log-fd=1" From 27404ed7eea8752352930b8811d8ab5cf16d36ae Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 18 May 2022 10:48:07 +0200 Subject: [PATCH 089/134] maintain ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 348a5b288d..e5b10a6bfb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-18: tcpsrv/imtcp: slight performance improvements + This change slightly improves performance for tcpsrv-based servers. + This affects imtcp and imgssapi as well as some helpers. + No other functional change is included in this change. - 2022-05-12: imptcp bugfix: worker thread starvation on extreme traffic When connectes were totally busy, without any pause, the assigened worker did never terminate its reading loop. As such, it could not service any From d47b5c9b1c00bc2a107027bae1152d3e171059c2 Mon Sep 17 00:00:00 2001 From: Jarkko Oranen Date: Tue, 6 Apr 2021 20:42:46 +0300 Subject: [PATCH 090/134] omelastisearch: allow omitting _type field Allow omitting the _type field by setting it to an empty string. Setting this field has been deprecated since 6.0, and support will be removed in 8.0 Also add testbench test for empty searchType with ES 7.0 This checks for messages in the deprecation log and also provides avoids deprecation messages from usage of transport.tcp.port in the test configuration --- plugins/omelasticsearch/omelasticsearch.c | 15 ++++++---- tests/diag.sh | 4 +++ tests/es-basic-es7.0.sh | 36 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) create mode 100755 tests/es-basic-es7.0.sh diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index 76d5081d3b..f481ec3f7e 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -620,6 +620,8 @@ setPostURL(wrkrInstanceData_t *const pWrkrData, uchar **const tpls) uchar *parent; uchar *bulkId; char* baseUrl; + /* since 7.0, the API always requires /idx/_doc, so use that if searchType is not explicitly set */ + uchar* actualSearchType = (uchar*)"_doc"; es_str_t *url; int r; DEFiRet; @@ -645,11 +647,12 @@ setPostURL(wrkrInstanceData_t *const pWrkrData, uchar **const tpls) if(searchIndex != NULL) { r = es_addBuf(&url, (char*)searchIndex, ustrlen(searchIndex)); if(r == 0) r = es_addChar(&url, '/'); - if(searchType != NULL) { - if(r == 0) r = es_addBuf(&url, (char*)searchType, ustrlen(searchType)); - } - } else - r = 0; + + if(searchType != NULL) { + actualSearchType = searchType; + } + if(r == 0) r = es_addChar(&url, '/'); + if(r == 0) r = es_addBuf(&url, (char*)actualSearchType, ustrlen(actualSearchType)); if(pipelineName != NULL && (!pData->skipPipelineIfEmpty || pipelineName[0] != '\0')) { if(r == 0) r = es_addChar(&url, separator); if(r == 0) r = es_addBuf(&url, "pipeline=", sizeof("pipeline=")-1); @@ -693,7 +696,7 @@ computeMessageSize(const wrkrInstanceData_t *const pWrkrData, const uchar *const message, uchar **const tpls) { - size_t r = sizeof(META_TYPE)-1 + sizeof(META_END)-1 + sizeof("\n")-1; + size_t r = sizeof(META_END)-1 + sizeof("\n")-1; if (pWrkrData->pData->writeOperation == ES_WRITE_CREATE) r += sizeof(META_STRT_CREATE)-1; else diff --git a/tests/diag.sh b/tests/diag.sh index 94c5a3d03b..22f4779a56 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -2142,6 +2142,10 @@ prepare_elasticsearch() { cp -f $srcdir/testsuites/$dep_work_es_config $dep_work_dir/es/config/elasticsearch.yml fi + # Avoid deprecated parameter, new option introduced with 6.7 + echo "Setting transport tcp option to ${ES_PORT_OPTION:-transport.tcp.port}" + sed -i "s/transport.tcp.port/${ES_PORT_OPTION:-transport.tcp.port}/g" "$dep_work_dir/es/config/elasticsearch.yml" + if [ ! -d $dep_work_dir/es/data ]; then echo "Creating elastic search directories" mkdir -p $dep_work_dir/es/data diff --git a/tests/es-basic-es7.0.sh b/tests/es-basic-es7.0.sh new file mode 100755 index 0000000000..1e6b36d7c2 --- /dev/null +++ b/tests/es-basic-es7.0.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +export ES_DOWNLOAD=elasticsearch-7.10.2.tar.gz +export ES_PORT=19200 +# Using the default will cause deprecation failures +export ES_PORT_OPTION="transport.port" +export NUMMESSAGES=2000 # slow test +export QUEUE_EMPTY_CHECK_FUNC=es_shutdown_empty_check +ensure_elasticsearch_ready +generate_conf +add_conf ' +template(name="tpl" type="string" + string="{\"msgnum\":\"%msg:F,58:2%\"}") + +module(load="../plugins/omelasticsearch/.libs/omelasticsearch") + +if $msg contains "msgnum:" then + action(type="omelasticsearch" + server="127.0.0.1" + serverport="19200" + searchType="" + template="tpl" + searchIndex="rsyslog_testbench") +' +startup +injectmsg +shutdown_when_empty +wait_shutdown +es_getdata +seq_check +if grep "DEPRECATION" $dep_work_dir/es/logs/rsyslog-testbench_deprecation.log; then + echo "Found deprecations, failing!" + exit 1 +fi +exit_test From 18e3505b629d7fa0a16b1ffa3869ca37314fcb25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 15 Sep 2021 19:11:57 +0200 Subject: [PATCH 091/134] omelasticsearch: some amendments to contributed patch We disable ElasticSearch strict security in testbench, as we do not need it for test runs, and it complicates thing.s Note: this does NOT introduce a security weakness, because we use only temporary testing ES instances which are always immediately discarded after the test AND are run in our own test env with user permissions. --- plugins/omelasticsearch/omelasticsearch.c | 19 +++++++++++-------- tests/Makefile.am | 5 ++++- tests/diag.sh | 3 +++ ...-basic-es7.0.sh => es-searchType-empty.sh} | 1 - 4 files changed, 18 insertions(+), 10 deletions(-) rename tests/{es-basic-es7.0.sh => es-searchType-empty.sh} (95%) diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index f481ec3f7e..b297a9274f 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -623,7 +623,7 @@ setPostURL(wrkrInstanceData_t *const pWrkrData, uchar **const tpls) /* since 7.0, the API always requires /idx/_doc, so use that if searchType is not explicitly set */ uchar* actualSearchType = (uchar*)"_doc"; es_str_t *url; - int r; + int r = 0; DEFiRet; instanceData *const pData = pWrkrData->pData; char separator; @@ -646,13 +646,12 @@ setPostURL(wrkrInstanceData_t *const pWrkrData, uchar **const tpls) getIndexTypeAndParent(pData, tpls, &searchIndex, &searchType, &parent, &bulkId, &pipelineName); if(searchIndex != NULL) { r = es_addBuf(&url, (char*)searchIndex, ustrlen(searchIndex)); + if(searchType != NULL && searchType[0] != '\0') { + actualSearchType = searchType; + } if(r == 0) r = es_addChar(&url, '/'); - - if(searchType != NULL) { - actualSearchType = searchType; + if(r == 0) r = es_addBuf(&url, (char*)actualSearchType, ustrlen(actualSearchType)); } - if(r == 0) r = es_addChar(&url, '/'); - if(r == 0) r = es_addBuf(&url, (char*)actualSearchType, ustrlen(actualSearchType)); if(pipelineName != NULL && (!pData->skipPipelineIfEmpty || pipelineName[0] != '\0')) { if(r == 0) r = es_addChar(&url, separator); if(r == 0) r = es_addBuf(&url, "pipeline=", sizeof("pipeline=")-1); @@ -714,7 +713,11 @@ computeMessageSize(const wrkrInstanceData_t *const pWrkrData, r += ustrlen(searchIndex); } if(searchType != NULL) { - r += ustrlen(searchType); + if(searchType[0] == '\0') { + r += 4; // "_doc" + } else { + r += ustrlen(searchType); + } } if(parent != NULL) { r += sizeof(META_PARENT)-1 + ustrlen(parent); @@ -759,7 +762,7 @@ buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_IX, sizeof(META_IX)-1); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchIndex, ustrlen(searchIndex)); - if(searchType != NULL) { + if(searchType != NULL && searchType[0] != '\0') { if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_TYPE, sizeof(META_TYPE)-1); if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchType, ustrlen(searchType)); diff --git a/tests/Makefile.am b/tests/Makefile.am index f14cd19808..c190e528b3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -104,6 +104,7 @@ TESTS += \ es-basic-errfile-popul.sh \ es-bulk-errfile-empty.sh \ es-bulk-errfile-popul.sh \ + es-searchType-empty.sh \ diskqueue-multithread-es.sh \ es-writeoperation.sh @@ -114,7 +115,8 @@ es-basic-errfile-empty.log: es-maxbytes-bulk.log es-basic-errfile-popul.log: es-basic-errfile-empty.log es-bulk-errfile-empty.log: es-basic-errfile-popul.log es-bulk-errfile-popul.log: es-bulk-errfile-empty.log -diskqueue-multithread-es.log: es-bulk-errfile-popul.log +es-searchType-empty.log: es-bulk-errfile-popul.log +diskqueue-multithread-es.log: es-searchType-empty.log es-writeoperation.log: diskqueue-multithread-es.log elasticsearch-stop.log: es-writeoperation.log @@ -2183,6 +2185,7 @@ EXTRA_DIST= \ es-bulk-errfile-popul-erronly.sh \ es-bulk-errfile-popul-erronly-interleaved.sh \ es-bulk-errfile-popul-def-interleaved.sh \ + es-searchType-empty.sh \ diskqueue-multithread-es.sh \ es-basic-vg.sh \ es-basic-bulk-vg.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index 22f4779a56..1a98a8e0b0 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -2138,6 +2138,9 @@ prepare_elasticsearch() { if [ -n "${ES_PORT:-}" ] ; then rm -f $dep_work_dir/es/config/elasticsearch.yml sed "s/^http.port:.*\$/http.port: ${ES_PORT}/" $srcdir/testsuites/$dep_work_es_config > $dep_work_dir/es/config/elasticsearch.yml + if [ "$ES_DOWNLOAD" != "elasticsearch-6.0.0.tar.gz" ]; then + printf 'xpack.security.enabled: false\n' >> $dep_work_dir/es/config/elasticsearch.yml + fi else cp -f $srcdir/testsuites/$dep_work_es_config $dep_work_dir/es/config/elasticsearch.yml fi diff --git a/tests/es-basic-es7.0.sh b/tests/es-searchType-empty.sh similarity index 95% rename from tests/es-basic-es7.0.sh rename to tests/es-searchType-empty.sh index 1e6b36d7c2..bc0b01eaf4 100755 --- a/tests/es-basic-es7.0.sh +++ b/tests/es-searchType-empty.sh @@ -1,7 +1,6 @@ #!/bin/bash # This file is part of the rsyslog project, released under ASL 2.0 . ${srcdir:=.}/diag.sh init -export ES_DOWNLOAD=elasticsearch-7.10.2.tar.gz export ES_PORT=19200 # Using the default will cause deprecation failures export ES_PORT_OPTION="transport.port" From fa3ec461333963f510812c15720d499869d73f7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 May 2022 09:36:14 +0200 Subject: [PATCH 092/134] maintain ChangeLog --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index e5b10a6bfb..ddd87ea68e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +- 2022-05-25: omelastisearch: allow omitting _type field + Allow omitting the _type field by setting it to an empty string. + Setting this field has been deprecated since 6.0, and support will + be removed in 8.0 + Also add testbench test for empty searchType with ES 7.0 + This checks for messages in the deprecation log and also provides + avoids deprecation messages from usage of transport.tcp.port in the + test configuration + Thanks to Jarkko Oranen for the patch. - 2022-05-18: tcpsrv/imtcp: slight performance improvements This change slightly improves performance for tcpsrv-based servers. This affects imtcp and imgssapi as well as some helpers. From 05587611937b84cc2504a0821250a130c8ec4220 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 13 Jun 2022 11:06:45 +0200 Subject: [PATCH 093/134] prepare for 8.2206.0 scheduled stable release --- ChangeLog | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ddd87ea68e..3c4cba49e5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ ---------------------------------------------------------------------------------------- -Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-?? +Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-14 - 2022-05-25: omelastisearch: allow omitting _type field Allow omitting the _type field by setting it to an empty string. Setting this field has been deprecated since 6.0, and support will diff --git a/configure.ac b/configure.ac index a78f3093c9..b0e6526dc4 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2206.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2206.0],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 06, [month part of real rsyslog version]) # UPDATE on release From f791afe5c09167cabef12205629d619e883c9172 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Jun 2022 10:45:02 +0200 Subject: [PATCH 094/134] update versions for daily stable --- ChangeLog | 2 ++ configure.ac | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3c4cba49e5..a49107113f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ ---------------------------------------------------------------------------------------- +Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-14 - 2022-05-25: omelastisearch: allow omitting _type field Allow omitting the _type field by setting it to an empty string. diff --git a/configure.ac b/configure.ac index b0e6526dc4..31d9d7af2c 100644 --- a/configure.ac +++ b/configure.ac @@ -2,9 +2,9 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2206.0],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2208.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release -AC_DEFINE(VERSION_MONTH, 06, [month part of real rsyslog version]) # UPDATE on release +AC_DEFINE(VERSION_MONTH, 8, [month part of real rsyslog version]) # UPDATE on release AM_INIT_AUTOMAKE([subdir-objects]) From ee3f88ffe4915243fffa95afa68652eaabf2065a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Wed, 15 Jun 2022 16:49:24 +0200 Subject: [PATCH 095/134] Import when index() is used. --- runtime/nsd_ossl.c | 1 + tests/tcpflood.c | 1 + 2 files changed, 2 insertions(+) diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 4f605bef6a..f22e19a71e 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 278c3d6f00..f08bdad583 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -107,6 +107,7 @@ #include #include #include +#include #include #include #ifdef ENABLE_RELP From bd4d4a6c5ccf6c0d279e6c45bf719433bc48105d Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Mon, 27 Jun 2022 14:39:07 +0200 Subject: [PATCH 096/134] OpenSSL: fix depreacted API issues for OpenSSL 3.x - OpenSSL error strings are loaded automatically now - Debug Callback has changed - See for more: https://www.openssl.org/docs/manmaster/man7/migration_guide.html closes: https://github.com/rsyslog/rsyslog/issues/4912 --- runtime/nsd_ossl.c | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 4f605bef6a..2f1e851318 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -31,6 +31,9 @@ #include #include #include +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) +# include +#endif #include #include #include @@ -288,16 +291,20 @@ int verify_callback(int status, X509_STORE_CTX *store) return status; } +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) +long BIO_debug_callback_ex(BIO *bio, int cmd, const char __attribute__((unused)) *argp, + size_t __attribute__((unused)) len, int argi, long __attribute__((unused)) argl, + int ret, size_t __attribute__((unused)) *processed) +#else long BIO_debug_callback(BIO *bio, int cmd, const char __attribute__((unused)) *argp, int argi, long __attribute__((unused)) argl, long ret) +#endif { + long ret2 = ret; // Helper value to avoid printf compile errors long<>int long r = 1; - if (BIO_CB_RETURN & cmd) r = ret; - dbgprintf("openssl debugmsg: BIO[%p]: ", (void *)bio); - switch (cmd) { case BIO_CB_FREE: dbgprintf("Free - %s\n", RSYSLOG_BIO_method_name(bio)); @@ -344,19 +351,19 @@ long BIO_debug_callback(BIO *bio, int cmd, const char __attribute__((unused)) *a RSYSLOG_BIO_method_name(bio)); break; case BIO_CB_RETURN | BIO_CB_READ: - dbgprintf("read return %ld\n", ret); + dbgprintf("read return %ld\n", ret2); break; case BIO_CB_RETURN | BIO_CB_WRITE: - dbgprintf("write return %ld\n", ret); + dbgprintf("write return %ld\n", ret2); break; case BIO_CB_RETURN | BIO_CB_GETS: - dbgprintf("gets return %ld\n", ret); + dbgprintf("gets return %ld\n", ret2); break; case BIO_CB_RETURN | BIO_CB_PUTS: - dbgprintf("puts return %ld\n", ret); + dbgprintf("puts return %ld\n", ret2); break; case BIO_CB_RETURN | BIO_CB_CTRL: - dbgprintf("ctrl return %ld\n", ret); + dbgprintf("ctrl return %ld\n", ret2); break; default: dbgprintf("bio callback - unknown type (%d)\n", cmd); @@ -414,9 +421,19 @@ osslGlblInit(void) /* Load readable error strings */ SSL_load_error_strings(); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) + /* + * ERR_load_*(), ERR_func_error_string(), ERR_get_error_line(), ERR_get_error_line_data(), ERR_get_state() + * OpenSSL now loads error strings automatically so these functions are not needed. + * SEE FOR MORE: + * https://www.openssl.org/docs/manmaster/man7/migration_guide.html + * + */ +#else + /* Load error strings into mem*/ ERR_load_BIO_strings(); ERR_load_crypto_strings(); - +#endif RETiRet; } @@ -551,7 +568,11 @@ osslInitSession(nsd_ossl_t *pThis, osslSslState_t osslType) /* , nsd_ossl_t *pSe dbgprintf("osslInitSession: Init conn BIO[%p] done\n", (void *)conn); /* Set debug Callback for conn BIO as well! */ +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) + BIO_set_callback_ex(conn, BIO_debug_callback_ex); +#else BIO_set_callback(conn, BIO_debug_callback); +#endif /* TODO: still needed? Set to NON blocking ! */ BIO_set_nbio( conn, 1 ); From 8521c016317174baf40df152c23bc14af8333b95 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Jun 2022 17:17:41 +0200 Subject: [PATCH 097/134] maintain ChangeLog --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index a49107113f..b15c71726e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-06-22: OpenSSL: fix depreacted API issues for OpenSSL 3.x + - OpenSSL error strings are loaded automatically now + - Debug Callback has changed + - See for more: + https://www.openssl.org/docs/manmaster/man7/migration_guide.html + closes: https://github.com/rsyslog/rsyslog/issues/4912 ---------------------------------------------------------------------------------------- Scheduled Release 8.2206.0 (aka 2022.06) 2022-06-14 - 2022-05-25: omelastisearch: allow omitting _type field From e92f85b5888f6732d91c686dbf62f192000797e4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Jun 2022 17:26:14 +0200 Subject: [PATCH 098/134] build error fix: libbson requires out-of-date language constructs --- plugins/ommongodb/ommongodb.c | 5 ++++- runtime/rsyslog.h | 4 ++++ tests/diag.sh | 2 +- tools/logctl.c | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index 6470637ed9..08e587787a 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -42,6 +42,8 @@ PRAGMA_IGNORE_Wpragmas PRAGMA_IGNORE_Wunknown_warning_option PRAGMA_IGNORE_Wunknown_attribute PRAGMA_IGNORE_Wexpansion_to_defined +PRAGMA_IGNORE_Wstrict_prototypes +PRAGMA_IGNORE_Wold_style_definition #include #include PRAGMA_DIAGNOSTIC_POP @@ -196,6 +198,7 @@ reportMongoError(instanceData *pData) static rsRetVal initMongoDB(instanceData *pData, int bSilent) { DEFiRet; + unsigned char retval; DBGPRINTF("ommongodb: uristr is '%s'", pData->uristr); mongoc_init (); @@ -224,7 +227,7 @@ static rsRetVal initMongoDB(instanceData *pData, int bSilent) bson_t *command, reply; bson_error_t error; command = BCON_NEW ("ping", BCON_INT32 (1)); - unsigned char retval = mongoc_client_command_simple(pData->client, pData->db, command, NULL, &reply, &error); + retval = mongoc_client_command_simple(pData->client, pData->db, command, NULL, &reply, &error); bson_destroy(&reply); bson_destroy(command); if( !retval ) { diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index b641fc9952..4be62905bb 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -88,6 +88,8 @@ #if defined(__GNUC__) #define PRAGMA_INGORE_Wswitch_enum _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"") #define PRAGMA_IGNORE_Wempty_body _Pragma("GCC diagnostic ignored \"-Wempty-body\"") + #define PRAGMA_IGNORE_Wstrict_prototypes _Pragma("GCC diagnostic ignored \"-Wstrict-prototypes\"") + #define PRAGMA_IGNORE_Wold_style_definition _Pragma("GCC diagnostic ignored \"-Wold-style-definition\"") #define PRAGMA_IGNORE_Wsign_compare _Pragma("GCC diagnostic ignored \"-Wsign-compare\"") #define PRAGMA_IGNORE_Wpragmas _Pragma("GCC diagnostic ignored \"-Wpragmas\"") #define PRAGMA_IGNORE_Wmissing_noreturn _Pragma("GCC diagnostic ignored \"-Wmissing-noreturn\"") @@ -110,6 +112,8 @@ #define PRAGMA_IGNORE_Wpragmas #define PRAGMA_IGNORE_Wmissing_noreturn #define PRAGMA_IGNORE_Wempty_body + #define PRAGMA_IGNORE_Wstrict_prototypes + #define PRAGMA_IGNORE_Wold_style_definition #define PRAGMA_IGNORE_Wdeprecated_declarations #define PRAGMA_IGNORE_Wexpansion_to_defined #define PRAGMA_IGNORE_Wunknown_attribute diff --git a/tests/diag.sh b/tests/diag.sh index 1a98a8e0b0..fb766ab8e2 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -1632,7 +1632,7 @@ dep_kafka_url="https://www.rsyslog.com/files/download/rsyslog/$RS_KAFKA_DOWNLOAD dep_kafka_cached_file=$dep_cache_dir/$RS_KAFKA_DOWNLOAD if [ -z "$ES_DOWNLOAD" ]; then - export ES_DOWNLOAD=elasticsearch-7.14.1-linux-x86_64.tar.gz #elasticsearch-5.6.9.tar.gz + export ES_DOWNLOAD=elasticsearch-7.14.1-linux-x86_64.tar.gz fi if [ -z "$ES_PORT" ]; then export ES_PORT=19200 diff --git a/tools/logctl.c b/tools/logctl.c index 8e2307bd34..1010409c89 100644 --- a/tools/logctl.c +++ b/tools/logctl.c @@ -2,7 +2,7 @@ * logctl - a tool to access lumberjack logs in MongoDB * ... and potentially other sources in the future. * - * Copyright 2012 Ulrike Gerhards and Adiscon GmbH. + * Copyright 2012-2022 Ulrike Gerhards and Adiscon GmbH. * * Copyright 2017 Hugo Soszynski and aDvens * @@ -58,6 +58,8 @@ #pragma GCC diagnostic ignored "-Wpragmas" #pragma GCC diagnostic ignored "-Wunknown-attributes" #pragma GCC diagnostic ignored "-Wexpansion-to-defined" +#pragma GCC diagnostic ignored "-Wstrict-prototypes" +#pragma GCC diagnostic ignored "-Wold-style-definition" #endif #include #include From f0c3675a57547223f4777e127e853c6f8d266796 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Jun 2022 08:29:42 +0200 Subject: [PATCH 099/134] maintain ChangeLog --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b15c71726e..fee04f7fdb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? -- 2022-06-22: OpenSSL: fix depreacted API issues for OpenSSL 3.x +- 2022-06-28: build error fix: libbson requires out-of-date language constructs +- 2022-06-27: OpenSSL: fix depreacted API issues for OpenSSL 3.x - OpenSSL error strings are loaded automatically now - Debug Callback has changed - See for more: From a1f88f6fdb626faf7281d582b3fea552bb9a7c36 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Jun 2022 09:21:55 +0200 Subject: [PATCH 100/134] CI: modernize platform - remove EOL platforms and add current ones - use github multi-build config files again (now we can restart failed runs instead of all) --- .github/workflows/ci_compile.yml | 20 ++++-- .../{run_ubuntu_16.yml => run_checks.yml} | 61 ++++++----------- .../{run_fedora_31.yml => run_fedora_35.yml} | 4 +- .github/workflows/run_ubuntu_20_codecov.yml | 66 +++++++++++++++++++ ...ian_10.yml => run_ubuntu_22.yml.DELETE_ME} | 25 ++++--- 5 files changed, 119 insertions(+), 57 deletions(-) rename .github/workflows/{run_ubuntu_16.yml => run_checks.yml} (54%) rename .github/workflows/{run_fedora_31.yml => run_fedora_35.yml} (98%) create mode 100644 .github/workflows/run_ubuntu_20_codecov.yml rename .github/workflows/{run_debian_10.yml => run_ubuntu_22.yml.DELETE_ME} (87%) diff --git a/.github/workflows/ci_compile.yml b/.github/workflows/ci_compile.yml index 5c25ae391f..f3a9f25e92 100644 --- a/.github/workflows/ci_compile.yml +++ b/.github/workflows/ci_compile.yml @@ -35,7 +35,7 @@ jobs: # When set to true, cancel all in-progress jobs if any matrix job fails. fail-fast: false matrix: - config: [clang8, clang9, clang10, gcc10, gcc9, gcc8] + config: [clang9, clang10, clang-14, gcc-11, gcc10, gcc9, gcc8] steps: - name: git checkout project @@ -45,29 +45,35 @@ jobs: run: | chmod -R go+rw . export RSYSLOG_CONTAINER_UID="" # use default - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' export CFLAGS='-g' case "${{ matrix.config }}" in - 'clang8') - export CC='clang-8' - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' - ;; 'clang9') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' export CC='clang-9' ;; 'clang10') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' export CC='clang-10' ;; + 'clang14') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CC='clang-14' + ;; 'gcc8') - export CC='gcc-8' export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' + export CC='gcc-8' ;; 'gcc9') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' export CC='gcc-9' ;; 'gcc10') export CC='gcc-10' ;; + 'gcc11') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CC='gcc-11' + ;; esac devtools/devcontainer.sh --rm devtools/run-configure.sh devtools/devcontainer.sh --rm make -j20 diff --git a/.github/workflows/run_ubuntu_16.yml b/.github/workflows/run_checks.yml similarity index 54% rename from .github/workflows/run_ubuntu_16.yml rename to .github/workflows/run_checks.yml index 0f40f715bd..9861cd7e06 100644 --- a/.github/workflows/run_ubuntu_16.yml +++ b/.github/workflows/run_checks.yml @@ -1,4 +1,4 @@ -# Copyright 2020 Rainer Gerhards and Others +# Copyright 2022 Rainer Gerhards and Others # # https://github.com/rsyslog/rsyslog-pkg-ubuntu # @@ -22,20 +22,20 @@ --- -name: check ubuntu 16 +name: check on: pull_request: jobs: CI: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 timeout-minutes: 50 strategy: # When set to true, cancel all in-progress jobs if any matrix job fails. fail-fast: false matrix: - config: [ubuntu_16] + config: [centos_7, debian_10, debian_11, ubuntu_22] steps: - name: git checkout project @@ -58,50 +58,31 @@ jobs: export RSYSLOG_CONFIGURE_OPTIONS_EXTRA='--disable-elasticsearch-tests --disable-kafka-tests --disable-snmp-tests' export CI_VALGRIND_SUPPRESSIONS='centos7.supp' ;; - 'centos_8') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:8' - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA='--disable-elasticsearch-tests --disable-kafka-tests --disable-snmp-tests' - ;; 'debian_10') export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_debian:10' export CI_VALGRIND_SUPPRESSIONS='centos7.supp' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests \ --without-valgrind-testbench" ;; - 'suse_tumbleweed') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_suse:tumbleweed' - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests" - ;; - 'ubuntu_16') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:16.04' + 'debian_11') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_debian:11' + export CI_VALGRIND_SUPPRESSIONS='centos7.supp' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests \ - --disable-omrabbitmq" - ;; - 'ubuntu_18_san') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' - export CC='clang' - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --enable-imfile-tests\ - --disable-libfaketime --without-valgrind-testbench --disable-valgrind" - export CFLAGS="-fstack-protector -D_FORTIFY_SOURCE=2 \ - -fsanitize=address,undefined,nullability,unsigned-integer-overflow \ - -fno-sanitize-recover=undefined,nullability,unsigned-integer-overflow \ - -g -O3 -fno-omit-frame-pointer -fno-color-diagnostics" - export LSAN_OPTIONS='detect_leaks=0' - export UBSAN_OPTIONS='print_stacktrace=1' + --without-valgrind-testbench" ;; - 'ubuntu_18_tsan') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' - export CC='clang-8' - export CFLAGS="-g -fstack-protector -D_FORTIFY_SOURCE=2 -fsanitize=thread \ - -O0 -fno-omit-frame-pointer -fno-color-diagnostics" - export CI_SANITIZE_BLACKLIST='tests/tsan.supp' - export TSAN_OPTIONS='halt_on_error=1' - export ABORT_ALL_ON_TEST_FAIL='YES' - # impstats has known and OK races - # mmpstrucdata TEMPORARILY disabled because of a threading hang on shutdown - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --enable-imfile-tests \ - --disable-impstats --disable-kafka-tests --disable-mmpstrucdata \ - --disable-libfaketime --without-valgrind-testbench --disable-valgrind" + 'ubuntu_22') + chmod -R go+rw . + export RSYSLOG_CONTAINER_UID="" # use default + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' + export CFLAGS='-g' + export CC='gcc' + export USE_AUTO_DEBUG='off' + export CI_MAKE_OPT='-j20' + export CI_MAKE_CHECK_OPT='-j8' + export CI_CHECK_CMD='check' + export CI_VALGRIND_SUPPRESSIONS="ubuntu20.04.supp" # they are still valid + devtools/devcontainer.sh --rm devtools/run-ci.sh ;; esac devtools/devcontainer.sh --rm devtools/run-ci.sh diff --git a/.github/workflows/run_fedora_31.yml b/.github/workflows/run_fedora_35.yml similarity index 98% rename from .github/workflows/run_fedora_31.yml rename to .github/workflows/run_fedora_35.yml index 0553ba5276..9f4a16e7d2 100644 --- a/.github/workflows/run_fedora_31.yml +++ b/.github/workflows/run_fedora_35.yml @@ -22,7 +22,7 @@ --- -name: check fedora 31 +name: check fedora 35 on: pull_request: @@ -47,7 +47,7 @@ jobs: export CI_MAKE_OPT='-j20' export CI_MAKE_CHECK_OPT='-j8' export CI_CHECK_CMD='check' - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:31' + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:35' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ --disable-kafka-tests --enable-debug" devtools/devcontainer.sh --rm devtools/run-ci.sh diff --git a/.github/workflows/run_ubuntu_20_codecov.yml b/.github/workflows/run_ubuntu_20_codecov.yml new file mode 100644 index 0000000000..a9afd189b5 --- /dev/null +++ b/.github/workflows/run_ubuntu_20_codecov.yml @@ -0,0 +1,66 @@ +# Copyright 2022 Rainer Gerhards and Others +# +# https://github.com/rsyslog/rsyslog-pkg-ubuntu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# References: +# +# https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options +# https://github.com/settings/notifications +# https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog + + +--- +name: ubuntu 20 codecov + +on: + pull_request: + push: + branches: + - master + +jobs: + check_run: + runs-on: ubuntu-18.04 + timeout-minutes: 50 + + steps: + - name: git checkout project + uses: actions/checkout@v1 + + - name: run container CI pipeline + run: | + chmod -R go+rw . + export RSYSLOG_CONTAINER_UID="" # use default + export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' + export CFLAGS='-g -O0 -coverage' + export LDFLAGS='-lgcov' + export CC='gcc' + export USE_AUTO_DEBUG='off' + export CI_MAKE_OPT='-j20' + export CI_MAKE_CHECK_OPT='-j8' + export CI_CHECK_CMD='check' + export CI_CODECOV_TOKEN='9958eda4-50ae-4f2c-b6b6-567d3e6a2e81' + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' + export ABORT_ALL_ON_TEST_FAIL='NO' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-kafka-tests --disable-imkafka \ + --disable-omkafka --enable-debug --disable-elasticsearch \ + --disable-elasticsearch-tests" + devtools/devcontainer.sh --rm devtools/run-ci.sh + + - name: show error logs (if we errored) + if: ${{ failure() || cancelled() }} + run: | + devtools/gather-check-logs.sh + cat failed-tests.log diff --git a/.github/workflows/run_debian_10.yml b/.github/workflows/run_ubuntu_22.yml.DELETE_ME similarity index 87% rename from .github/workflows/run_debian_10.yml rename to .github/workflows/run_ubuntu_22.yml.DELETE_ME index ef4bb0977b..4814c1a782 100644 --- a/.github/workflows/run_debian_10.yml +++ b/.github/workflows/run_ubuntu_22.yml.DELETE_ME @@ -1,4 +1,4 @@ -# Copyright 2020 Rainer Gerhards and Others +# Copyright 2022 Rainer Gerhards and Others # # https://github.com/rsyslog/rsyslog-pkg-ubuntu # @@ -22,7 +22,7 @@ --- -name: check +name: check ubuntu 22 on: pull_request: @@ -35,7 +35,7 @@ jobs: # When set to true, cancel all in-progress jobs if any matrix job fails. fail-fast: false matrix: - config: [debian_10] + config: [ubuntu_22] steps: - name: git checkout project @@ -72,11 +72,6 @@ jobs: export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_suse:tumbleweed' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests" ;; - 'ubuntu_16') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:16.04' - export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests \ - --disable-omrabbitmq" - ;; 'ubuntu_18_san') export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' export CC='clang' @@ -103,6 +98,20 @@ jobs: --disable-impstats --disable-kafka-tests --disable-mmpstrucdata \ --disable-libfaketime --without-valgrind-testbench --disable-valgrind" ;; + 'ubuntu_22') + chmod -R go+rw . + export RSYSLOG_CONTAINER_UID="" # use default + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' + export CFLAGS='-g' + export CC='gcc' + export USE_AUTO_DEBUG='off' + export CI_MAKE_OPT='-j20' + export CI_MAKE_CHECK_OPT='-j8' + export CI_CHECK_CMD='check' + export CI_VALGRIND_SUPPRESSIONS="ubuntu20.04.supp" # they are still valid + devtools/devcontainer.sh --rm devtools/run-ci.sh + ;; esac devtools/devcontainer.sh --rm devtools/run-ci.sh From 0869d725f297f50830ae4a026902d6aa35d8f8bd Mon Sep 17 00:00:00 2001 From: frikilax Date: Tue, 19 Apr 2022 09:56:06 +0200 Subject: [PATCH 101/134] MMDBLOOKUP::FIXED:: Don't crash Rsyslog on mmdb file errors --- plugins/mmdblookup/mmdblookup.c | 39 +++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/plugins/mmdblookup/mmdblookup.c b/plugins/mmdblookup/mmdblookup.c index affbc73578..f9f3c73340 100644 --- a/plugins/mmdblookup/mmdblookup.c +++ b/plugins/mmdblookup/mmdblookup.c @@ -68,6 +68,7 @@ typedef struct wrkrInstanceData { instanceData *pData; MMDB_s mmdb; pthread_mutex_t mmdbMutex; + sbool mmdb_is_open; } wrkrInstanceData_t; struct modConfData_s { @@ -122,15 +123,30 @@ int open_mmdb(const char *file, MMDB_s *mmdb) { dbgprintf(" IO error: %s\n", strerror(errno)); } LogError(0, RS_RET_SUSPENDED, "maxminddb error: cannot open database file"); + return RS_RET_SUSPENDED; } - return MMDB_SUCCESS != status; + return RS_RET_OK; } void close_mmdb(MMDB_s *mmdb) { MMDB_close(mmdb); } +static rsRetVal wrkr_reopen_mmdb(wrkrInstanceData_t *pWrkrData) { + DEFiRet; + pthread_mutex_lock(&pWrkrData->mmdbMutex); + LogMsg(0, NO_ERRCODE, LOG_INFO, "mmdblookup: reopening MMDB file"); + if (pWrkrData->mmdb_is_open) close_mmdb(&pWrkrData->mmdb); + pWrkrData->mmdb_is_open = 0; + CHKiRet(open_mmdb(pWrkrData->pData->pszMmdbFile, &pWrkrData->mmdb)); + pWrkrData->mmdb_is_open = 1; + +finalize_it: + pthread_mutex_unlock(&pWrkrData->mmdbMutex); + RETiRet; +} + BEGINbeginCnfLoad CODESTARTbeginCnfLoad loadModConf = pModConf; @@ -163,6 +179,7 @@ ENDcreateInstance BEGINcreateWrkrInstance CODESTARTcreateWrkrInstance CHKiRet(open_mmdb(pData->pszMmdbFile, &pWrkrData->mmdb)); + pWrkrData->mmdb_is_open = 1; CHKiConcCtrl(pthread_mutex_init(&pWrkrData->mmdbMutex, NULL)); finalize_it: ENDcreateWrkrInstance @@ -190,7 +207,8 @@ ENDfreeInstance BEGINfreeWrkrInstance CODESTARTfreeWrkrInstance - close_mmdb(&pWrkrData->mmdb); + if (pWrkrData->mmdb_is_open) close_mmdb(&pWrkrData->mmdb); + pWrkrData->mmdb_is_open = 0; pthread_mutex_destroy(&pWrkrData->mmdbMutex); ENDfreeWrkrInstance @@ -312,6 +330,7 @@ ENDdbgPrintInstInfo BEGINtryResume CODESTARTtryResume + iRet = wrkr_reopen_mmdb(pWrkrData); ENDtryResume @@ -364,6 +383,11 @@ BEGINdoAction_NoStrings json_object *total_json = NULL; MMDB_entry_data_list_s *entry_data_list = NULL; CODESTARTdoAction + /* ensure file is open before beginning */ + if (!pWrkrData->mmdb_is_open) { + CHKiRet(wrkr_reopen_mmdb(pWrkrData)); + } + /* key is given, so get the property json */ msgPropDescr_t pProp; msgPropDescrFill(&pProp, (uchar*)pData->pszKey, strlen(pData->pszKey)); @@ -390,7 +414,9 @@ CODESTARTdoAction } if (MMDB_SUCCESS != mmdb_err) { dbgprintf("Got an error from the maxminddb library: %s\n", MMDB_strerror(mmdb_err)); - ABORT_FINALIZE(RS_RET_OK); + close_mmdb(&pWrkrData->mmdb); + pWrkrData->mmdb_is_open = 0; + ABORT_FINALIZE(RS_RET_IO_ERROR); } if (!result.found_entry) { dbgprintf("No entry found in database for '%s'\n", pszValue); @@ -458,12 +484,7 @@ BEGINdoHUPWrkr CODESTARTdoHUPWrkr dbgprintf("mmdblookup: HUP received\n"); if (pWrkrData->pData->reloadOnHup) { - // a mutex is needed, as it's the main thread that runs this handler - pthread_mutex_lock(&pWrkrData->mmdbMutex); - LogMsg(0, NO_ERRCODE, LOG_INFO, "mmdblookup: reloading MMDB file"); - close_mmdb(&pWrkrData->mmdb); - iRet = open_mmdb(pWrkrData->pData->pszMmdbFile, &pWrkrData->mmdb); - pthread_mutex_unlock(&pWrkrData->mmdbMutex); + iRet = wrkr_reopen_mmdb(pWrkrData); } ENDdoHUPWrkr From 829a6f6a64849c1cfbc73f71e959b550df4a4c25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Jun 2022 10:18:55 +0200 Subject: [PATCH 102/134] CI: improve use of github action matrix feature; optimization Matrix feature was initially barely usable because github did not permit to restart just failed jobs. This has changed. So we now migrate back to the use of matrix feature where it is useful. We also reduce the number of plain compile tests to newest and oldest compiler version only. This saves CI ressources. The important other ones (distro default!) are used during other CI tests anyways. We also fix some minor coding errors detected by new compiler builds. None of them seems to be related to any real issues. --- .github/workflows/ci_compile.yml | 16 +-- ...entos_7.yml => run_centos_7.yml.TO_DELETE} | 0 ...entos_8.yml => run_centos_8.yml.TO_DELETE} | 0 .github/workflows/run_checks.yml | 118 ++++++++++++++++-- ...ch.yml => run_elasticsearch.yml.TO_DELETE} | 0 ...dora_34.yml => run_fedora_34.yml.TO_DELTE} | 0 ...dora_35.yml => run_fedora_35.yml.TO_DELTE} | 0 ...ov.yml => run_kafka_codecov.yml.TO_DELETE} | 2 +- ...eweed.yml => run_tumbleweed.yml.TO_DELETE} | 0 ...ml => run_ubuntu_18_codecov.yml.TO_DELETE} | 0 ... => run_ubuntu_18_gtls_only.yml.TO_DELETE} | 0 ...ntu_20.yml => run_ubuntu_20.yml.TO_DELETE} | 0 ...ml => run_ubuntu_20_codecov.yml.TO_DELETE} | 0 ... => run_ubuntu_20_distcheck.yml.TO_DELETE} | 0 ...an.yml => run_ubuntu_20_san.yml.TO_DELETE} | 0 ...n.yml => run_ubuntu_20_tsan.yml.TO_DELETE} | 0 plugins/omlibdbi/omlibdbi.c | 2 +- runtime/conf.c | 8 +- runtime/rsyslog.h | 10 +- tests/CI/ubuntu22.04.supp | 84 +++++++++++++ 20 files changed, 212 insertions(+), 28 deletions(-) rename .github/workflows/{run_centos_7.yml => run_centos_7.yml.TO_DELETE} (100%) rename .github/workflows/{run_centos_8.yml => run_centos_8.yml.TO_DELETE} (100%) rename .github/workflows/{run_elasticsearch.yml => run_elasticsearch.yml.TO_DELETE} (100%) rename .github/workflows/{run_fedora_34.yml => run_fedora_34.yml.TO_DELTE} (100%) rename .github/workflows/{run_fedora_35.yml => run_fedora_35.yml.TO_DELTE} (100%) rename .github/workflows/{run_kafka_codecov.yml => run_kafka_codecov.yml.TO_DELETE} (99%) rename .github/workflows/{run_tumbleweed.yml => run_tumbleweed.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_18_codecov.yml => run_ubuntu_18_codecov.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_18_gtls_only.yml => run_ubuntu_18_gtls_only.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_20.yml => run_ubuntu_20.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_20_codecov.yml => run_ubuntu_20_codecov.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_20_distcheck.yml => run_ubuntu_20_distcheck.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_20_san.yml => run_ubuntu_20_san.yml.TO_DELETE} (100%) rename .github/workflows/{run_ubuntu_20_tsan.yml => run_ubuntu_20_tsan.yml.TO_DELETE} (100%) create mode 100644 tests/CI/ubuntu22.04.supp diff --git a/.github/workflows/ci_compile.yml b/.github/workflows/ci_compile.yml index f3a9f25e92..b7e9532a51 100644 --- a/.github/workflows/ci_compile.yml +++ b/.github/workflows/ci_compile.yml @@ -35,7 +35,10 @@ jobs: # When set to true, cancel all in-progress jobs if any matrix job fails. fail-fast: false matrix: - config: [clang9, clang10, clang-14, gcc-11, gcc10, gcc9, gcc8] + # note: we compile only with oldest and newest support compiler to + # save ressources. Other important ones are used during the rest of + # the check runs (most importantly the distro-default ones). + config: [clang9, clang-14, gcc-11, gcc8] steps: - name: git checkout project @@ -51,10 +54,6 @@ jobs: export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' export CC='clang-9' ;; - 'clang10') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' - export CC='clang-10' - ;; 'clang14') export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' export CC='clang-14' @@ -63,13 +62,6 @@ jobs: export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' export CC='gcc-8' ;; - 'gcc9') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' - export CC='gcc-9' - ;; - 'gcc10') - export CC='gcc-10' - ;; 'gcc11') export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' export CC='gcc-11' diff --git a/.github/workflows/run_centos_7.yml b/.github/workflows/run_centos_7.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_centos_7.yml rename to .github/workflows/run_centos_7.yml.TO_DELETE diff --git a/.github/workflows/run_centos_8.yml b/.github/workflows/run_centos_8.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_centos_8.yml rename to .github/workflows/run_centos_8.yml.TO_DELETE diff --git a/.github/workflows/run_checks.yml b/.github/workflows/run_checks.yml index 9861cd7e06..6a1d6156bd 100644 --- a/.github/workflows/run_checks.yml +++ b/.github/workflows/run_checks.yml @@ -35,7 +35,11 @@ jobs: # When set to true, cancel all in-progress jobs if any matrix job fails. fail-fast: false matrix: - config: [centos_7, debian_10, debian_11, ubuntu_22] + config: [centos_7, centos_8, debian_10, debian_11, tumbleweed + fedora_34, fedora_35, + ubuntu_18, ubuntu_20, ubuntu_22, + ubuntu_22_san, ubuntu_22_tsan, ubuntu_22_codecov, + kafka_codecov, elasticsearch] steps: - name: git checkout project @@ -50,7 +54,7 @@ jobs: export CC='gcc' export USE_AUTO_DEBUG='off' export CI_MAKE_OPT='-j20' - export CI_MAKE_CHECK_OPT='-j8' + export CI_MAKE_CHECK_OPT='-j6' export CI_CHECK_CMD='check' case "${{ matrix.config }}" in 'centos_7') @@ -58,6 +62,10 @@ jobs: export RSYSLOG_CONFIGURE_OPTIONS_EXTRA='--disable-elasticsearch-tests --disable-kafka-tests --disable-snmp-tests' export CI_VALGRIND_SUPPRESSIONS='centos7.supp' ;; + 'centos_8') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:8' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA='--disable-elasticsearch-tests --disable-kafka-tests --disable-snmp-tests' + ;; 'debian_10') export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_debian:10' export CI_VALGRIND_SUPPRESSIONS='centos7.supp' @@ -70,19 +78,105 @@ jobs: export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests \ --without-valgrind-testbench" ;; + 'fedora_34') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:34' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ + --disable-kafka-tests --enable-debug" + ;; + 'fedora_35') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:35' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ + --disable-kafka-tests --enable-debug" + ;; + 'ubuntu_18') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:18.04' + ;; + 'ubuntu_20') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:20.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu20.04.supp" + ;; + 'tumbleweed') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_suse:tumbleweed' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests" + ;; 'ubuntu_22') - chmod -R go+rw . - export RSYSLOG_CONTAINER_UID="" # use default export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' - export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' - export CFLAGS='-g' - export CC='gcc' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + ;; + 'ubuntu_22_distcheck') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + export CI_CHECK_CMD='distcheck' + export ABORT_ALL_ON_TEST_FAIL='YES' + # DO WE NEED THIS? export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-omkafka --disable-imkafka --disable-kafka-tests" + ;; + 'ubuntu_22_san') + export CI_SANITIZE_BLACKLIST='tests/asan.supp' + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CC='clang' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ + --disable-libfaketime --without-valgrind-testbench --disable-valgrind \ + --disable-kafka-tests" + export CFLAGS="-fstack-protector -D_FORTIFY_SOURCE=2 \ + -fsanitize=address,undefined,nullability,unsigned-integer-overflow \ + -fno-sanitize-recover=undefined,nullability,unsigned-integer-overflow \ + -g -O3 -fno-omit-frame-pointer -fno-color-diagnostics" + export LSAN_OPTIONS='detect_leaks=0' + export UBSAN_OPTIONS='print_stacktrace=1' + ;; + 'ubuntu_22_tsan') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + export CI_SANITIZE_BLACKLIST='tests/tsan.supp' + export CC='clang' + # impstats has known and OK races + # mmpstrucdata TEMPORARILY disabled because of a threading hang on shutdown + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --enable-imfile-tests \ + --disable-impstats --disable-kafka-tests --disable-mmpstrucdata \ + --disable-clickhouse --disable-clickhouse-tests --disable-kafka-tests \ + --disable-libfaketime --without-valgrind-testbench --disable-valgrind" + export CFLAGS="-g -fstack-protector -D_FORTIFY_SOURCE=2 -fsanitize=thread \ + -O0 -fno-omit-frame-pointer -fno-color-diagnostics" + # note: we need pathes in container, thus /rsyslog vs. $(pwd) in TSAN_OPTIONS + export TSAN_OPTIONS="halt_on_error=1 suppressions=/rsyslog/tests/tsan-rt.supp" + export ABORT_ALL_ON_TEST_FAIL='YES' + ;; + 'ubuntu_22_codecov') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + export CFLAGS='-g -O0 -coverage' + export LDFLAGS='-lgcov' export USE_AUTO_DEBUG='off' - export CI_MAKE_OPT='-j20' - export CI_MAKE_CHECK_OPT='-j8' - export CI_CHECK_CMD='check' - export CI_VALGRIND_SUPPRESSIONS="ubuntu20.04.supp" # they are still valid - devtools/devcontainer.sh --rm devtools/run-ci.sh + export CI_CODECOV_TOKEN='9958eda4-50ae-4f2c-b6b6-567d3e6a2e81' + export ABORT_ALL_ON_TEST_FAIL='NO' + export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-kafka-tests --disable-imkafka \ + --disable-omkafka --enable-debug --disable-elasticsearch \ + --disable-elasticsearch-tests" + ;; + 'kafka_codecov') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + export CFLAGS='-g -O0 -coverage' + export LDFLAGS='-lgcov' + export CI_CODECOV_TOKEN='9958eda4-50ae-4f2c-b6b6-567d3e6a2e81' + export ABORT_ALL_ON_TEST_FAIL='NO' + # Note: we completely override the container configure options here! + export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE="--enable-testbench --enable-omstdout \ + --enable-imdiag --disable-impstats --enable-imfile --disable-imfile-tests \ + --disable-fmhttp --enable-valgrind --enable-valgrind-testbench \ + --disable-helgrind --disable-default-tests --enable-kafka-tests \ + --enable-omkafka --enable-gnutls --disable-gnutls-tests --enable-imkafka" + ;; + 'elasticsearch') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_ubuntu:22.04' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" + export CI_CODECOV_TOKEN='9958eda4-50ae-4f2c-b6b6-567d3e6a2e81' + export CI_MAKE_CHECK_OPT='-j1' + export ABORT_ALL_ON_TEST_FAIL='NO' + export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE="--enable-testbench --enable-omstdout \ + --enable-imdiag --enable-impstats --enable-imfile --disable-imfile-tests \ + --disable-fmhttp --enable-valgrind --disable-default-tests \ + --enable-elasticsearch-tests --enable-elasticsearch" ;; esac devtools/devcontainer.sh --rm devtools/run-ci.sh diff --git a/.github/workflows/run_elasticsearch.yml b/.github/workflows/run_elasticsearch.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_elasticsearch.yml rename to .github/workflows/run_elasticsearch.yml.TO_DELETE diff --git a/.github/workflows/run_fedora_34.yml b/.github/workflows/run_fedora_34.yml.TO_DELTE similarity index 100% rename from .github/workflows/run_fedora_34.yml rename to .github/workflows/run_fedora_34.yml.TO_DELTE diff --git a/.github/workflows/run_fedora_35.yml b/.github/workflows/run_fedora_35.yml.TO_DELTE similarity index 100% rename from .github/workflows/run_fedora_35.yml rename to .github/workflows/run_fedora_35.yml.TO_DELTE diff --git a/.github/workflows/run_kafka_codecov.yml b/.github/workflows/run_kafka_codecov.yml.TO_DELETE similarity index 99% rename from .github/workflows/run_kafka_codecov.yml rename to .github/workflows/run_kafka_codecov.yml.TO_DELETE index add8560a9f..b4f922027b 100644 --- a/.github/workflows/run_kafka_codecov.yml +++ b/.github/workflows/run_kafka_codecov.yml.TO_DELETE @@ -52,7 +52,7 @@ jobs: export CI_MAKE_CHECK_OPT='-j8' export CI_CHECK_CMD='check' export CI_CODECOV_TOKEN='9958eda4-50ae-4f2c-b6b6-567d3e6a2e81' - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:30' + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:35' export ABORT_ALL_ON_TEST_FAIL='NO' # Note: we completely override the container configure options here! export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE="--enable-testbench --enable-omstdout \ diff --git a/.github/workflows/run_tumbleweed.yml b/.github/workflows/run_tumbleweed.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_tumbleweed.yml rename to .github/workflows/run_tumbleweed.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_18_codecov.yml b/.github/workflows/run_ubuntu_18_codecov.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_18_codecov.yml rename to .github/workflows/run_ubuntu_18_codecov.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_18_gtls_only.yml b/.github/workflows/run_ubuntu_18_gtls_only.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_18_gtls_only.yml rename to .github/workflows/run_ubuntu_18_gtls_only.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_20.yml b/.github/workflows/run_ubuntu_20.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_20.yml rename to .github/workflows/run_ubuntu_20.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_20_codecov.yml b/.github/workflows/run_ubuntu_20_codecov.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_20_codecov.yml rename to .github/workflows/run_ubuntu_20_codecov.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_20_distcheck.yml b/.github/workflows/run_ubuntu_20_distcheck.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_20_distcheck.yml rename to .github/workflows/run_ubuntu_20_distcheck.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_20_san.yml b/.github/workflows/run_ubuntu_20_san.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_20_san.yml rename to .github/workflows/run_ubuntu_20_san.yml.TO_DELETE diff --git a/.github/workflows/run_ubuntu_20_tsan.yml b/.github/workflows/run_ubuntu_20_tsan.yml.TO_DELETE similarity index 100% rename from .github/workflows/run_ubuntu_20_tsan.yml rename to .github/workflows/run_ubuntu_20_tsan.yml.TO_DELETE diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 4ad8fcbd0b..c3f3810b46 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -89,7 +89,7 @@ typedef struct configSettings_s { uchar *dbName; /* database to use */ } configSettings_t; static configSettings_t cs; -uchar *pszFileDfltTplName; /* name of the default template to use */ +static uchar *pszFileDfltTplName; /* name of the default template to use */ struct modConfData_s { rsconf_t *pConf; /* our overall config object */ diff --git a/runtime/conf.c b/runtime/conf.c index 50b1b71ee0..3e56301ef1 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -165,7 +165,13 @@ doNameLine(uchar **pp, void* pVal) p = *pp; assert(p != NULL); - eDir = (enum eDirective) pVal; /* this time, it actually is NOT a pointer! */ + PRAGMA_DIAGNOSTIC_PUSH + PRAGMA_IGNORE_Wvoid_pointer_to_enum_cast; + /* this time, pVal actually is NOT a pointer! It is save to case, as + * the enum was written to it, so there can be no loss of bits (ptr is larger). + */ + eDir = (enum eDirective) pVal; + PRAGMA_DIAGNOSTIC_POP if(getSubString(&p, szName, sizeof(szName), ',') != 0) { LogError(0, RS_RET_NOT_FOUND, "Invalid config line: could not extract name - line ignored"); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 4be62905bb..3f7cfc62cb 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -89,7 +89,14 @@ #define PRAGMA_INGORE_Wswitch_enum _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"") #define PRAGMA_IGNORE_Wempty_body _Pragma("GCC diagnostic ignored \"-Wempty-body\"") #define PRAGMA_IGNORE_Wstrict_prototypes _Pragma("GCC diagnostic ignored \"-Wstrict-prototypes\"") - #define PRAGMA_IGNORE_Wold_style_definition _Pragma("GCC diagnostic ignored \"-Wold-style-definition\"") + #define PRAGMA_IGNORE_Wold_style_definition \ + _Pragma("GCC diagnostic ignored \"-Wold-style-definition\"") + #if defined(__clang_major__) && __clang_major__ >= 14 + #define PRAGMA_IGNORE_Wvoid_pointer_to_enum_cast \ + _Pragma("GCC diagnostic ignored \"-Wvoid-pointer-to-enum-cast\"") + #else + #define PRAGMA_IGNORE_Wvoid_pointer_to_enum_cast + #endif #define PRAGMA_IGNORE_Wsign_compare _Pragma("GCC diagnostic ignored \"-Wsign-compare\"") #define PRAGMA_IGNORE_Wpragmas _Pragma("GCC diagnostic ignored \"-Wpragmas\"") #define PRAGMA_IGNORE_Wmissing_noreturn _Pragma("GCC diagnostic ignored \"-Wmissing-noreturn\"") @@ -114,6 +121,7 @@ #define PRAGMA_IGNORE_Wempty_body #define PRAGMA_IGNORE_Wstrict_prototypes #define PRAGMA_IGNORE_Wold_style_definition + #define PRAGMA_IGNORE_Wvoid_pointer_to_enum_cast #define PRAGMA_IGNORE_Wdeprecated_declarations #define PRAGMA_IGNORE_Wexpansion_to_defined #define PRAGMA_IGNORE_Wunknown_attribute diff --git a/tests/CI/ubuntu22.04.supp b/tests/CI/ubuntu22.04.supp new file mode 100644 index 0000000000..0c35882014 --- /dev/null +++ b/tests/CI/ubuntu22.04.supp @@ -0,0 +1,84 @@ +{ + + Memcheck:Param + pselect6(sig) + fun:pselect64_syscall + fun:pselect + fun:wait_timeout + fun:mainloop + fun:main +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:malloc + fun:_dl_find_object_update + fun:dl_open_worker_begin + fun:_dl_catch_exception + ... + fun:__libc_unwind_link_get +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:malloc + fun:_dl_find_object_update + fun:dl_open_worker_begin + fun:_dl_catch_exception + fun:dl_open_worker + fun:_dl_catch_exception + fun:_dl_open + fun:do_dlopen + fun:_dl_catch_exception + fun:_dl_catch_error + fun:dlerror_run + fun:__libc_dlopen_mode + fun:__libc_unwind_link_get + fun:__libc_unwind_link_get +} +{ + librdkafka issue + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + fun:checkInstance + fun:activateCnf + fun:tellModulesActivateConfig + fun:activate + fun:initAll +} + +{ + librdkafka issue + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + ... + obj:* + fun:checkInstance + fun:activateCnf + fun:tellModulesActivateConfig + fun:activate + fun:initAll + fun:main +} + +{ + invalid atexit call - seems to stem back to libcurl + Helgrind:Misc + ... + fun:_dl_fini + fun:__run_exit_handlers + fun:exit + fun:(below main) +} From d27e6a9c1978be85a8718490a53d6eb3b1b084c7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Jul 2022 10:41:25 +0200 Subject: [PATCH 103/134] maintain ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index fee04f7fdb..8724596be3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-07-04: mmdblookup bugfix: Don't crash Rsyslog on mmdb file errors + Thanks to Théo Bertin (frikilax) for the patch. - 2022-06-28: build error fix: libbson requires out-of-date language constructs - 2022-06-27: OpenSSL: fix depreacted API issues for OpenSSL 3.x - OpenSSL error strings are loaded automatically now From 393ea2405f0a96ef23a1c86a5c2da09fed7fb5d9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Jul 2022 13:22:07 +0200 Subject: [PATCH 104/134] CI: further modernizations - drop Fedora 34, add Fedora 36 --- .github/workflows/run_checks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_checks.yml b/.github/workflows/run_checks.yml index 6a1d6156bd..cfb1b6db32 100644 --- a/.github/workflows/run_checks.yml +++ b/.github/workflows/run_checks.yml @@ -36,7 +36,7 @@ jobs: fail-fast: false matrix: config: [centos_7, centos_8, debian_10, debian_11, tumbleweed - fedora_34, fedora_35, + fedora_35, fedora_36, ubuntu_18, ubuntu_20, ubuntu_22, ubuntu_22_san, ubuntu_22_tsan, ubuntu_22_codecov, kafka_codecov, elasticsearch] @@ -78,13 +78,13 @@ jobs: export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests --disable-kafka-tests \ --without-valgrind-testbench" ;; - 'fedora_34') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:34' + 'fedora_35') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:35' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ --disable-kafka-tests --enable-debug" ;; - 'fedora_35') - export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:35' + 'fedora_36') + export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_fedora:36' export RSYSLOG_CONFIGURE_OPTIONS_EXTRA="--disable-elasticsearch-tests \ --disable-kafka-tests --enable-debug" ;; From 6ef5e826d730cb420c46bf5f796855b65524dae4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 5 Jul 2022 12:10:38 +0200 Subject: [PATCH 105/134] CI: suppress TSAN exception for known-good case We do not use a mutex an epd, but we do always access it in pure sequence. Adding a mutex just to cover this "cosmetic" would result in uncesseary performance penalty. --- plugins/imptcp/imptcp.c | 18 ++++++++++++++---- tests/tsan-rt.supp | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 1ec3946fa8..2d3f4b1113 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -397,18 +397,28 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a static rsRetVal addLstn(ptcpsrv_t *pSrv, int sock, int isIPv6); +/* function to suppress TSAN known-good case + * We do not use a mutex an epd, but we do always access it in + * pure sequence. Adding a mutex just to cover this "cosmetic" + * would result in uncesseary performance penalty. + */ +static void ATTR_NONNULL() +imptcp_destruct_epd(ptcpsess_t *const pSess) { + free(pSess->epd); + pSess->epd = NULL; +} + /* some simple constructors/destructors */ -static void -destructSess(ptcpsess_t *pSess) +static void ATTR_NONNULL() +destructSess(ptcpsess_t *const pSess) { + imptcp_destruct_epd(pSess); free(pSess->pMsg_save); free(pSess->pMsg); - free(pSess->epd); prop.Destruct(&pSess->peerName); prop.Destruct(&pSess->peerIP); /* TODO: make these inits compile-time switch depending: */ pSess->pMsg = NULL; - pSess->epd = NULL; free(pSess); } diff --git a/tests/tsan-rt.supp b/tests/tsan-rt.supp index 8e52abd855..591913d746 100644 --- a/tests/tsan-rt.supp +++ b/tests/tsan-rt.supp @@ -1,3 +1,6 @@ +# supressions for LLVM TSAN +# doc: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:^imptcp_destruct_epd$ race:doLogMsg race:setlocale race:doSIGTTIN From 812c495ae3a2143afb480d84b8be02dbf321eb32 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 4 Jul 2022 09:17:07 +0200 Subject: [PATCH 106/134] stream cleanup: move error message to debug log, only This error message is most probably rooted in a kernel problem. At least knowbody knows how it can happen. It's definitely not a rsyslog issue. We also can recover from it for a long time now so there is no reason to irritate users by emitteing this "error" message. --- runtime/stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/stream.c b/runtime/stream.c index 064fe5ded6..94e36ca4a1 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1467,11 +1467,11 @@ doWriteCall(strm_t *pThis, uchar *pBuf, size_t *pLenBuf) const int err = errno; iWritten = 0; /* we have written NO bytes! */ if(err == EBADF) { - LogError(err, RS_RET_IO_ERROR, "file %s: fd %d no longer valid, recovery by " + DBGPRINTF("file %s: errno %d, fd %d no longer valid, recovery by " "reopen; if you see this, consider reporting at " "https://github.com/rsyslog/rsyslog/issues/3404 " "so that we know when it happens. Include output of uname -a. " - "OS error reason", pThis->pszCurrFName, pThis->fd); + "OS error reason", pThis->pszCurrFName, err, pThis->fd); pThis->fd = -1; CHKiRet(doPhysOpen(pThis)); } else { From 8e13d9d718d26b15263ecd53e26fed1a2af3f3e9 Mon Sep 17 00:00:00 2001 From: seuzw930 <76191785+seuzw930@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:47:11 +0800 Subject: [PATCH 107/134] Fix memory leak when globally de-initialize GnuTLS During globally de-initialize GnuTLS, server anon credentials and server DH parameters for anon mode were not null. It resulted in memory leak and this patch fixes this behaviour. --- runtime/nsd_gtls.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 9e65d2943b..d82fceba68 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -1307,6 +1307,8 @@ static rsRetVal gtlsGlblExit(void) { DEFiRet; + gnutls_anon_free_server_credentials(anoncredSrv); + gnutls_dh_params_deinit(dh_params); gnutls_global_deinit(); RETiRet; } From 4d27e62aa77a44a701cc580e87b4bbc3b7041ea9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 9 Jul 2022 10:42:17 +0200 Subject: [PATCH 108/134] maintain ChangeLog --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8724596be3..38dea551da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-07-07: stream cleanup: move error message to debug log, only + This error message is most probably rooted in a kernel problem. At + least knowbody knows how it can happen. It's definitely not a + rsyslog issue. We also can recover from it for a long time now + so there is no reason to irritate users by emitteing this + "error" message. - 2022-07-04: mmdblookup bugfix: Don't crash Rsyslog on mmdb file errors Thanks to Théo Bertin (frikilax) for the patch. - 2022-06-28: build error fix: libbson requires out-of-date language constructs From 36392e745dd82f7c8af3ccbeaad5286bdf7108fd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 13 Jul 2022 12:14:51 +0200 Subject: [PATCH 109/134] CI: add new TSAN suppression This is a cosmetic issue, only seen during debug logging. We actually cannot fix it, as it would alter runtime behaviour too much and would pontentially render debug runs useless. It also causes no harm. --- tests/tsan-rt.supp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tsan-rt.supp b/tests/tsan-rt.supp index 591913d746..37c741d460 100644 --- a/tests/tsan-rt.supp +++ b/tests/tsan-rt.supp @@ -1,5 +1,6 @@ # supressions for LLVM TSAN # doc: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:^bs_arrcmp_glblDbgFiles$ race:^imptcp_destruct_epd$ race:doLogMsg race:setlocale From 90feae08e6eadaa71873ddc482f82b4c32232754 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Jun 2022 15:44:41 +0200 Subject: [PATCH 110/134] omfile: support for zstd compression The zstd library provides better and faster compression than zlib. This patch integrates zstd as a dynamically-loadable functionality. As such, no further dependencies need to be added to the rsyslog base package. Due to the increased performance, usage of zstd is highly recommended for high-volume use cases. This patch also refactor zlib compression in order to unify handling in both compression cases. --- .github/workflows/run_checks.yml | 5 + .github/workflows/run_journal.yml | 4 +- configure.ac | 37 ++++++ runtime/Makefile.am | 11 ++ runtime/stream.c | 138 ++++++++------------- runtime/stream.h | 17 ++- runtime/zlibw.c | 117 +++++++++++++++++- runtime/zlibw.h | 11 +- runtime/zstdw.c | 191 ++++++++++++++++++++++++++++++ runtime/zstdw.h | 43 +++++++ tests/Makefile.am | 11 ++ tests/diag.sh | 9 +- tests/zstd-vg.sh | 4 + tests/zstd.sh | 38 ++++++ tools/omfile.c | 20 +++- 15 files changed, 561 insertions(+), 95 deletions(-) create mode 100644 runtime/zstdw.c create mode 100644 runtime/zstdw.h create mode 100755 tests/zstd-vg.sh create mode 100755 tests/zstd.sh diff --git a/.github/workflows/run_checks.yml b/.github/workflows/run_checks.yml index cfb1b6db32..e20c0c59b4 100644 --- a/.github/workflows/run_checks.yml +++ b/.github/workflows/run_checks.yml @@ -177,6 +177,11 @@ jobs: --enable-imdiag --enable-impstats --enable-imfile --disable-imfile-tests \ --disable-fmhttp --enable-valgrind --disable-default-tests \ --enable-elasticsearch-tests --enable-elasticsearch" + export CI_MAKE_OPT='-j20' + export CI_MAKE_CHECK_OPT='-j8' + export CI_CHECK_CMD='check' + export CI_VALGRIND_SUPPRESSIONS="ubuntu22.04.supp" # they are still valid + devtools/devcontainer.sh --rm devtools/run-ci.sh ;; esac devtools/devcontainer.sh --rm devtools/run-ci.sh diff --git a/.github/workflows/run_journal.yml b/.github/workflows/run_journal.yml index 0c17df139f..a778f73d56 100644 --- a/.github/workflows/run_journal.yml +++ b/.github/workflows/run_journal.yml @@ -53,6 +53,7 @@ jobs: libsystemd-dev \ libtool \ libtool-bin \ + libzstd-dev \ lsof \ make \ net-tools \ @@ -60,7 +61,8 @@ jobs: python-docutils \ software-properties-common \ valgrind \ - wget + wget \ + zstd - name: git checkout project uses: actions/checkout@v1 diff --git a/configure.ac b/configure.ac index 31d9d7af2c..6da9ddbbed 100644 --- a/configure.ac +++ b/configure.ac @@ -1182,6 +1182,42 @@ AM_CONDITIONAL(ENABLE_LIBGCRYPT, test x$enable_libgcrypt = xyes) AC_SUBST(LIBGCRYPT_CFLAGS) AC_SUBST(LIBGCRYPT_LIBS) +# libzstd support +AC_ARG_ENABLE(libzstd, + [AS_HELP_STRING([--enable-libzstd],[Enable log file compression support via libzstd @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_libzstd="yes" ;; + no) enable_libzstd="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-libzstd) ;; + esac], + [enable_libzstd=no] +) +if test "x$enable_libzstd" = "xyes"; then + save_LIBS="$LIBS" + PKG_CHECK_MODULES([ZSTD], zstd >= 1.4.0, [], + [ + LIBS="$LIBS -lzstd" + ZSTD_LIBS="-lzstd" + AC_SUBST(ZSTD_LIBS) + AC_SEARCH_LIBS(ZSTD_compressStream2, zstd, + [AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM( + [[ #include ]], + [[ + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 5); + ]] + )], + [], + [AC_MSG_ERROR([libzstd version must be >= 1.4.0])] + )], + [AC_MSG_ERROR([libzstd not found])] + )] + ) + LIBS="$save_LIBS" +fi +AM_CONDITIONAL(ENABLE_LIBZSTD, test x$enable_libzstd = xyes) + # support for building the rsyslogd runtime AC_ARG_ENABLE(rsyslogrt, @@ -2777,6 +2813,7 @@ echo " Unlimited select() support enabled: $enable_unlimited_select" echo " uuid support enabled: $enable_uuid" echo " Log file signing support via KSI LS12: $enable_ksi_ls12" echo " Log file encryption support: $enable_libgcrypt" +echo " Log file compression via zstd support: $enable_libzstd" echo " anonymization support enabled: $enable_mmanon" echo " message counting support enabled: $enable_mmcount" echo " liblogging-stdlog support enabled: $enable_liblogging_stdlog" diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 0e31a01064..b752a8c0c1 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -244,6 +244,17 @@ if ENABLE_LIBGCRYPT lmcry_gcry_la_LIBADD = libgcry.la $(LIBGCRYPT_LIBS) endif +# +# support library for zstd +# +if ENABLE_LIBZSTD +pkglib_LTLIBRARIES += lmzstdw.la + lmzstdw_la_SOURCES = zstdw.c zstdw.h + lmzstdw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) + lmzstdw_la_LDFLAGS = -module -avoid-version $(ZSTD_LIBS) + lmzstdw_la_LIBADD = -lzstd +endif + # # gssapi support diff --git a/runtime/stream.c b/runtime/stream.c index 94e36ca4a1..9cadcc6bab 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -16,7 +16,7 @@ * it turns out to be problematic. Then, we need to quasi-refcount the number of accesses * to the object. * - * Copyright 2008-2018 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2022 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -62,6 +62,7 @@ #include "unicode-helper.h" #include "module-template.h" #include "errmsg.h" +#include "zstdw.h" #include "cryprov.h" #include "datetime.h" #include "rsconf.h" @@ -77,6 +78,7 @@ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(zlibw) +DEFobjCurrIf(zstdw) /* forward definitions */ static rsRetVal strmFlushInternal(strm_t *pThis, int bFlushZip); @@ -93,6 +95,7 @@ static rsRetVal strmSeekCurrOffs(strm_t *pThis); /* methods */ + /* note: this may return NULL if not line segment is currently set */ // TODO: due to the cstrFinalize() this is not totally clean, albeit for our // current use case it does not hurt -- refactor! rgerhards, 2018-03-27 @@ -1230,6 +1233,7 @@ BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); pThis->tOpenMode = 0600; + pThis->compressionDriver = STRM_COMPRESS_ZIP; pThis->pszSizeLimitCmd = NULL; pThis->prevLineSegment = NULL; pThis->prevMsgSegment = NULL; @@ -1256,18 +1260,27 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) pThis->iBufPtrMax = 0; /* results in immediate read request */ if(pThis->iZipLevel) { /* do we need a zip buf? */ - localRet = objUse(zlibw, LM_ZLIBW_FILENAME); - if(localRet != RS_RET_OK) { - pThis->iZipLevel = 0; - DBGPRINTF("stream was requested with zip mode, but zlibw module unavailable (%d) - using " - "without zip\n", localRet); + if(pThis->compressionDriver == STRM_COMPRESS_ZSTD) { + localRet = objUse(zstdw, LM_ZSTDW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + LogError(0, localRet, "stream was requested with zstd compression mode, " + "but zstdw module unavailable - using without compression\n"); + } } else { - /* we use the same size as the original buf, as we would like - * to make sure we can write out everything with a SINGLE api call! - * We add another 128 bytes to take care of the gzip header and "all eventualities". - */ - CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(pThis->sIOBufSize + 128)); + assert(pThis->compressionDriver == STRM_COMPRESS_ZIP); + localRet = objUse(zlibw, LM_ZLIBW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + LogError(0, localRet, "stream was requested with zip mode, but zlibw " + "module unavailable - using without zip\n"); + } } + /* we use the same size as the original buf, as we would like + * to make sure we can write out everything with a SINGLE api call! + * We add another 128 bytes to take care of the gzip header and "all eventualities". + */ + CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(pThis->sIOBufSize + 128)); } /* if we are set to sync, we must obtain a file handle to the directory for fsync() purposes */ @@ -1354,6 +1367,9 @@ CODESTARTobjDestruct(strm) * IMPORTANT: we MUST free this only AFTER the ansyncWriter has been stopped, else * we get random errors... */ + if(pThis->compressionDriver == STRM_COMPRESS_ZSTD) { + zstdw.Destruct(pThis); + } if(pThis->prevLineSegment) cstrDestruct(&pThis->prevLineSegment); if(pThis->prevMsgSegment) @@ -1802,53 +1818,11 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, const int bFlush) { - int zRet; /* zlib return state */ - DEFiRet; - unsigned outavail = 0; - assert(pThis != NULL); - assert(pBuf != NULL); - - if(!pThis->bzInitDone) { - /* allocate deflate state */ - pThis->zstrm.zalloc = Z_NULL; - pThis->zstrm.zfree = Z_NULL; - pThis->zstrm.opaque = Z_NULL; - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&pThis->zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/deflateInit2()", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } - pThis->bzInitDone = RSTRUE; - } - - /* now doing the compression */ - pThis->zstrm.next_in = (Bytef*) pBuf; - pThis->zstrm.avail_in = lenBuf; - /* run deflate() on buffer until everything has been compressed */ - do { - DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld, bFlush %d\n", - pThis->zstrm.avail_in, pThis->zstrm.total_in, bFlush); - pThis->zstrm.avail_out = pThis->sIOBufSize; - pThis->zstrm.next_out = pThis->pZipBuf; - zRet = zlibw.Deflate(&pThis->zstrm, bFlush ? Z_SYNC_FLUSH : Z_NO_FLUSH); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d, to write %d\n", - zRet, pThis->zstrm.avail_out, outavail); - if(zRet != Z_OK) { - LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/Deflate()", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } - outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; - if(outavail != 0) { - CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); - } - } while (pThis->zstrm.avail_out == 0); - -finalize_it: - if(pThis->bzInitDone && pThis->bVeryReliableZip) { - doZipFinish(pThis); + if(pThis->compressionDriver == STRM_COMPRESS_ZSTD) { + return zstdw.doStrmWrite(pThis, pBuf, lenBuf, bFlush, strmPhysWrite); + } else { + return zlibw.doStrmWrite(pThis, pBuf, lenBuf, bFlush, strmPhysWrite); } - RETiRet; } @@ -1859,37 +1833,11 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, const int bFlush) static rsRetVal doZipFinish(strm_t *pThis) { - int zRet; /* zlib return state */ - DEFiRet; - unsigned outavail; - assert(pThis != NULL); - - if(!pThis->bzInitDone) - goto done; - - pThis->zstrm.avail_in = 0; - /* run deflate() on buffer until everything has been compressed */ - do { - DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, - pThis->zstrm.total_in); - pThis->zstrm.avail_out = pThis->sIOBufSize; - pThis->zstrm.next_out = pThis->pZipBuf; - zRet = zlibw.Deflate(&pThis->zstrm, Z_FINISH); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); - outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; - if(outavail != 0) { - CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); - } - } while (pThis->zstrm.avail_out == 0); - -finalize_it: - zRet = zlibw.DeflateEnd(&pThis->zstrm); - if(zRet != Z_OK) { - LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/deflateEnd()", zRet); + if(pThis->compressionDriver == STRM_COMPRESS_ZSTD) { + return zstdw.doCompressFinish(pThis, strmPhysWrite); + } else { + return zlibw.doCompressFinish(pThis, strmPhysWrite); } - - pThis->bzInitDone = 0; -done: RETiRet; } /* flush stream output buffer to persistent storage. This can be called at any time @@ -2192,6 +2140,7 @@ DEFpropSetMeth(strm, iMaxFileSize, int64) DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) +DEFpropSetMeth(strm, compressionDriver, strm_compressionDriver_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bVeryReliableZip, int) @@ -2438,6 +2387,7 @@ strmDup(strm_t *const pThis, strm_t **ppNew) pNew->lenDir = pThis->lenDir; pNew->tOperationsMode = pThis->tOperationsMode; pNew->tOpenMode = pThis->tOpenMode; + pNew->compressionDriver = pThis->compressionDriver; pNew->iMaxFileSize = pThis->iMaxFileSize; pNew->iMaxFiles = pThis->iMaxFiles; pNew->iFileNumDigits = pThis->iFileNumDigits; @@ -2454,6 +2404,18 @@ strmDup(strm_t *const pThis, strm_t **ppNew) RETiRet; } + +static rsRetVal +SetCompressionWorkers(strm_t *const pThis, int num_wrkrs) +{ + ISOBJ_TYPE_assert(pThis, strm); + if(num_wrkrs < 0) + num_wrkrs = 1; + pThis->zstd.num_wrkrs = num_wrkrs; + return RS_RET_OK; +} + + /* set a user write-counter. This counter is initialized to zero and * receives the number of bytes written. It is accurate only after a * flush(). This hook is provided as a means to control disk size usage. @@ -2582,6 +2544,7 @@ CODESTARTobjQueryInterface(strm) pIf->Serialize = strmSerialize; pIf->GetCurrOffset = strmGetCurrOffset; pIf->Dup = strmDup; + pIf->SetCompressionWorkers = SetCompressionWorkers; pIf->SetWCntr = strmSetWCntr; pIf->CheckFileChange = CheckFileChange; /* set methods */ @@ -2591,6 +2554,7 @@ CODESTARTobjQueryInterface(strm) pIf->SetiFileNumDigits = strmSetiFileNumDigits; pIf->SettOperationsMode = strmSettOperationsMode; pIf->SettOpenMode = strmSettOpenMode; + pIf->SetcompressionDriver = strmSetcompressionDriver; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; pIf->SetbVeryReliableZip = strmSetbVeryReliableZip; diff --git a/runtime/stream.h b/runtime/stream.h index 2830f7c115..4af3d3345b 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -65,6 +65,8 @@ #ifndef STREAM_H_INCLUDED #define STREAM_H_INCLUDED +typedef struct strm_s strm_t; /* forward reference because of zlib... */ + #include // TODO: fix via own module #include #include @@ -90,6 +92,10 @@ typedef enum { /* when extending, do NOT change existing modes! */ STREAMMODE_WRITE_TRUNC = 3, STREAMMODE_WRITE_APPEND = 4 } strmMode_t; +typedef enum { + STRM_COMPRESS_ZIP = 0, + STRM_COMPRESS_ZSTD = 1 + } strm_compressionDriver_t; /* settings for stream rotation (applies not to all processing modes!) */ #define STRM_ROTATION_DO_CHECK 0 @@ -97,7 +103,7 @@ typedef enum { /* when extending, do NOT change existing modes! */ #define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */ /* The strm_t data structure */ -typedef struct strm_s { +struct strm_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ strmType_t sType; /* descriptive properties */ @@ -159,6 +165,10 @@ typedef struct strm_s { uchar *pBuf; size_t lenBuf; } asyncBuf[STREAM_ASYNC_NUMBUFS]; + struct { + int num_wrkrs; /* nbr of worker threads */ + void *cctx; + } zstd; /* supporting per-instance data if zstd is used */ pthread_t writerThreadID; /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ @@ -170,7 +180,8 @@ typedef struct strm_s { int fileNotFoundError; /* boolean; if set, report file not found errors, else silently ignore */ int noRepeatedErrorOutput; /* if a file is missing the Error is only given once */ int ignoringMsg; -} strm_t; + strm_compressionDriver_t compressionDriver; +}; /* interfaces */ @@ -195,12 +206,14 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); rsRetVal (*Dup)(strm_t *pThis, strm_t **ppNew); + rsRetVal (*SetCompressionWorkers)(strm_t *const pThis, int num_wrkrs); INTERFACEpropSetMeth(strm, bDeleteOnClose, int); INTERFACEpropSetMeth(strm, iMaxFileSize, int64); INTERFACEpropSetMeth(strm, iMaxFiles, int); INTERFACEpropSetMeth(strm, iFileNumDigits, int); INTERFACEpropSetMeth(strm, tOperationsMode, int); INTERFACEpropSetMeth(strm, tOpenMode, mode_t); + INTERFACEpropSetMeth(strm, compressionDriver, strm_compressionDriver_t); INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); INTERFACEpropSetMeth(strm, bSync, int); diff --git a/runtime/zlibw.c b/runtime/zlibw.c index 05aa1f7c95..77fd2e7a4d 100644 --- a/runtime/zlibw.c +++ b/runtime/zlibw.c @@ -2,7 +2,7 @@ * * This is an rsyslog object wrapper around zlib. * - * Copyright 2009-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2022 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -29,6 +29,7 @@ #include "rsyslog.h" #include "module-template.h" #include "obj.h" +#include "stream.h" #include "zlibw.h" MODULE_TYPE_LIB @@ -65,6 +66,117 @@ static int myDeflate(z_streamp strm, int flush) return deflate(strm, flush); } +/* finish zlib buffer */ +static rsRetVal +doCompressFinish(strm_t *pThis, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail; + assert(pThis != NULL); + + assert(pThis->compressionDriver == STRM_COMPRESS_ZIP); + + if(!pThis->bzInitDone) + goto done; + + pThis->zstrm.avail_in = 0; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, + pThis->zstrm.total_in); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = deflate(&pThis->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); + } + } while (pThis->zstrm.avail_out == 0); + +finalize_it: + zRet = deflateEnd(&pThis->zstrm); + if(zRet != Z_OK) { + LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/deflateEnd()", zRet); + } + + pThis->bzInitDone = 0; +done: RETiRet; +} + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + * Note that we always do a full deflateInit ... deflate ... deflateEnd + * sequence. While this is not optimal, we need to do it because we need + * to ensure that the file is readable even when we are aborted. Doing the + * full sequence brings us as far towards this goal as possible (and not + * doing it would be a total failure). It may be worth considering to + * add a config switch so that the user can decide the risk he is ready + * to take, but so far this is not yet implemented (not even requested ;)). + * rgerhards, 2009-06-04 + */ +static rsRetVal +doStrmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, const int bFlush, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail = 0; + assert(pThis != NULL); + assert(pBuf != NULL); + + assert(pThis->compressionDriver == STRM_COMPRESS_ZIP); + + if(!pThis->bzInitDone) { + /* allocate deflate state */ + pThis->zstrm.zalloc = Z_NULL; + pThis->zstrm.zfree = Z_NULL; + pThis->zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = deflateInit2(&pThis->zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/deflateInit2()", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + pThis->bzInitDone = RSTRUE; + } + + /* now doing the compression */ + pThis->zstrm.next_in = (Bytef*) pBuf; + pThis->zstrm.avail_in = lenBuf; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld, bFlush %d\n", + pThis->zstrm.avail_in, pThis->zstrm.total_in, bFlush); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = deflate(&pThis->zstrm, bFlush ? Z_SYNC_FLUSH : Z_NO_FLUSH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d, to write %d\n", + zRet, pThis->zstrm.avail_out, outavail); + if(zRet != Z_OK) { + LogError(0, RS_RET_ZLIB_ERR, "error %d returned from zlib/Deflate()", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); + } + } while (pThis->zstrm.avail_out == 0); + +finalize_it: + if(pThis->bzInitDone && pThis->bVeryReliableZip) { + doCompressFinish(pThis, strmPhysWrite); + } + RETiRet; +} +/* destruction of caller's zlib ressources - a dummy for us */ +static rsRetVal +zlib_Destruct(ATTR_UNUSED strm_t *pThis) +{ + return RS_RET_OK; +} /* queryInterface function * rgerhards, 2008-03-05 @@ -84,6 +196,9 @@ CODESTARTobjQueryInterface(zlibw) pIf->DeflateInit2 = myDeflateInit2; pIf->Deflate = myDeflate; pIf->DeflateEnd = myDeflateEnd; + pIf->doStrmWrite = doStrmWrite; + pIf->doCompressFinish = doCompressFinish; + pIf->Destruct = zlib_Destruct; finalize_it: ENDobjQueryInterface(zlibw) diff --git a/runtime/zlibw.h b/runtime/zlibw.h index ac9f5fc8f0..fdbd5203eb 100644 --- a/runtime/zlibw.h +++ b/runtime/zlibw.h @@ -2,7 +2,7 @@ * purpose of this wrapper class is to enable rsyslogd core to be build without * zlib libraries. * - * Copyright 2009-2012 Adiscon GmbH. + * Copyright 2009-2022 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -25,14 +25,21 @@ #include +#include "errmsg.h" + /* interfaces */ BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ int (*DeflateInit)(z_streamp strm, int); int (*DeflateInit2)(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy); int (*Deflate)(z_streamp strm, int); int (*DeflateEnd)(z_streamp strm); + rsRetVal (*doStrmWrite)(strm_t *pThis, uchar *const pBuf, const size_t lenBuf, const int bFlush, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ); + rsRetVal (*doCompressFinish)(strm_t *pThis, + rsRetVal (*Destruct)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ); + rsRetVal (*Destruct)(strm_t *pThis); ENDinterface(zlibw) -#define zlibwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define zlibwCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/zstdw.c b/runtime/zstdw.c new file mode 100644 index 0000000000..7f80daecc7 --- /dev/null +++ b/runtime/zstdw.c @@ -0,0 +1,191 @@ +/* The zstdw object. + * + * This is an rsyslog object wrapper around zstd. + * + * Copyright 2022 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include +#include +#include + +#include "rsyslog.h" +#include "errmsg.h" +#include "stream.h" +#include "module-template.h" +#include "obj.h" +#include "zstdw.h" + +MODULE_TYPE_LIB +MODULE_TYPE_NOKEEP + +/* static data */ +DEFobjStaticHelpers + + +/* finish buffer, to be called before closing the zstd file. */ +static rsRetVal +zstd_doCompressFinish(strm_t *pThis, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ) +{ + size_t remaining = 0; + DEFiRet; + assert(pThis != NULL); + + if(!pThis->bzInitDone) + goto done; + + char dummybuf; /* not sure if we can pass in NULL as buffer address in this special case */ + ZSTD_inBuffer input = { &dummybuf, 0, 0 }; + + do { + ZSTD_outBuffer output = { pThis->pZipBuf, pThis->sIOBufSize, 0 }; + remaining = ZSTD_compressStream2(pThis->zstd.cctx, &output , &input, ZSTD_e_end); + if(ZSTD_isError(remaining)) { + LogError(0, RS_RET_ZLIB_ERR, + "error returned from ZSTD_compressStream2(): %s\n", + ZSTD_getErrorName(remaining)); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, output.pos)); + + } while (remaining != 0); + +finalize_it: +done: RETiRet; +} + + +static rsRetVal +zstd_doStrmWrite(strm_t *pThis, uchar *const pBuf, const size_t lenBuf, const int bFlush, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ) +{ + DEFiRet; + assert(pThis != NULL); + assert(pBuf != NULL); + if(!pThis->bzInitDone) { + pThis->zstd.cctx = (void*) ZSTD_createCCtx(); + if(pThis->zstd.cctx == NULL) { + LogError(0, RS_RET_ZLIB_ERR, "error creating zstd context (ZSTD_createCCtx failed, " + "that's all we know"); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + + ZSTD_CCtx_setParameter(pThis->zstd.cctx, ZSTD_c_compressionLevel, pThis->iZipLevel); + ZSTD_CCtx_setParameter(pThis->zstd.cctx, ZSTD_c_checksumFlag, 1); + if(pThis->zstd.num_wrkrs > 0) { + ZSTD_CCtx_setParameter(pThis->zstd.cctx, ZSTD_c_nbWorkers, + pThis->zstd.num_wrkrs); + } + pThis->bzInitDone = RSTRUE; + } + + /* now doing the compression */ + ZSTD_inBuffer input = { pBuf, lenBuf, 0 }; + + // This following needs to be configurable? It's possibly sufficient to use e_flush + // only, as this can also be controlled by veryRobustZip. However, testbench will than + // not be able to check when all file lines are complete. + ZSTD_EndDirective const mode = bFlush ? ZSTD_e_flush : ZSTD_e_continue; + size_t remaining; + do { + ZSTD_outBuffer output = { pThis->pZipBuf, 128, 0 }; + remaining = ZSTD_compressStream2(pThis->zstd.cctx, &output , &input, mode); + if(ZSTD_isError(remaining)) { + LogError(0, RS_RET_ZLIB_ERR, "error returned from ZSTD_compressStream2(): %s", + ZSTD_getErrorName(remaining)); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, output.pos)); + + } while ( mode == ZSTD_e_end ? (remaining != 0) : (input.pos != input.size)); + +finalize_it: + if(pThis->bzInitDone && pThis->bVeryReliableZip) { + zstd_doCompressFinish(pThis, strmPhysWrite); + } + RETiRet; +} + +/* destruction of caller's zstd ressources */ +static rsRetVal +zstd_Destruct(strm_t *const pThis) +{ + DEFiRet; + assert(pThis != NULL); + + if(!pThis->bzInitDone) + goto done; + + const int result = ZSTD_freeCCtx(pThis->zstd.cctx); + if(ZSTD_isError(result)) { + LogError(0, RS_RET_ZLIB_ERR, "error from ZSTD_freeCCtx(): %s", + ZSTD_getErrorName(result)); + } + + pThis->bzInitDone = 0; +done: RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-03-05 + */ +BEGINobjQueryInterface(zstdw) +CODESTARTobjQueryInterface(zstdw) + if(pIf->ifVersion != zstdwCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + pIf->doStrmWrite = zstd_doStrmWrite; + pIf->doCompressFinish = zstd_doCompressFinish; + pIf->Destruct = zstd_Destruct; +finalize_it: +ENDobjQueryInterface(zstdw) + + +/* Initialize the zstdw class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINAbstractObjClassInit(zstdw, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ +ENDObjClassInit(zstdw) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + CHKiRet(zstdwClassInit(pModInfo)); +ENDmodInit diff --git a/runtime/zstdw.h b/runtime/zstdw.h new file mode 100644 index 0000000000..0dbc46867c --- /dev/null +++ b/runtime/zstdw.h @@ -0,0 +1,43 @@ +/* The zstdw object. It encapsulates the zstd functionality. The primary + * purpose of this wrapper class is to enable rsyslogd core to be build without + * zstd libraries. + * + * Copyright 2022 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_ZSTDW_H +#define INCLUDED_ZSTDW_H + +/* interfaces */ +BEGINinterface(zstdw) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*doStrmWrite)(strm_t *pThis, uchar *const pBuf, const size_t lenBuf, const int bFlush, + rsRetVal (*strmPhysWrite)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ); + rsRetVal (*doCompressFinish)(strm_t *pThis, + rsRetVal (*Destruct)(strm_t *pThis, uchar *pBuf, size_t lenBuf) ); + rsRetVal (*Destruct)(strm_t *pThis); +ENDinterface(zstdw) +#define zstdwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(zstdw); + +/* the name of our library binary */ +#define LM_ZSTDW_FILENAME "lmzstdw" + +#endif /* #ifndef INCLUDED_ZSTDW_H */ diff --git a/tests/Makefile.am b/tests/Makefile.am index c190e528b3..2c5e27e0a6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -503,6 +503,15 @@ TESTS += \ parsertest-snare_ccoff_udp.sh \ parsertest-snare_ccoff_udp2.sh +if ENABLE_LIBZSTD +TESTS += \ + zstd.sh +if HAVE_VALGRIND +TESTS += \ + zstd-vg.sh +endif # if HAVE_VALGRIND +endif + if ENABLE_LIBGCRYPT TESTS += \ queue-encryption-disk.sh \ @@ -1979,6 +1988,8 @@ EXTRA_DIST= \ omfwd-keepalive.sh \ omfwd_fast_imuxsock.sh \ omfile_hup-vg.sh \ + zstd.sh \ + zstd-vg.sh \ gzipwr_hup-vg.sh \ omusrmsg-noabort-legacy.sh \ omusrmsg-errmsg-no-params.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index fb766ab8e2..c72453b95a 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -303,7 +303,7 @@ wait_startup_pid() { if [ $(date +%s) -gt $(( TB_STARTTEST + TB_TEST_MAX_RUNTIME )) ]; then printf '%s ABORT! Timeout waiting on startup (pid file %s)\n' "$(tb_timestamp)" "$1" ls -l "$1" - ps -fp $(cat "$1") + ps -fp $($SUDO cat "$1") error_exit 1 fi done @@ -1397,6 +1397,9 @@ seq_check() { fi if [ "${SEQ_CHECK_FILE##*.}" == "gz" ]; then gunzip -c "${SEQ_CHECK_FILE}" | $RS_SORTCMD $RS_SORT_NUMERIC_OPT | ./chkseq -s$startnum -e$endnum $3 $4 $5 $6 $7 $SEQ_CHECK_OPTIONS + elif [ "${SEQ_CHECK_FILE##*.}" == "zst" ]; then + ls -l "${SEQ_CHECK_FILE}" + unzstd < "${SEQ_CHECK_FILE}" | $RS_SORTCMD $RS_SORT_NUMERIC_OPT | ./chkseq -s$startnum -e$endnum $3 $4 $5 $6 $7 $SEQ_CHECK_OPTIONS else $RS_SORTCMD $RS_SORT_NUMERIC_OPT < "${SEQ_CHECK_FILE}" | ./chkseq -s$startnum -e$endnum $3 $4 $5 $6 $7 $SEQ_CHECK_OPTIONS fi @@ -1409,6 +1412,10 @@ seq_check() { gunzip -c "${SEQ_CHECK_FILE}" | $RS_SORTCMD $RS_SORT_NUMERIC_OPT \ | ./chkseq -s$startnum -e$endnum $3 $4 $5 $6 $7 $SEQ_CHECK_OPTIONS \ > $RSYSLOG_DYNNAME.error.log + elif [ "${SEQ_CHECK_FILE##*.}" == "zst" ]; then + unzstd < "${SEQ_CHECK_FILE}" | $RS_SORTCMD $RS_SORT_NUMERIC_OPT \ + | ./chkseq -s$startnum -e$endnum $3 $4 $5 $6 $7 $SEQ_CHECK_OPTIONS \ + > $RSYSLOG_DYNNAME.error.log else $RS_SORTCMD $RS_SORT_NUMERIC_OPT < ${SEQ_CHECK_FILE} \ > $RSYSLOG_DYNNAME.error.log diff --git a/tests/zstd-vg.sh b/tests/zstd-vg.sh new file mode 100755 index 0000000000..e9b11e2b96 --- /dev/null +++ b/tests/zstd-vg.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export USE_VALGRIND="YES" +export NUMMESSAGES=10000 # valgrind is pretty slow, so we need to user lower nbr of msgs +source ${srcdir:-.}/zstd.sh diff --git a/tests/zstd.sh b/tests/zstd.sh new file mode 100755 index 0000000000..a109644f8e --- /dev/null +++ b/tests/zstd.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# This tests writing large data records in zst mode. We use up to 10K +# record size. +# +# added 2022-06-21 by Rgerhards +# +# This file is part of the rsyslog project, released under ASL 2.0 +. ${srcdir:=.}/diag.sh init +export NUMMESSAGES=${NUMMESSAGES:-50000} +export QUEUE_EMPTY_CHECK_FUNC=wait_seq_check +generate_conf +# Note: we right now use the non-compressed file as indicator for "processing complete" +#export SEQ_CHECK_FILE=$RSYSLOG_OUT_LOG.zst +add_conf ' +$MaxMessageSize 10k +$MainMsgQueueTimeoutShutdown 10000 + +module(load="builtin:omfile" compression.driver="zstd" compression.zstd.workers="5") +module(load="../plugins/imptcp/.libs/imptcp") +input(type="imptcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port") + +template(name="outfmt" type="string" string="%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n") +local0.* action(type="omfile" file="'$RSYSLOG_OUT_LOG'.zst" template="outfmt" + zipLevel="20" iobuffersize="64k" veryRobustZIP="off") +local0.* action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") +' +# rgerhards, 2019-08-14: Note: veryRobustZip may need to be "on". Do this if the test +# still prematurely terminates. In that case it is likely that gunzip got confused +# by the missing zip close record. My initial testing shows that while gunzip emits an +# error message, everything is properly extracted. Only stressed CI runs will show how +# it works in reality. +startup +assign_tcpflood_port $RSYSLOG_DYNNAME.tcpflood_port +tcpflood -m$NUMMESSAGES -r -d10000 -P129 +shutdown_when_empty +wait_shutdown +seq_check 0 $((NUMMESSAGES - 1)) -E +exit_test diff --git a/tools/omfile.c b/tools/omfile.c index 6bb97e64dd..6a94daade1 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -17,7 +17,7 @@ * pipes. These have been moved to ompipe, to reduced the entanglement * between the two different functionalities. -- rgerhards * - * Copyright 2007-2018 Adiscon GmbH. + * Copyright 2007-2022 Adiscon GmbH. * * This file is part of rsyslog. * @@ -224,6 +224,8 @@ struct modConfData_s { gid_t fileGID; gid_t dirGID; int bDynafileDoNotSuspend; + strm_compressionDriver_t compressionDriver; + int compressionDriver_workers; }; static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ @@ -233,6 +235,8 @@ static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current ex /* module-global parameters */ static struct cnfparamdescr modpdescr[] = { { "template", eCmdHdlrGetWord, 0 }, + { "compression.driver", eCmdHdlrGetWord, 0 }, + { "compression.zstd.workers", eCmdHdlrPositiveInt, 0 }, { "dircreatemode", eCmdHdlrFileCreateMode, 0 }, { "filecreatemode", eCmdHdlrFileCreateMode, 0 }, { "dirowner", eCmdHdlrUID, 0 }, @@ -630,6 +634,8 @@ prepareFile(instanceData *__restrict__ const pData, const uchar *__restrict__ co CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, cs.fCreateMode)); + CHKiRet(strm.SetcompressionDriver(pData->pStrm, runModConf->compressionDriver)); + CHKiRet(strm.SetCompressionWorkers(pData->pStrm, runModConf->compressionDriver_workers)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit)); @@ -897,6 +903,18 @@ CODESTARTsetModCnf "set via legacy directive - may lead to inconsistent " "results."); } + } else if(!strcmp(modpblk.descr[i].name, "compression.driver")) { + if(!es_strcasebufcmp(pvals[i].val.d.estr, (const unsigned char*) "zlib", 4)) { + loadModConf->compressionDriver = STRM_COMPRESS_ZIP; + } else if(!es_strcasebufcmp(pvals[i].val.d.estr, (const unsigned char*) "zstd", 4)) { + loadModConf->compressionDriver = STRM_COMPRESS_ZSTD; + } else { + parser_errmsg("omfile: error: invalid compression.driver driver " + "name - noch applying setting. Valid drivers: 'zlib' and " + "'zstd'."); + } + } else if(!strcmp(modpblk.descr[i].name, "compression.zstd.workers")) { + loadModConf->compressionDriver_workers = (int) pvals[i].val.d.n; } else if(!strcmp(modpblk.descr[i].name, "dircreatemode")) { loadModConf->fDirCreateMode = (int) pvals[i].val.d.n; } else if(!strcmp(modpblk.descr[i].name, "filecreatemode")) { From d9870419a9b830f7711e5fd12a9e9ab5ceb42282 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 13 Jul 2022 15:33:12 +0200 Subject: [PATCH 111/134] maintain ChangeLog --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index 38dea551da..93c6fd71a3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-07-13: omfile: support for zstd compression + The zstd library provides better and faster compression than zlib. + This patch integrates zstd as a dynamically-loadable functionality. + As such, no further dependencies need to be added to the rsyslog + base package. + Due to the increased performance, usage of zstd is highly recommended + for high-volume use cases. + This patch also refactor zlib compression in order to unify handling + in both compression cases. - 2022-07-07: stream cleanup: move error message to debug log, only This error message is most probably rooted in a kernel problem. At least knowbody knows how it can happen. It's definitely not a From 6cf38c0ce728759a92dca2a889a4f8418fdae53a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 May 2022 10:11:10 +0200 Subject: [PATCH 112/134] template procesing/json: performance optimization This patch improves the performance of important parts of template JSON processing. --- runtime/msg.c | 156 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 38 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index e82a4dc845..555ba6237f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -7,7 +7,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007-2020 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2022 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -3290,13 +3290,17 @@ msgGetJSONPropJSON(smsg_t * const pMsg, msgPropDescr_t *pProp, struct json_objec } -/* Encode a JSON value and add it to provided string. Note that - * the string object may be NULL. In this case, it is created - * if and only if escaping is needed. if escapeAll is false, previously - * escaped strings are left as is +/* Helper for jsonAddVal(), to be called onces we know there are actually + * json escapes inside the string. If so, this function takes over. + * Splitting the functions permits us to make some performance optimizations. + * For further details, see jsonAddVal(). */ -static rsRetVal -jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst, int escapeAll) +static rsRetVal ATTR_NONNULL(1, 4) +jsonAddVal_escaped(uchar *const pSrc, + const unsigned buflen, + const unsigned len_none_escaped_head, + es_str_t **dst, + const int escapeAll) { unsigned char c; es_size_t i; @@ -3304,39 +3308,69 @@ jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst, int escapeAll) unsigned ni; unsigned char nc; int j; + uchar wrkbuf[100000]; + size_t dst_realloc_size; + size_t dst_size; + uchar *dst_base; + uchar *dst_w; + uchar *newbuf; DEFiRet; - for(i = 0 ; i < buflen ; ++i) { + assert(len_none_escaped_head <= buflen); + /* first copy over unescaped head string */ + if(len_none_escaped_head+10 > sizeof(wrkbuf)) { + dst_size = 2 * len_none_escaped_head; + CHKmalloc(dst_base = malloc(dst_size)); + } else { + dst_size = sizeof(wrkbuf); + dst_base = wrkbuf; + } + dst_realloc_size = dst_size - 10; /* some buffer for escaping */ + dst_w = dst_base; + memcpy(dst_w, pSrc, len_none_escaped_head); + dst_w += len_none_escaped_head; + + /* now do the escaping */ + for(i = len_none_escaped_head ; i < buflen ; ++i) { + const size_t dst_offset = dst_w - dst_base; + if(dst_offset >= dst_realloc_size) { + const size_t new_size = 2 * dst_size; + if(dst_base == wrkbuf) { + CHKmalloc(newbuf = malloc(new_size)); + memcpy(newbuf, dst_base, dst_offset); + } else { + CHKmalloc(newbuf = realloc(dst_base, new_size)); + } + dst_size = new_size; + dst_realloc_size = new_size - 10; /* some buffer for escaping */ + dst_base = newbuf; + dst_w = dst_base + dst_offset; + } c = pSrc[i]; - if( (c >= 0x23 && c <= 0x2e) - || (c >= 0x30 && c <= 0x5b) + if( (c >= 0x30 && c <= 0x5b) + || (c >= 0x23 && c <= 0x2e) || (c >= 0x5d /* && c <= 0x10FFFF*/) || c == 0x20 || c == 0x21) { /* no need to escape */ - if(*dst != NULL) - es_addChar(dst, c); + *dst_w++ = c; } else { - if(*dst == NULL) { - if(i == 0) { - /* we hope we have only few escapes... */ - *dst = es_newStr(buflen+10); - } else { - *dst = es_newStrFromBuf((char*)pSrc, i); - } - if(*dst == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - } /* we must escape, try RFC4627-defined special sequences first */ switch(c) { case '\0': - es_addBuf(dst, "\\u0000", 6); + *dst_w++ = '\\'; + *dst_w++ = 'u'; + *dst_w++ = '0'; + *dst_w++ = '0'; + *dst_w++ = '0'; + *dst_w++ = '0'; break; case '\"': - es_addBuf(dst, "\\\"", 2); + *dst_w++ = '\\'; + *dst_w++ = '"'; break; case '/': - es_addBuf(dst, "\\/", 2); + *dst_w++ = '\\'; + *dst_w++ = '/'; break; case '\\': if (escapeAll == RSFALSE) { @@ -3347,31 +3381,35 @@ jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst, int escapeAll) /* Attempt to not double encode */ if ( nc == '"' || nc == '/' || nc == '\\' || nc == 'b' || nc == 'f' || nc == 'n' || nc == 'r' || nc == 't' || nc == 'u') { - - es_addChar(dst, c); - es_addChar(dst, nc); + *dst_w++ = c; + *dst_w++ = nc; i = ni; break; } } } - - es_addBuf(dst, "\\\\", 2); + *dst_w++ = '\\'; + *dst_w++ = '\\'; break; case '\010': - es_addBuf(dst, "\\b", 2); + *dst_w++ = '\\'; + *dst_w++ = 'b'; break; case '\014': - es_addBuf(dst, "\\f", 2); + *dst_w++ = '\\'; + *dst_w++ = 'f'; break; case '\n': - es_addBuf(dst, "\\n", 2); + *dst_w++ = '\\'; + *dst_w++ = 'n'; break; case '\r': - es_addBuf(dst, "\\r", 2); + *dst_w++ = '\\'; + *dst_w++ = 'r'; break; case '\t': - es_addBuf(dst, "\\t", 2); + *dst_w++ = '\\'; + *dst_w++ = 't'; break; default: /* TODO : proper Unicode encoding (see header comment) */ @@ -3379,12 +3417,54 @@ jsonAddVal(uchar *pSrc, unsigned buflen, es_str_t **dst, int escapeAll) numbuf[3-j] = hexdigit[c % 16]; c = c / 16; } - es_addBuf(dst, "\\u", 2); - es_addBuf(dst, numbuf, 4); + *dst_w++ = '\\'; + *dst_w++ = 'u'; + *dst_w++ = numbuf[0]; + *dst_w++ = numbuf[1]; + *dst_w++ = numbuf[2]; + *dst_w++ = numbuf[3]; break; } } } + if(*dst == NULL) { + *dst = es_newStrFromBuf((char *) dst_base, dst_w - dst_base); + } else { + es_addBuf(dst, (const char *) dst_base, dst_w - dst_base); + } +finalize_it: + if(dst_base != wrkbuf) { + free(dst_base); + } + RETiRet; +} + + +/* Encode a JSON value and add it to provided string. Note that + * the string object may be NULL. In this case, it is created + * if and only if escaping is needed. if escapeAll is false, previously + * escaped strings are left as is + */ +static rsRetVal ATTR_NONNULL(1, 3) +jsonAddVal(uchar *const pSrc, const unsigned buflen, es_str_t **dst, const int escapeAll) +{ + es_size_t i; + DEFiRet; + + for(i = 0 ; i < buflen ; ++i) { + const uchar c = pSrc[i]; + if(! ( (c >= 0x30 && c <= 0x5b) + || (c >= 0x23 && c <= 0x2e) + || (c >= 0x5d /* && c <= 0x10FFFF*/) + || c == 0x20 || c == 0x21) + ) { + iRet = jsonAddVal_escaped(pSrc, buflen, i, dst, escapeAll); + FINALIZE; + } + } + if(*dst != NULL) { + es_addBuf(dst, (const char *) pSrc, buflen); + } finalize_it: RETiRet; } From 050a611fa2e16514341c019ed3725a1d74e09e74 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 15 Jul 2022 09:19:09 +0200 Subject: [PATCH 113/134] CI: suppress TSAN false positive Close of fd previously added to an epoll set is ok. --- tests/tsan-rt.supp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tsan-rt.supp b/tests/tsan-rt.supp index 37c741d460..bf587a4c8b 100644 --- a/tests/tsan-rt.supp +++ b/tests/tsan-rt.supp @@ -1,5 +1,7 @@ # supressions for LLVM TSAN # doc: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:^close$ +race:^closeSess$ race:^bs_arrcmp_glblDbgFiles$ race:^imptcp_destruct_epd$ race:doLogMsg From f052717178dac12f6891932667bdca07af2a0d46 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 13 Jul 2022 10:04:34 +0200 Subject: [PATCH 114/134] imptcp: slight tuning - reduce indirect addressing to obtain more speed - also a fix for an annoying typo - minor other optimizations - modernization of one test --- plugins/imptcp/imptcp.c | 76 ++++++++++++++++++++------------------ tests/imptcp_multi_line.sh | 10 ++--- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c index 2d3f4b1113..2401de75a9 100644 --- a/plugins/imptcp/imptcp.c +++ b/plugins/imptcp/imptcp.c @@ -320,6 +320,8 @@ struct ptcpsess_s { uchar *pMsg_save; /* message (fragment) save area in regex framing mode */ prop_t *peerName; /* host name we received messages from */ prop_t *peerIP; + const uchar *startRegex;/* cache for performance reasons */ + int iAddtlFrameDelim; /* cache for performance reasons */ }; @@ -1076,7 +1078,7 @@ processDataRcvd_regexFraming(ptcpsess_t *const __restrict__ pThis, * rgerhards, 2008-03-14 * EXTRACT from tcps_sess.c */ -static rsRetVal +static rsRetVal ATTR_NONNULL(1, 2) processDataRcvd(ptcpsess_t *const __restrict__ pThis, char **buff, const int buffLen, @@ -1086,14 +1088,10 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, unsigned *const __restrict__ pnMsgs) { DEFiRet; - char c = **buff; - int octatesToCopy, octatesToDiscard; - uchar *propPeerName = NULL; - int lenPeerName = 0; - uchar *propPeerIP = NULL; - int lenPeerIP = 0; - - if(pThis->pLstn->pSrv->inst->startRegex != NULL) { + const char c = **buff; + int octetsToCopy, octetsToDiscard; + + if(pThis->startRegex != NULL) { processDataRcvd_regexFraming(pThis, buff, stTime, ttGenTime, pMultiSub, pnMsgs); FINALIZE; } @@ -1117,6 +1115,10 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, } if(pThis->inputState == eInOctetCnt) { + uchar *propPeerName = NULL; + int lenPeerName = 0; + uchar *propPeerIP = NULL; + int lenPeerIP = 0; if(isdigit(c)) { if(pThis->iOctetsRemain <= 200000000) { pThis->iOctetsRemain = pThis->iOctetsRemain * 10 + c - '0'; @@ -1163,21 +1165,21 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, } } else if(pThis->inputState == eInMsgTruncation) { if ((c == '\n') - || ((pThis->pLstn->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) - && (c == pThis->pLstn->pSrv->iAddtlFrameDelim))) { + || ((pThis->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) + && (c == pThis->iAddtlFrameDelim))) { pThis->inputState = eAtStrtFram; } } else { assert(pThis->inputState == eInMsg); - if (pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { - if(pThis->iMsg >= iMaxLine) { + int iMsg = pThis->iMsg; /* cache value for faster access */ + if(iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ int i = 1; char currBuffChar; while(i < buffLen && ((currBuffChar = (*buff)[i]) != '\n' - && (pThis->pLstn->pSrv->iAddtlFrameDelim == TCPSRV_NO_ADDTL_DELIMITER - || currBuffChar != pThis->pLstn->pSrv->iAddtlFrameDelim))) { + && (pThis->iAddtlFrameDelim == TCPSRV_NO_ADDTL_DELIMITER + || currBuffChar != pThis->iAddtlFrameDelim))) { i++; } LogError(0, NO_ERRCODE, "imptcp %s: message received is at least %d byte larger than " @@ -1195,21 +1197,23 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, */ } if ((c == '\n') - || ((pThis->pLstn->pSrv->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) - && (c == pThis->pLstn->pSrv->iAddtlFrameDelim)) + || ((pThis->iAddtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) + && (c == pThis->iAddtlFrameDelim)) ) { /* record delimiter? */ if(pThis->pLstn->pSrv->multiLine) { if((buffLen == 1) || ((*buff)[1] == '<')) { doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); + iMsg = 0; /* Reset cached value! */ ++(*pnMsgs); pThis->inputState = eAtStrtFram; } else { - if(pThis->iMsg < iMaxLine) { - *(pThis->pMsg + pThis->iMsg++) = c; + if(iMsg < iMaxLine) { + pThis->pMsg[iMsg++] = c; } } } else { doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); + iMsg = 0; /* Reset cached value! */ ++(*pnMsgs); pThis->inputState = eAtStrtFram; } @@ -1219,26 +1223,27 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, * we truncate it. This is the best we can do in light of what the engine supports. * -- rgerhards, 2008-03-14 */ - if(pThis->iMsg < iMaxLine) { - *(pThis->pMsg + pThis->iMsg++) = c; + if(likely(iMsg < iMaxLine)) { + pThis->pMsg[iMsg++] = c; } } + pThis->iMsg = iMsg; /* update "real value" with cached one */ } else { assert(pThis->eFraming == TCP_FRAMING_OCTET_COUNTING); - octatesToCopy = pThis->iOctetsRemain; - octatesToDiscard = 0; - if (buffLen < octatesToCopy) { - octatesToCopy = buffLen; + octetsToCopy = pThis->iOctetsRemain; + octetsToDiscard = 0; + if (buffLen < octetsToCopy) { + octetsToCopy = buffLen; } - if (octatesToCopy + pThis->iMsg > iMaxLine) { - octatesToDiscard = octatesToCopy - (iMaxLine - pThis->iMsg); - octatesToCopy = iMaxLine - pThis->iMsg; + if (octetsToCopy + pThis->iMsg > iMaxLine) { + octetsToDiscard = octetsToCopy - (iMaxLine - pThis->iMsg); + octetsToCopy = iMaxLine - pThis->iMsg; } - memcpy(pThis->pMsg + pThis->iMsg, *buff, octatesToCopy); - pThis->iMsg += octatesToCopy; - pThis->iOctetsRemain -= (octatesToCopy + octatesToDiscard); - *buff += (octatesToCopy + octatesToDiscard - 1); + memcpy(pThis->pMsg + pThis->iMsg, *buff, octetsToCopy); + pThis->iMsg += octetsToCopy; + pThis->iOctetsRemain -= (octetsToCopy + octetsToDiscard); + *buff += (octetsToCopy + octetsToDiscard - 1); if (pThis->iOctetsRemain == 0) { /* we have end of frame! */ doSubmitMsg(pThis, stTime, ttGenTime, pMultiSub); @@ -1270,8 +1275,8 @@ processDataRcvd(ptcpsess_t *const __restrict__ pThis, * we have just received a bunch of data! -- rgerhards, 2009-06-16 * EXTRACT from tcps_sess.c */ -static rsRetVal -DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, struct syslogTime *stTime, time_t ttGenTime) +static rsRetVal ATTR_NONNULL(1, 2) +DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, const size_t iLen, struct syslogTime *stTime, time_t ttGenTime) { multi_submit_t multiSub; smsg_t *pMsgs[CONF_NUM_MULTISUB]; @@ -1279,7 +1284,6 @@ DataRcvdUncompressed(ptcpsess_t *pThis, char *pData, size_t iLen, struct syslogT unsigned nMsgs = 0; DEFiRet; - assert(pData != NULL); assert(iLen > 0); if(ttGenTime == 0) @@ -1533,6 +1537,8 @@ addSess(ptcplstn_t *pLstn, int sock, prop_t *peerName, prop_t *peerIP) pSess->peerName = peerName; pSess->peerIP = peerIP; pSess->compressionMode = pLstn->pSrv->compressionMode; + pSess->startRegex = pLstn->pSrv->inst->startRegex; + pSess->iAddtlFrameDelim = pLstn->pSrv->iAddtlFrameDelim; /* add to start of server's listener list */ pSess->prev = NULL; diff --git a/tests/imptcp_multi_line.sh b/tests/imptcp_multi_line.sh index 3c058ac70e..f21fae847c 100755 --- a/tests/imptcp_multi_line.sh +++ b/tests/imptcp_multi_line.sh @@ -15,17 +15,13 @@ startup tcpflood -B -I ${srcdir}/testsuites/imptcp_multi_line.testdata shutdown_when_empty # shut down rsyslogd when done processing messages wait_shutdown # and wait for it to terminate -echo 'NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test1 +export EXPECTED='NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test1 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test2 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag multi#012line1 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag multi#012l#012i#012n#012#012e2 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test3 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag multi#012line3 NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test4 -NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test end' | cmp - $RSYSLOG_OUT_LOG -if [ ! $? -eq 0 ]; then - echo "invalid response generated, $RSYSLOG_OUT_LOG is:" - cat $RSYSLOG_OUT_LOG - error_exit 1 -fi; +NEWMSG: <133>Mar 1 01:00:00 172.20.245.8 tag test end' +cmp_exact exit_test From d59feba46b8d8c2c3c5c25c6fc6e99f93bdae8b9 Mon Sep 17 00:00:00 2001 From: seuzw930 <76191785+seuzw930@users.noreply.github.com> Date: Mon, 18 Jul 2022 15:43:17 +0800 Subject: [PATCH 115/134] Fix memory leak when free action worker data table During free action worker data table when action destruct, worker instance in worker data table were not null. It resulted in memory leak and this patch fixes this behaviour. --- action.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/action.c b/action.c index 4c3b581b34..8f7ba8f68b 100644 --- a/action.c +++ b/action.c @@ -316,6 +316,20 @@ actionResetQueueParams(void) RETiRet; } +/* free action worker data table +*/ +static void freeWrkrDataTable(action_t * const pThis) +{ + int freeSpot; + for(freeSpot = 0; freeSpot < pThis->wrkrDataTableSize; ++freeSpot) { + if(pThis->wrkrDataTable[freeSpot] != NULL) { + pThis->pMod->mod.om.freeWrkrInstance(pThis->wrkrDataTable[freeSpot]); + pThis->wrkrDataTable[freeSpot] = NULL; + } + } + free(pThis->wrkrDataTable); + return; +} /* destructs an action descriptor object * rgerhards, 2007-08-01 @@ -353,7 +367,7 @@ rsRetVal actionDestruct(action_t * const pThis) free(pThis->pszName); free(pThis->ppTpl); free(pThis->peParamPassing); - free(pThis->wrkrDataTable); + freeWrkrDataTable(pThis); finalize_it: free(pThis); From 48d85dcdde2adbbece709a629ec2fa52695c8da5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 Jul 2022 12:45:13 +0200 Subject: [PATCH 116/134] maintain ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 93c6fd71a3..17285466af 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-07.19: core bugfix: memory leak when free action worker data table + During free action worker data table when action destruct, worker instance in worker + data table were not null. It resulted in memory leak. + Thanks to github user seuzw930 for the patch. - 2022-07-13: omfile: support for zstd compression The zstd library provides better and faster compression than zlib. This patch integrates zstd as a dynamically-loadable functionality. From 04c3f21bd46703e70f54e030c1cf5611948f0bbe Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Jul 2022 13:02:18 +0200 Subject: [PATCH 117/134] maintain ChangeLog --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 17285466af..81ec0d9219 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? -- 2022-07.19: core bugfix: memory leak when free action worker data table +- 2022-07-20: template procesing/json: performance optimization +- 2022-07-19: core bugfix: memory leak when free action worker data table During free action worker data table when action destruct, worker instance in worker data table were not null. It resulted in memory leak. Thanks to github user seuzw930 for the patch. From 8a52bf055394b38a0da609ea1104ca336e3d5252 Mon Sep 17 00:00:00 2001 From: cody Date: Thu, 7 Jul 2022 16:11:43 +0200 Subject: [PATCH 118/134] support sha256 for StreamDriverAuthMode="x509/fingerprint" --- runtime/nsd_gtls.c | 22 ++++++++++++++++++---- runtime/nsd_ossl.c | 24 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 9e65d2943b..66ad580b6d 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -494,7 +494,7 @@ print_info(nsd_gtls_t *pThis) * rgerhards, 2008-05-08 */ static rsRetVal -GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) +GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr, const char* prefix) { cstr_t *pStr = NULL; uchar buf[4]; @@ -502,7 +502,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) DEFiRet; CHKiRet(rsCStrConstruct(&pStr)); - CHKiRet(rsCStrAppendStrWithLen(pStr, (uchar*)"SHA1", 4)); + CHKiRet(rsCStrAppendStrWithLen(pStr, (uchar*) prefix, strlen(prefix))); for(i = 0 ; i < sizeFingerprint ; ++i) { snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); @@ -919,8 +919,11 @@ static rsRetVal gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt_t *pCert) { uchar fingerprint[20]; + uchar fingerprintSha256[32]; size_t size; + size_t sizeSha256; cstr_t *pstrFingerprint = NULL; + cstr_t *pstrFingerprintSha256 = NULL; int bFoundPositiveMatch; permittedPeers_t *pPeer; int gnuRet; @@ -930,17 +933,27 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt_t *pCert) /* obtain the SHA1 fingerprint */ size = sizeof(fingerprint); + sizeSha256 = sizeof(fingerprintSha256); CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA1, fingerprint, &size)); - CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); + CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA256, fingerprintSha256, &sizeSha256)); + CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint, "SHA1")); + CHKiRet(GenFingerprintStr(fingerprintSha256, sizeSha256, &pstrFingerprintSha256, "SHA256")); dbgprintf("peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStrNoNULL(pstrFingerprint)); + dbgprintf("peer's certificate SHA256 fingerprint: %s\n", cstrGetSzStrNoNULL(pstrFingerprintSha256)); + /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; pPeer = pThis->pPermPeers; while(pPeer != NULL && !bFoundPositiveMatch) { if(!rsCStrSzStrCmp(pstrFingerprint, pPeer->pszID, strlen((char*) pPeer->pszID))) { + dbgprintf("gtlsChkPeerFingerprint: peer's certificate SHA1 MATCH found: %s\n", pPeer->pszID); bFoundPositiveMatch = 1; - } else { + } else if(!rsCStrSzStrCmp(pstrFingerprintSha256 , pPeer->pszID, strlen((char*) pPeer->pszID))) { + dbgprintf("gtlsChkPeerFingerprint: peer's certificate SHA256 MATCH found: %s\n", pPeer->pszID); + bFoundPositiveMatch = 1; + } + else { pPeer = pPeer->pNext; } } @@ -2373,3 +2386,4 @@ CODESTARTmodInit ENDmodInit /* vi:set ai: */ + diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 2f1e851318..9dafd148a6 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -381,7 +381,7 @@ long BIO_debug_callback(BIO *bio, int cmd, const char __attribute__((unused)) *a * rgerhards, 2008-05-08 */ static rsRetVal -GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) +GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr, const char* prefix) { cstr_t *pStr = NULL; uchar buf[4]; @@ -389,7 +389,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) DEFiRet; CHKiRet(rsCStrConstruct(&pStr)); - CHKiRet(rsCStrAppendStrWithLen(pStr, (uchar*)"SHA1", 4)); + CHKiRet(rsCStrAppendStrWithLen(pStr, (uchar*) prefix, strlen(prefix))); for(i = 0 ; i < sizeFingerprint ; ++i) { snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); @@ -607,11 +607,15 @@ osslChkPeerFingerprint(nsd_ossl_t *pThis, X509 *pCert) unsigned int n; uchar *fromHostIP = NULL; uchar fingerprint[20 /*EVP_MAX_MD_SIZE**/]; + uchar fingerprintSha256[32 /*EVP_MAX_MD_SIZE**/]; size_t size; + size_t sizeSha256; cstr_t *pstrFingerprint = NULL; + cstr_t *pstrFingerprintSha256 = NULL; int bFoundPositiveMatch; permittedPeers_t *pPeer; const EVP_MD *fdig = EVP_sha1(); + const EVP_MD *fdigSha256 = EVP_sha256(); ISOBJ_TYPE_assert(pThis, nsd_ossl); @@ -621,17 +625,27 @@ osslChkPeerFingerprint(nsd_ossl_t *pThis, X509 *pCert) dbgprintf("osslChkPeerFingerprint: error X509cert is not valid!\n"); ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); } - - CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); + sizeSha256 = sizeof(fingerprintSha256); + if (!X509_digest(pCert,fdigSha256,fingerprintSha256,&n)) { + dbgprintf("osslChkPeerFingerprint: error X509cert is not valid!\n"); + ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); + } + CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint, "SHA1")); dbgprintf("osslChkPeerFingerprint: peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStrNoNULL(pstrFingerprint)); + CHKiRet(GenFingerprintStr(fingerprintSha256, sizeSha256, &pstrFingerprintSha256, "SHA256")); + dbgprintf("osslChkPeerFingerprint: peer's certificate SHA256 fingerprint: %s\n", + cstrGetSzStrNoNULL(pstrFingerprintSha256)); /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; pPeer = pThis->pPermPeers; while(pPeer != NULL && !bFoundPositiveMatch) { if(!rsCStrSzStrCmp(pstrFingerprint, pPeer->pszID, strlen((char*) pPeer->pszID))) { - dbgprintf("osslChkPeerFingerprint: peer's certificate MATCH found: %s\n", pPeer->pszID); + dbgprintf("osslChkPeerFingerprint: peer's certificate SHA1 MATCH found: %s\n", pPeer->pszID); + bFoundPositiveMatch = 1; + } else if(!rsCStrSzStrCmp(pstrFingerprintSha256, pPeer->pszID, strlen((char*) pPeer->pszID))) { + dbgprintf("osslChkPeerFingerprint: peer's certificate SHA256 MATCH found: %s\n", pPeer->pszID); bFoundPositiveMatch = 1; } else { dbgprintf("osslChkPeerFingerprint: NOMATCH peer certificate: %s\n", pPeer->pszID); From 8001a614bae43ff65303ddf2789e6072152894a9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Jul 2022 15:25:59 +0200 Subject: [PATCH 119/134] maintain ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 81ec0d9219..a8696d1bdc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-07-21: imptcp: slight tuning + - reduce indirect addressing to obtain more speed + - also a fix for an annoying typo + - minor other optimizations + - modernization of one test - 2022-07-20: template procesing/json: performance optimization - 2022-07-19: core bugfix: memory leak when free action worker data table During free action worker data table when action destruct, worker instance in worker From d21d9cfa57e8ff9d2cb428ea60f2beaecd3c06d4 Mon Sep 17 00:00:00 2001 From: "taavi.valjaots" Date: Wed, 27 Jul 2022 10:43:55 +0300 Subject: [PATCH 120/134] ksi bugfix: possible crash fixed when several log files are opened. KSI module in async mode used to request aggregator conf. every time a log file was opened. When several log files were opened simultaneously corresponding amount of pointless concurrent conf. requests were posted. Concurrent conf. requests lead to a bug in libksi, where internal count of pending requests was not decremented correctly causing system to crash. Fix for the issue is to optimize the frequency of conf. requests so that only one conf. requests is handled at once. Instead of checking conf. every time log file is opened, conf is requested periodically after conf timeout. This will affect both sync and async mode. New option for KSI module introduced - sig.confinterval="time, s". --- runtime/lib_ksils12.c | 39 +++++++++++++++++++++++++++++---------- runtime/lib_ksils12.h | 3 +++ runtime/lmsig_ksi-ls12.c | 9 +++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/runtime/lib_ksils12.c b/runtime/lib_ksils12.c index 3320c6acf3..f5a11739a0 100644 --- a/runtime/lib_ksils12.c +++ b/runtime/lib_ksils12.c @@ -732,6 +732,19 @@ static void handle_ksi_config(rsksictx ctx, KSI_AsyncService *as, KSI_Config *co } } +static int +isAggrConfNeeded(rsksictx ctx) { + time_t now = 0; + + now = time(NULL); + + if ((uint64_t)ctx->tConfRequested + ctx->confInterval <= (uint64_t)now || ctx->tConfRequested == 0) { + ctx->tConfRequested = now; + return 1; + } + + return 0; +} /* note: if file exists, the last hash for chaining must * be read from file. @@ -741,7 +754,6 @@ ksiOpenSigFile(ksifile ksi) { int r = 0, tmpRes = 0; const char *header; FILE* signatureFile = NULL; - KSI_Config *config = NULL; if (ksi->ctx->syncMode == LOGSIG_ASYNCHRONOUS) header = LS12_BLOCKFILE_HEADER; @@ -777,14 +789,17 @@ ksiOpenSigFile(ksifile ksi) { ksiReadStateFile(ksi); if (ksi->ctx->syncMode == LOGSIG_SYNCHRONOUS) { - tmpRes = KSI_receiveAggregatorConfig(ksi->ctx->ksi_ctx, &config); - if(tmpRes == KSI_OK) { - handle_ksi_config(ksi->ctx, NULL, config); - } - else { - reportKSIAPIErr(ksi->ctx, NULL, "KSI_receiveAggregatorConfig", tmpRes); + if (isAggrConfNeeded(ksi->ctx)) { + KSI_Config *config = NULL; + + tmpRes = KSI_receiveAggregatorConfig(ksi->ctx->ksi_ctx, &config); + if (tmpRes == KSI_OK) { + handle_ksi_config(ksi->ctx, NULL, config); + } else { + reportKSIAPIErr(ksi->ctx, NULL, "KSI_receiveAggregatorConfig", tmpRes); + } + KSI_Config_free(config); } - KSI_Config_free(config); } done: return r; @@ -856,6 +871,8 @@ rsksiCtxNew(void) { ctx->bKeepTreeHashes = false; ctx->bKeepRecordHashes = true; ctx->max_requests = (1 << 8); + ctx->confInterval = 3600; + ctx->tConfRequested = 0; ctx->errFunc = NULL; ctx->usrptr = NULL; ctx->fileUID = -1; @@ -1998,6 +2015,10 @@ void *signer_thread(void *arg) { while (true) { timeout = 1; + if (isAggrConfNeeded(ctx)) { + request_async_config(ctx, ksi_ctx, as); + } + /* Wait for a work item or timeout*/ ProtectedQueue_waitForItem(ctx->signer_queue, NULL, timeout * 1000); /* Check for block time limit*/ @@ -2035,8 +2056,6 @@ void *signer_thread(void *arg) { if (ksiFileCount > 0) ksiFileCount--; } else if (item->type == QITEM_NEW_FILE) { ksiFileCount++; - request_async_config(ctx, ksi_ctx, as); - /* renew the config when opening a new file */ } else if (item->type == QITEM_QUIT) { free(item); diff --git a/runtime/lib_ksils12.h b/runtime/lib_ksils12.h index 6938bd20d7..fb69fa95f5 100644 --- a/runtime/lib_ksils12.h +++ b/runtime/lib_ksils12.h @@ -90,6 +90,8 @@ struct rsksictx_s { KSI_HashAlgorithm hmacAlg; uint8_t bKeepRecordHashes; uint8_t bKeepTreeHashes; + uint64_t confInterval; + time_t tConfRequested; uint64_t blockLevelLimit; uint32_t blockTimeLimit; uint32_t effectiveBlockLevelLimit; /* level limit adjusted by gateway settings */ @@ -201,6 +203,7 @@ struct rsksistatefile { #define getIVLenKSI(bh) (hashOutputLengthOctetsKSI((bh)->hashID)) #define rsksiSetBlockLevelLimit(ctx, limit) ((ctx)->blockLevelLimit = (ctx)->effectiveBlockLevelLimit = limit) #define rsksiSetBlockTimeLimit(ctx, limit) ((ctx)->blockTimeLimit = limit) +#define rsksiSetConfInterval(ctx, val) ((ctx)->confInterval = val) #define rsksiSetKeepRecordHashes(ctx, val) ((ctx)->bKeepRecordHashes = val) #define rsksiSetKeepTreeHashes(ctx, val) ((ctx)->bKeepTreeHashes = val) #define rsksiSetFileFormat(ctx, val) ((ctx)->fileFormat = val) diff --git a/runtime/lmsig_ksi-ls12.c b/runtime/lmsig_ksi-ls12.c index a2414254f3..9ac57b7385 100644 --- a/runtime/lmsig_ksi-ls12.c +++ b/runtime/lmsig_ksi-ls12.c @@ -49,6 +49,7 @@ static struct cnfparamdescr cnfpdescr[] = { { "sig.aggregator.hmacAlg", eCmdHdlrGetWord, 0 }, { "sig.block.levelLimit", eCmdHdlrSize, CNFPARAM_REQUIRED}, { "sig.block.timeLimit", eCmdHdlrInt, 0}, + { "sig.confinterval", eCmdHdlrInt, 0}, { "sig.keeprecordhashes", eCmdHdlrBinary, 0 }, { "sig.keeptreehashes", eCmdHdlrBinary, 0}, { "sig.fileformat", eCmdHdlrString, 0}, @@ -168,6 +169,14 @@ SetCnfParam(void *pT, struct nvlst *lst) } else { rsksiSetBlockTimeLimit(pThis->ctx, pvals[i].val.d.n); } + } else if (!strcmp(pblk.descr[i].name, "sig.confinterval")) { + if (pvals[i].val.d.n < 0) { + LogError(0, RS_RET_ERR, "sig.confinterval " + "%llu invalid - signing disabled", pvals[i].val.d.n); + pThis->ctx->disabled = true; + } else { + rsksiSetConfInterval(pThis->ctx, pvals[i].val.d.n); + } } else if (!strcmp(pblk.descr[i].name, "sig.keeprecordhashes")) { rsksiSetKeepRecordHashes(pThis->ctx, pvals[i].val.d.n); } else if(!strcmp(pblk.descr[i].name, "sig.keeptreehashes")) { From a1bcf143047834899a51d31979997e3f08e2af65 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Wed, 27 Apr 2022 14:59:44 +0200 Subject: [PATCH 121/134] OpenSSL: improved dbeug output after handshake success --- runtime/nsd_ossl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 2f1e851318..8a72d19760 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -1443,13 +1443,16 @@ osslPostHandshakeCheck(nsd_ossl_t *pNsd) "Information, no shared curve between syslog client and server"); } #endif + dbgprintf("osslPostHandshakeCheck: Debug Protocol Version: %s\n", + SSL_get_version(pNsd->ssl)); + sslCipher = (const SSL_CIPHER*) SSL_get_current_cipher(pNsd->ssl); if (sslCipher != NULL){ if(SSL_CIPHER_get_version(sslCipher) == NULL) { LogError(0, RS_RET_NO_ERRCODE, "nsd_ossl:" "TLS version mismatch between syslog client and server."); } - dbgprintf("osslPostHandshakeCheck: Debug Version: %s Name: %s\n", + dbgprintf("osslPostHandshakeCheck: Debug Cipher Version: %s Name: %s\n", SSL_CIPHER_get_version(sslCipher), SSL_CIPHER_get_name(sslCipher)); }else { LogError(0, RS_RET_NO_ERRCODE, "nsd_ossl:No shared ciphers between syslog client and server."); From dc36db742a7c59a3880d8fe8318673b5cfdf5c76 Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Wed, 15 Jun 2022 15:42:01 +0200 Subject: [PATCH 122/134] openssl: Add support to split tls commands by semicolon - Add support to split tls commands by semicolon. - Changed one test with multiple tls commands to use semicolon as seperator instead of newline. closes: https://github.com/rsyslog/rsyslog/issues/4852 --- runtime/nsd_ossl.c | 1 + tests/sndrcv_tls_ossl_anon_ciphers.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 8a72d19760..83a367bf53 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -1915,6 +1915,7 @@ applyGnutlsPriorityString(nsd_ossl_t *const pThis) pszCmd = strndup(pCurrentPos, pNextPos-pCurrentPos); pCurrentPos = pNextPos+1; pNextPos = index(pCurrentPos, '\n'); + pNextPos = (pNextPos == NULL ? index(pCurrentPos, ';') : pNextPos); pszValue = (pNextPos == NULL ? strdup(pCurrentPos) : strndup(pCurrentPos, pNextPos - pCurrentPos)); diff --git a/tests/sndrcv_tls_ossl_anon_ciphers.sh b/tests/sndrcv_tls_ossl_anon_ciphers.sh index b2088594e8..80201edd6c 100755 --- a/tests/sndrcv_tls_ossl_anon_ciphers.sh +++ b/tests/sndrcv_tls_ossl_anon_ciphers.sh @@ -18,7 +18,7 @@ module( load="../plugins/imtcp/.libs/imtcp" StreamDriver.Mode="1" StreamDriver.AuthMode="anon" StreamDriver.PermitExpiredCerts="off" - gnutlsPriorityString="CipherString=ECDHE-RSA-AES256-SHA384\nCiphersuites=TLS_AES_256_GCM_SHA384" + gnutlsPriorityString="CipherString=ECDHE-RSA-AES256-SHA384;Ciphersuites=TLS_AES_256_GCM_SHA384" ) input( type="imtcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port") @@ -40,7 +40,7 @@ action( type="omfwd" port="'$PORT_RCVR'" StreamDriverMode="1" StreamDriverAuthMode="anon" - gnutlsPriorityString="CipherString=ECDHE-RSA-AES256-SHA384\nCiphersuites=TLS_AES_128_GCM_SHA256" + gnutlsPriorityString="CipherString=ECDHE-RSA-AES256-SHA384;Ciphersuites=TLS_AES_128_GCM_SHA256" ) ' 2 startup 2 From 7cb9038288a1fe7cdea79c083e07f3637222abfa Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Mon, 1 Aug 2022 14:23:09 +0200 Subject: [PATCH 123/134] openssl: add more details to LogError in osslLastSSLErrorMsg - Avoid LogMsg outputs osslEndSess on successfull terminated connection. Only LogMsg if the connection was terminated unsuccessfully. - Handle SSL_ERROR_SYSCALL in both Send / osslRecordRecv, do not log as error if underlaying socket was terminated (ECONNRESET). Log as information instead. closes: https://github.com/rsyslog/rsyslog/issues/4946 --- runtime/nsd_ossl.c | 101 +++++++++++++++++++++++++++++---------------- runtime/nsd_ossl.h | 2 +- 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/runtime/nsd_ossl.c b/runtime/nsd_ossl.c index 2f1e851318..e8615d7038 100644 --- a/runtime/nsd_ossl.c +++ b/runtime/nsd_ossl.c @@ -185,7 +185,7 @@ int opensslh_THREAD_cleanup(void) /*-------------------------------------- MT OpenSSL helpers -----------------------------------------*/ /*--------------------------------------OpenSSL helpers ------------------------------------------*/ -void osslLastSSLErrorMsg(int ret, SSL *ssl, int severity, const char* pszCallSource) +void osslLastSSLErrorMsg(int ret, SSL *ssl, int severity, const char* pszCallSource, const char* pszOsslApi) { unsigned long un_error = 0; int iSSLErr = 0; @@ -197,10 +197,15 @@ void osslLastSSLErrorMsg(int ret, SSL *ssl, int severity, const char* pszCallSou iSSLErr = SSL_get_error(ssl, ret); /* Output error message */ - LogMsg(0, RS_RET_NO_ERRCODE, severity, "%s Error in '%s': '%s(%d)' with ret=%d\n", + LogMsg(0, RS_RET_NO_ERRCODE, severity, + "%s Error in '%s': '%s(%d)' with ret=%d, errno=%d, sslapi='%s'\n", (iSSLErr == SSL_ERROR_SSL ? "SSL_ERROR_SSL" : (iSSLErr == SSL_ERROR_SYSCALL ? "SSL_ERROR_SYSCALL" : "SSL_ERROR_UNKNOWN")), - pszCallSource, ERR_error_string(iSSLErr, NULL), iSSLErr, ret); + pszCallSource, ERR_error_string(iSSLErr, NULL), + iSSLErr, + ret, + errno, + pszOsslApi); } /* Loop through ERR_get_error */ @@ -451,7 +456,6 @@ osslRecordRecv(nsd_ossl_t *pThis) ssize_t lenRcvd; DEFiRet; int err; - int local_errno; ISOBJ_TYPE_assert(pThis, nsd_ossl); DBGPRINTF("osslRecordRecv: start\n"); @@ -488,22 +492,27 @@ osslRecordRecv(nsd_ossl_t *pThis) DBGPRINTF("osslRecordRecv: SSL_ERROR_ZERO_RETURN received, connection may closed already\n"); ABORT_FINALIZE(RS_RET_RETRY); } + else if(err == SSL_ERROR_SYSCALL) { + /* Output error and abort */ + osslLastSSLErrorMsg(lenRcvd, pThis->ssl, LOG_INFO, "osslRecordRecv", "SSL_read"); + iRet = RS_RET_NO_ERRCODE; + /* Check for underlaying socket errors **/ + if ( errno == ECONNRESET) { + dbgprintf("osslRecordRecv: SSL_ERROR_SYSCALL Errno %d, connection reset by peer\n", + errno); + /* Connection was dropped from remote site */ + iRet = RS_RET_CLOSED; + } else { + DBGPRINTF("osslRecordRecv: SSL_ERROR_SYSCALLErrno %d\n", errno); + } + ABORT_FINALIZE(iRet); + } else if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) { DBGPRINTF("osslRecordRecv: SSL_get_error #1 = %d, lenRcvd=%zd\n", err, lenRcvd); - /* Save errno */ - local_errno = errno; - /* Output OpenSSL error*/ - osslLastSSLErrorMsg(lenRcvd, pThis->ssl, LOG_ERR, "osslRecordRecv"); - /* Check for underlaying socket errors **/ - if (local_errno == ECONNRESET) { - DBGPRINTF("osslRecordRecv: Errno %d, connection reset by peer\n", local_errno); - ABORT_FINALIZE(RS_RET_CLOSED); - } else { - DBGPRINTF("osslRecordRecv: Errno %d\n", local_errno); - ABORT_FINALIZE(RS_RET_NO_ERRCODE); - } + osslLastSSLErrorMsg(lenRcvd, pThis->ssl, LOG_ERR, "osslRecordRecv", "SSL_read"); + ABORT_FINALIZE(RS_RET_NO_ERRCODE); } else { DBGPRINTF("osslRecordRecv: SSL_get_error #2 = %d, lenRcvd=%zd\n", err, lenRcvd); pThis->rtryCall = osslRtry_recv; @@ -530,7 +539,7 @@ osslInitSession(nsd_ossl_t *pThis, osslSslState_t osslType) /* , nsd_ossl_t *pSe if(!(pThis->ssl = SSL_new(pThis->ctx))) { pThis->ssl = NULL; - osslLastSSLErrorMsg(0, pThis->ssl, LOG_ERR, "osslInitSession"); + osslLastSSLErrorMsg(0, pThis->ssl, LOG_ERR, "osslInitSession", "SSL_new"); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } @@ -708,7 +717,8 @@ osslChkOnePeerName(nsd_ossl_t *pThis, X509 *pCert, uchar *pszPeerID, int *pbFoun *pbFoundPositiveMatch = 1; break; } else if ( osslRet < 0 ) { - osslLastSSLErrorMsg(osslRet, pThis->ssl, LOG_ERR, "osslChkOnePeerName"); + osslLastSSLErrorMsg(osslRet, pThis->ssl, LOG_ERR, "osslChkOnePeerName", + "X509_check_host"); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } #endif @@ -954,9 +964,8 @@ osslEndSess(nsd_ossl_t *pThis) err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) { /* Output Warning only */ - osslLastSSLErrorMsg(ret, pThis->ssl, LOG_WARNING, "osslEndSess"); + osslLastSSLErrorMsg(ret, pThis->ssl, LOG_WARNING, "osslEndSess", "SSL_shutdown"); } - /* Shutdown not finished, call SSL_read to do a bidirectional shutdown, see doc for more: * https://www.openssl.org/docs/man1.1.1/man3/SSL_shutdown.html */ @@ -964,13 +973,17 @@ osslEndSess(nsd_ossl_t *pThis) int iBytesRet = SSL_read(pThis->ssl, rcvBuf, NSD_OSSL_MAX_RCVBUF); DBGPRINTF("osslEndSess: Forcing ssl shutdown SSL_read (%d) to do a bidirectional shutdown\n", iBytesRet); - LogMsg(0, RS_RET_NO_ERRCODE, LOG_INFO, "nsd_ossl:" - "TLS session terminated with remote syslog server '%s': End Session", fromHostIP); - DBGPRINTF("osslEndSess: session closed (un)successfully \n"); + if (ret < 0) { + /* Unsuccessful shutdown, log as INFO */ + LogMsg(0, RS_RET_NO_ERRCODE, LOG_INFO, "nsd_ossl: " + "TLS session terminated successfully to remote syslog server '%s' with SSL Error '%d':" + " End Session", fromHostIP, ret); + } + dbgprintf( "osslEndSess: TLS session terminated successfully to remote syslog server '%s'" + " End Session", fromHostIP); } else { - LogMsg(0, RS_RET_NO_ERRCODE, LOG_INFO, "nsd_ossl:" - "TLS session terminated with remote syslog server '%s': End Session", fromHostIP); - DBGPRINTF("osslEndSess: session closed successfully \n"); + dbgprintf("osslEndSess: TLS session terminated successfully with remote syslog server '%s':" + " End Session", fromHostIP); } /* Session closed */ @@ -1279,7 +1292,7 @@ osslInit_ctx(nsd_ossl_t *const pThis) "Check at least: 1) file path is correct, 2) file exist, " "3) permissions are correct, 4) file content is correct. " "Open ssl error info may follow in next messages"); - osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit"); + osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit", "SSL_CTX_load_verify_locations"); ABORT_FINALIZE(RS_RET_TLS_CERT_ERR); } if(bHaveCert == 1 && SSL_CTX_use_certificate_chain_file(pThis->ctx, certFile) != 1) { @@ -1287,7 +1300,7 @@ osslInit_ctx(nsd_ossl_t *const pThis) "Check at least: 1) file path is correct, 2) file exist, " "3) permissions are correct, 4) file content is correct. " "Open ssl error info may follow in next messages"); - osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit"); + osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit", "SSL_CTX_use_certificate_chain_file"); ABORT_FINALIZE(RS_RET_TLS_CERT_ERR); } if(bHaveKey == 1 && SSL_CTX_use_PrivateKey_file(pThis->ctx, keyFile, SSL_FILETYPE_PEM) != 1) { @@ -1295,7 +1308,7 @@ osslInit_ctx(nsd_ossl_t *const pThis) "Check at least: 1) file path is correct, 2) file exist, " "3) permissions are correct, 4) file content is correct. " "Open ssl error info may follow in next messages"); - osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit"); + osslLastSSLErrorMsg(0, NULL, LOG_ERR, "osslGlblInit", "SSL_CTX_use_PrivateKey_file"); ABORT_FINALIZE(RS_RET_TLS_KEY_ERR); } @@ -1488,13 +1501,15 @@ osslHandshakeCheck(nsd_ossl_t *pNsd) } else if(resErr == SSL_ERROR_SYSCALL) { dbgprintf("osslHandshakeCheck: OpenSSL Server handshake failed with SSL_ERROR_SYSCALL " "- Aborting handshake.\n"); - osslLastSSLErrorMsg(res, pNsd->ssl, LOG_WARNING, "osslHandshakeCheck Server"); + osslLastSSLErrorMsg(res, pNsd->ssl, LOG_WARNING, "osslHandshakeCheck Server", + "SSL_accept"); LogMsg(0, RS_RET_NO_ERRCODE, LOG_WARNING, "nsd_ossl:TLS session terminated with remote client '%s': " "Handshake failed with SSL_ERROR_SYSCALL", fromHostIP); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } else { - osslLastSSLErrorMsg(res, pNsd->ssl, LOG_ERR, "osslHandshakeCheck Server"); + osslLastSSLErrorMsg(res, pNsd->ssl, LOG_ERR, "osslHandshakeCheck Server", + "SSL_accept"); LogMsg(0, RS_RET_NO_ERRCODE, LOG_WARNING, "nsd_ossl:TLS session terminated with remote client '%s': " "Handshake failed with error code: %d", fromHostIP, resErr); @@ -1517,10 +1532,12 @@ osslHandshakeCheck(nsd_ossl_t *pNsd) } else if(resErr == SSL_ERROR_SYSCALL) { dbgprintf("osslHandshakeCheck: OpenSSL Client handshake failed with SSL_ERROR_SYSCALL " "- Aborting handshake.\n"); - osslLastSSLErrorMsg(res, pNsd->ssl, LOG_WARNING, "osslHandshakeCheck Client"); + osslLastSSLErrorMsg(res, pNsd->ssl, LOG_WARNING, "osslHandshakeCheck Client", + "SSL_do_handshake"); ABORT_FINALIZE(RS_RET_NO_ERRCODE /*RS_RET_RETRY*/); } else { - osslLastSSLErrorMsg(res, pNsd->ssl, LOG_ERR, "osslHandshakeCheck Client"); + osslLastSSLErrorMsg(res, pNsd->ssl, LOG_ERR, "osslHandshakeCheck Client", + "SSL_do_handshake"); LogMsg(0, RS_RET_NO_ERRCODE, LOG_WARNING, "nsd_ossl:TLS session terminated with remote syslog server '%s':" "Handshake failed with error code: %d", fromHostIP, resErr); @@ -1750,10 +1767,24 @@ Send(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) DBGPRINTF("Send: SSL_ERROR_ZERO_RETURN received, retry next time\n"); ABORT_FINALIZE(RS_RET_RETRY); } + else if(err == SSL_ERROR_SYSCALL) { + /* Output error and abort */ + osslLastSSLErrorMsg(iSent, pThis->ssl, LOG_INFO, "Send", "SSL_write"); + iRet = RS_RET_NO_ERRCODE; + /* Check for underlaying socket errors **/ + if ( errno == ECONNRESET) { + dbgprintf("Send: SSL_ERROR_SYSCALL Connection was reset by remote\n"); + /* Connection was dropped from remote site */ + iRet = RS_RET_CLOSED; + } else { + DBGPRINTF("Send: SSL_ERROR_SYSCALLErrno %d\n", errno); + } + ABORT_FINALIZE(iRet); + } else if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) { /* Output error and abort */ - osslLastSSLErrorMsg(iSent, pThis->ssl, LOG_ERR, "Send"); + osslLastSSLErrorMsg(iSent, pThis->ssl, LOG_ERR, "Send", "SSL_write"); ABORT_FINALIZE(RS_RET_NO_ERRCODE); } else { /* Check for SSL Shutdown */ @@ -1945,7 +1976,7 @@ applyGnutlsPriorityString(nsd_ossl_t *const pThis) LogError(0, RS_RET_SYS_ERR, "Error: setting openssl command parameters: %s" "Open ssl error info may follow in next messages", pThis->gnutlsPriorityString); - osslLastSSLErrorMsg(0, NULL, LOG_ERR, "SetGnutlsPriorityString"); + osslLastSSLErrorMsg(0, NULL, LOG_ERR, "SetGnutlsPriorityString", "SSL_CONF_CTX_finish"); } SSL_CONF_CTX_free(cctx); } diff --git a/runtime/nsd_ossl.h b/runtime/nsd_ossl.h index 3779fee586..c55cf55f1c 100644 --- a/runtime/nsd_ossl.h +++ b/runtime/nsd_ossl.h @@ -105,7 +105,7 @@ rsRetVal osslRecordRecv(nsd_ossl_t *pThis); rsRetVal osslHandshakeCheck(nsd_ossl_t *pNsd); /* some more prototypes to avoid warnings ... */ -void osslLastSSLErrorMsg(int ret, SSL *ssl, int severity, const char* pszCallSource); +void osslLastSSLErrorMsg(int ret, SSL *ssl, int severity, const char* pszCallSource, const char* pszOsslApi); int verify_callback(int status, X509_STORE_CTX *store); rsRetVal osslPostHandshakeCheck(nsd_ossl_t *pNsd); From a335ec06f0897a71356afee3362f67e68b91a3de Mon Sep 17 00:00:00 2001 From: Andre lorbach Date: Thu, 28 Jul 2022 16:17:41 +0200 Subject: [PATCH 124/134] mmanon: Simplified and fixed IPv4 digit detection. - Fixed an issue with numbers above int64 in syntax_ipv4. Numbers that were up to 256 above the max of an int64 could incorrectly be detected as valid ipv4 digit. - Simplified the IPv4 digit detection function and renamed to isPosByte. - added testcasse for malformed IPvc4 addresses closes: https://github.com/rsyslog/rsyslog/issues/4940 --- plugins/mmanon/mmanon.c | 55 ++++++++++++++------------ tests/Makefile.am | 2 + tests/mmanon_recognize_ipv4.sh | 4 ++ tests/mmanon_simple_mallformed_ipv4.sh | 37 +++++++++++++++++ 4 files changed, 73 insertions(+), 25 deletions(-) create mode 100755 tests/mmanon_simple_mallformed_ipv4.sh diff --git a/plugins/mmanon/mmanon.c b/plugins/mmanon/mmanon.c index 607787ca02..811b763d75 100644 --- a/plugins/mmanon/mmanon.c +++ b/plugins/mmanon/mmanon.c @@ -22,6 +22,7 @@ #include "config.h" #include "rsyslog.h" #include +#include #include #include #include @@ -388,72 +389,76 @@ getHexVal(char c) } -/* returns -1 if no integer found, else integer */ -static int64_t -getPosInt(const uchar *const __restrict__ buf, +/* returns 1 if valid IPv4 digit, 0 if not */ +static int +isPosByte(const uchar *const __restrict__ buf, const size_t buflen, size_t *const __restrict__ nprocessed) { - int64_t val = 0; + int val = 0; /* Default means no byte found */ size_t i; - for(i = 0 ; i < buflen ; i++) { - if('0' <= buf[i] && buf[i] <= '9') - val = val*10 + buf[i]-'0'; - else + for(i = 0 ; i < buflen; i++) { + if('0' <= buf[i] && buf[i] <= '9') { + /* Maximum 3 digits for single IPv4 Number, we only copy up to 4 numbers + * but process forward to non digits */ + if (i < 4) { + val = val*10 + buf[i]-'0'; + } + } else break; } *nprocessed = i; - if(i == 0) - val = -1; - return val; + /* Return 1 if more than 1 and less the 4 digits and between 0 and 255 */ + if( i > 0 && + i < 4 && + (val >= 0 && val <= 255)) { + return 1; + } else { + return 0; + } } /* 1 - is IPv4, 0 not */ - static int syntax_ipv4(const uchar *const __restrict__ buf, const size_t buflen, size_t *const __restrict__ nprocessed) { - int64_t val; - size_t nproc; + size_t nproc = 0; size_t i; int r = 0; - - val = getPosInt(buf, buflen, &i); - if(val < 0 || val > 255) + if(isPosByte(buf, buflen, &i) == 0) { goto done; - + } if(i >= buflen || buf[i] != '.') { goto done; } i++; - val = getPosInt(buf+i, buflen-i, &nproc); - if(val < 0 || val > 255) + if(isdigit(buf[i]) == 0 || isPosByte(buf+i, buflen-i, &nproc) == 0) { goto done; + } i += nproc; if(i >= buflen || buf[i] != '.') { goto done; } i++; - val = getPosInt(buf+i, buflen-i, &nproc); - if(val < 0 || val > 255) + if(isdigit(buf[i]) == 0 || isPosByte(buf+i, buflen-i, &nproc) == 0) { goto done; + } i += nproc; if(i >= buflen || buf[i] != '.') { goto done; } i++; - val = getPosInt(buf+i, buflen-i, &nproc); - if(val < 0 || val > 255) + if(isdigit(buf[i]) == 0 || isPosByte(buf+i, buflen-i, &nproc) == 0) { goto done; + } i += nproc; *nprocessed = i; r = 1; - done: return r; } diff --git a/tests/Makefile.am b/tests/Makefile.am index 2c5e27e0a6..a807133fb5 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -617,6 +617,7 @@ TESTS += \ mmanon_simple_12_ipv4.sh \ mmanon_simple_33_ipv4.sh \ mmanon_simple_8_ipv4.sh \ + mmanon_simple_mallformed_ipv4.sh \ mmanon_random_128_ipv6.sh \ mmanon_zero_128_ipv6.sh \ mmanon_zero_96_ipv6.sh \ @@ -1966,6 +1967,7 @@ EXTRA_DIST= \ mmanon_simple_12_ipv4.sh \ mmanon_simple_33_ipv4.sh \ mmanon_simple_8_ipv4.sh \ + mmanon_simple_mallformed_ipv4.sh \ mmanon_random_128_ipv6.sh \ mmanon_zero_128_ipv6.sh \ mmanon_zero_96_ipv6.sh \ diff --git a/tests/mmanon_recognize_ipv4.sh b/tests/mmanon_recognize_ipv4.sh index 211befc197..05c9eff2ae 100755 --- a/tests/mmanon_recognize_ipv4.sh +++ b/tests/mmanon_recognize_ipv4.sh @@ -2,6 +2,10 @@ # add 2016-11-22 by Jan Gerhards, released under ASL 2.0 . ${srcdir:=.}/diag.sh init + +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debuglog" + generate_conf add_conf ' template(name="outfmt" type="string" string="%msg%\n") diff --git a/tests/mmanon_simple_mallformed_ipv4.sh b/tests/mmanon_simple_mallformed_ipv4.sh new file mode 100755 index 0000000000..7ef8899f66 --- /dev/null +++ b/tests/mmanon_simple_mallformed_ipv4.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# add 2022-07-28 by Andre Lorbach, released under ASL 2.0 + +. ${srcdir:=.}/diag.sh init +#export USE_VALGRIND="YES" # this test only makes sense with valgrind enabled +#export RS_TEST_VALGRIND_EXTRA_OPTS="--keep-debuginfo=yes" + +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="$RSYSLOG_DYNNAME.debuglog" + +generate_conf +add_conf ' +template(name="outfmt" type="string" string="%msg%\n") + +module(load="../plugins/mmanon/.libs/mmanon") +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="0" listenPortFileName="'$RSYSLOG_DYNNAME'.tcpflood_port" ruleset="testing") + +ruleset(name="testing") { + action(type="mmanon" ipv4.bits="32" ipv4.mode="simple") + action(type="omfile" file=`echo $RSYSLOG_OUT_LOG` template="outfmt") +}' + +startup +tcpflood -m1 -M "\"<129>Mar 10 01:00:00 172.20.245.8 tag: 165874883373.1.15599155266856607338.91@whatever +<129>Mar 10 01:00:00 172.20.245.8 tag: 1.165874883373.15599155266856607338.91@whatever +<129>Mar 10 01:00:00 172.20.245.8 tag: 15599155266856607338.165874883373.1.91@whatever +<129>Mar 10 01:00:00 172.20.245.8 tag: 91.165874883373.1.15599155266856607338.@whatever\"" + +shutdown_when_empty +wait_shutdown +export EXPECTED=' 165874883373.1.15599155266856607338.91@whatever + 1.165874883373.15599155266856607338.91@whatever + 15599155266856607338.165874883373.1.91@whatever + 91.165874883373.1.15599155266856607338.@whatever' +cmp_exact +exit_test From 6cf8260f854a470a9b6ace9dbeceb332bdbd0377 Mon Sep 17 00:00:00 2001 From: Victor Kustov Date: Wed, 3 Aug 2022 18:45:50 +0300 Subject: [PATCH 125/134] omclickhouse: capture additional exceptions - DB::NetException - DB::ParsingExceptions Signed-off-by: Victor Kustov --- plugins/omclickhouse/omclickhouse.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/omclickhouse/omclickhouse.c b/plugins/omclickhouse/omclickhouse.c index e75f6507b5..2c67e5e223 100644 --- a/plugins/omclickhouse/omclickhouse.c +++ b/plugins/omclickhouse/omclickhouse.c @@ -377,7 +377,9 @@ checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) { DEFiRet; - if(strstr(pWrkrData->reply, " = DB::Exception") != NULL) { + if((strstr(pWrkrData->reply, " = DB::Exception" ) != NULL) + || (strstr(pWrkrData->reply, "DB::NetException" ) != NULL) + || (strstr(pWrkrData->reply, "DB::ParsingException" ) != NULL)) { dbgprintf("omclickhouse: action failed with error: %s\n", pWrkrData->reply); iRet = RS_RET_DATAFAIL; } From 74787c6a4fc8ab665e6221ee9c2445566201f4b8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Aug 2022 12:52:52 +0200 Subject: [PATCH 126/134] maintain ChangeLog --- ChangeLog | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ChangeLog b/ChangeLog index a8696d1bdc..0f7cbc8302 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,35 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-08-04: openssl: add support to split tls commands by semicolon + - Add support to split tls commands by semicolon. + - Changed one test with multiple tls commands to use semicolon as + seperator instead of newline. + closes: https://github.com/rsyslog/rsyslog/issues/4852 +- 2022-08-04: openssl subsystem bugfix: build issue on Solaris + Needed header file was added. Platforms other than Solaris did not actually need it, + so this bug was discovered late. + Thanks to Jakub Kulík for the patch. + Import when index() is used. +- 2022-08-04: openssl: add more details to error messages + - Avoid LogMsg outputs osslEndSess on successfull terminated + connection. Only LogMsg if the connection was terminated + unsuccessfully. + - Handle SSL_ERROR_SYSCALL in both Send / osslRecordRecv, + do not log as error if underlaying socket was terminated + (ECONNRESET). Log as information instead. + closes: https://github.com/rsyslog/rsyslog/issues/4946 +- 2022-08-04: omclickhouse: capture additional exceptions + - DB::NetException + - DB::ParsingExceptions + Thanks to Victor Kustov for the patch. +- 2022-08-04: mmanon bugfix: Simplified and fixed IPv4 digit detection. + - Fixed an issue with numbers above int64 in syntax_ipv4. + Numbers that were up to 256 above the max of an int64 + could incorrectly be detected as valid ipv4 digit. + - Simplified the IPv4 digit detection function and renamed + to isPosByte. + - added testcasse for malformed IPvc4 addresses + closes: https://github.com/rsyslog/rsyslog/issues/4940 - 2022-07-21: imptcp: slight tuning - reduce indirect addressing to obtain more speed - also a fix for an annoying typo From 6be9a266ea73466840fa6571f86807069e63340d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 27 May 2022 13:08:04 +0200 Subject: [PATCH 127/134] testbench: add testcase for frequent imfile input file change PoC test, yet incomplete This patch not only contains the new test but also supporting changes to testbench tooling. see also: https://github.com/rsyslog/rsyslog/issues/4797 --- tests/Makefile.am | 2 + tests/imfile-logrotate-async.sh | 79 +++++++++++++++++++++++++ tests/inputfilegen.c | 102 ++++++++++++++++++++++++++++++-- 3 files changed, 178 insertions(+), 5 deletions(-) create mode 100755 tests/imfile-logrotate-async.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index a807133fb5..cd3b5b254f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1565,6 +1565,7 @@ TESTS += \ imfile-symlink.sh \ imfile-symlink-multi.sh \ imfile-logrotate.sh \ + imfile-logrotate-async.sh \ imfile-logrotate-multiple.sh \ imfile-logrotate-copytruncate.sh \ imfile-logrotate-nocopytruncate.sh \ @@ -2575,6 +2576,7 @@ EXTRA_DIST= \ imfile-symlink.sh \ imfile-symlink-multi.sh \ imfile-logrotate.sh \ + imfile-logrotate-async.sh \ imfile-logrotate-copytruncate.sh \ imfile-logrotate-nocopytruncate.sh \ imfile-logrotate-multiple.sh \ diff --git a/tests/imfile-logrotate-async.sh b/tests/imfile-logrotate-async.sh new file mode 100755 index 0000000000..1210b201c5 --- /dev/null +++ b/tests/imfile-logrotate-async.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# This is part of the rsyslog testbench, licensed under ASL 2.0 +. $srcdir/diag.sh check-inotify-only +. ${srcdir:=.}/diag.sh init +check_command_available logrotate +export NUMMESSAGES=10000 +export RETRIES=50 + +# Write logrotate config file +echo '"./'$RSYSLOG_DYNNAME'.input*.log" +{ + #daily + rotate 60 + missingok + notifempty + sharedscripts + postrotate + kill -HUP $(cat '$RSYSLOG_DYNNAME'.inputfilegen_pid) + endscript + #olddir /logs/old + +}' > $RSYSLOG_DYNNAME.logrotate + + +generate_conf +add_conf ' +$WorkDirectory '$RSYSLOG_DYNNAME'.spool + +global( debug.whitelist="off" + debug.files=["rainerscript.c", "ratelimit.c", "ruleset.c", "main Q", "msg.c", "../action.c"] + ) + +module(load="../plugins/imfile/.libs/imfile" mode="inotify" PollingInterval="2") + +input(type="imfile" File="./'$RSYSLOG_DYNNAME'.input*.log" Tag="file:" + Severity="error" Facility="local7" addMetadata="on" reopenOnTruncate="on") + +$template outfmt,"%msg:F,58:2%\n" +if $msg contains "msgnum:" then + action(type="omfile" file="'$RSYSLOG_OUT_LOG'" template="outfmt") +' +startup + +./inputfilegen -m $NUMMESSAGES -S 5 -B 500 -f $RSYSLOG_DYNNAME.input.log & +INPUTFILEGEN_PID=$! +echo "$INPUTFILEGEN_PID" > $RSYSLOG_DYNNAME.inputfilegen_pid + + + +./msleep 1 +logrotate --state $RSYSLOG_DYNNAME.logrotate.state -f $RSYSLOG_DYNNAME.logrotate +./msleep 20 +echo INPUT FILES: +ls -li $RSYSLOG_DYNNAME.input* +logrotate --state $RSYSLOG_DYNNAME.logrotate.state -f $RSYSLOG_DYNNAME.logrotate +./msleep 20 +echo INPUT FILES: +ls -li $RSYSLOG_DYNNAME.input* +logrotate --state $RSYSLOG_DYNNAME.logrotate.state -f $RSYSLOG_DYNNAME.logrotate +echo INPUT FILES: +ls -li $RSYSLOG_DYNNAME.input* +echo ls ${RSYSLOG_DYNNAME}.spool: +ls -li ${RSYSLOG_DYNNAME}.spool +echo INPUT FILES: +ls -li $RSYSLOG_DYNNAME.input* + +# generate more input after logrotate into new logfile +#./inputfilegen -m $TESTMESSAGES -i $TESTMESSAGES >> $RSYSLOG_DYNNAME.input.1.log +#ls -l $RSYSLOG_DYNNAME.input* + +#msgcount=$((2* TESTMESSAGES)) +#wait_file_lines $RSYSLOG_OUT_LOG $msgcount $RETRIES +wait_file_lines + +shutdown_when_empty +wait_shutdown +seq_check +#seq_check 0 $TESTMESSAGESFULL +exit_test diff --git a/tests/inputfilegen.c b/tests/inputfilegen.c index 523f82a7a9..ef61229ab1 100644 --- a/tests/inputfilegen.c +++ b/tests/inputfilegen.c @@ -1,5 +1,5 @@ /* generate an input file suitable for use by the testbench - * Copyright (C) 2018 by Rainer Gerhards and Adiscon GmbH. + * Copyright (C) 2018-2022 by Rainer Gerhards and Adiscon GmbH. * Copyright (C) 2016-2018 by Pascal Withopf and Adiscon GmbH. * * usage: ./inputfilegen num-lines > file @@ -14,9 +14,20 @@ * This is especially useful with -s, as the testbench otherwise does * not know how to do a seq_check. To keep things flexible, can also be * used with -m (this may aid testbench framework generalization). + * -f outputfile + * Permits to write data to file "outputfile" instead of stdout. Also + * enables support for SIGHUP. + * -S sleep time + * ms to sleep between sending message bulks (bulks size given by -B) + * -B number of messages in bulk + * number of messages to send without sleeping as specified in -S. + * IGNORED IF -S is not also given! * Part of rsyslog, licensed under ASL 2.0 */ +#include "config.h" #include +#include +#include #include #include #include @@ -25,10 +36,60 @@ #else #include #endif +#if defined(__FreeBSD__) +#include +#else +#include +#endif +#if defined(HAVE_SYS_SELECT_H) +#include +#endif #define DEFMSGS 5 #define NOEXTRADATA -1 +static volatile int bHadHUP = 0; +static void +hdlr_sighup(int sig) +{ + fprintf(stderr, "had hup, sig %d\n", sig); + bHadHUP = 1; +} +static void +sighup_enable() +{ + struct sigaction sigAct; + memset(&sigAct, 0, sizeof (sigAct)); + sigemptyset(&sigAct.sa_mask); + sigAct.sa_handler = hdlr_sighup; + sigaction(SIGHUP, &sigAct, NULL); +} + +void msleep(const int sleepTime) +{ + struct timeval tvSelectTimeout; + + tvSelectTimeout.tv_sec = sleepTime / 1000; + tvSelectTimeout.tv_usec = (sleepTime % 1000) * 1000; /* micro seconds */ + if(select(0, NULL, NULL, NULL, &tvSelectTimeout) == -1) { + if(errno != EINTR) { + perror("select"); + exit(1); + } + } +} + +static FILE * +open_output(const char *fn) +{ + FILE *fh_output = fopen(fn, "w"); + if(fh_output == NULL) { + perror(fn); + exit(1); + } + return fh_output; +} + int main(int argc, char* argv[]) { int c, i; @@ -39,8 +100,13 @@ int main(int argc, char* argv[]) long long filesize = -1; char *extradata = NULL; const char *msgcntfile = NULL; + const char *outputfile = "-"; + FILE *fh_output; + int sleep_ms = 0; + int sleep_msgs = 0; /* messages to xmit between sleeps (if configured) */ + int ctr = 0; - while((c=getopt(argc, argv, "m:M:i:d:s:")) != -1) { + while((c=getopt(argc, argv, "m:M:i:d:s:f:S:B:")) != -1) { switch(c) { case 'm': nmsgs = atoi(optarg); @@ -57,6 +123,16 @@ int main(int argc, char* argv[]) case 's': filesize = atoll(optarg); break; + case 'S': + sleep_ms = atoi(optarg); + break; + case 'B': + sleep_msgs = atoi(optarg); + break; + case 'f': + outputfile = optarg; + sighup_enable(); + break; case ':': fprintf(stderr, "Option -%c requires an operand\n", optopt); errflg++; @@ -97,17 +173,33 @@ int main(int argc, char* argv[]) fclose(fh); } + if(strcmp(outputfile, "-")) { + fh_output = open_output(outputfile); + } else { + fh_output = stdout; + } + if(nchars != NOEXTRADATA) { extradata = (char *)malloc(nchars + 1); memset(extradata, 'X', nchars); extradata[nchars] = '\0'; } for(i = nmsgstart; i < (nmsgs+nmsgstart); ++i) { - printf("msgnum:%8.8d:", i); + if(sleep_ms > 0 && ctr++ >= sleep_msgs) { + msleep(sleep_ms); + ctr = 0; + } + if(bHadHUP) { + fclose(fh_output); + fh_output = open_output(outputfile); + fprintf(stderr, "%s reopened\n", outputfile); + bHadHUP = 0; + } + fprintf(fh_output, "msgnum:%8.8d:", i); if(nchars != NOEXTRADATA) { - printf("%s", extradata); + fprintf(fh_output, "%s", extradata); } - printf("\n"); + fprintf(fh_output, "\n"); } free(extradata); return 0; From 5520373959c4513fc60a1be8785ea45f926231b5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 May 2022 12:03:18 +0200 Subject: [PATCH 128/134] imfile bugfix: message loss/duplication when monitored file is rotated When a to-be-monitored file is being rotated, some messages may be lost or duplicated. In case of duplication, many file lines may be duplicated depending on actual timing. The whole bug was primarily timing depenedent in general. It most often was visible in practice when the monitored file was very frequently rotated (we had some report with every few seconds). Note that while we try hard to not lose any messages, input file rotation always has some loss potential. This is inevitable if the monitored file is being truncated. Also note that this bugfix affects imfile, only. It has nothing to do and no relation to rsyslog output files being rotated on HUP. closes: https://github.com/rsyslog/rsyslog/issues/4797 --- plugins/imfile/imfile.c | 49 +++++++++++++++++++++------- runtime/stream.c | 58 +++------------------------------ runtime/stream.h | 5 --- tests/imfile-logrotate-async.sh | 7 ++-- 4 files changed, 47 insertions(+), 72 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 78f1cba0f9..3e28192844 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -84,6 +84,7 @@ MODULE_CNFNAME("imfile") /* defines */ #define FILE_ID_HASH_SIZE 20 /* max size of a file_id hash */ #define FILE_ID_SIZE 512 /* how many bytes are used for file-id? */ +#define FILE_DELETE_DELAY 5 /* how many seconds to wait before finally deleting a gone file */ /* Module static data */ DEF_IMOD_STATIC_DATA /* must be present, starts static data */ @@ -209,6 +210,7 @@ struct act_obj_s { ratelimit_t *ratelimiter; multi_submit_t multiSub; int is_symlink; + time_t time_to_delete; /* Helper variable to DELAY the actual file delete in act_obj_unlink */ }; struct fs_edge_s { fs_node_t *parent; /* node pointing to this edge */ @@ -774,6 +776,7 @@ act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file, act->file_id_prev[0] = '\0'; act->is_symlink = is_symlink; act->ratelimiter = NULL; + act->time_to_delete = 0; if (source) { /* we are target of symlink */ CHKmalloc(act->source_name = strdup(source)); } else { @@ -827,26 +830,49 @@ detect_updates(fs_edge_t *const edge) for(act = edge->active ; act != NULL ; act = act->next) { DBGPRINTF("detect_updates checking active obj '%s'\n", act->name); - const int r = lstat(act->name, &fileInfo); + // lstat() has the disadvantage, that we get "deleted" when the name has changed + // but inode is still the same (like with logrotate) + int r = lstat(act->name, &fileInfo); if(r == -1) { /* object gone away? */ - DBGPRINTF("object gone away, unlinking: '%s'\n", act->name); - act_obj_unlink(act); - restart = 1; + /* now let's see if the file itself already exist (e.g. rotated away) */ + /* NOTE: this will NOT stall the file. The reason is that when a new file + * with the same name is detected, we will not run into this code. + TODO: check the full implications, there are for sure some! + e.g. file has been closed, so we will never have old inode (but + why was it closed then? --> check) + */ + r = fstat(act->ino, &fileInfo); + if(r == -1) { + time_t ttNow; + time(&ttNow); + if (act->time_to_delete == 0) { + act->time_to_delete = ttNow; + } + /* First time we run into this code, we need to give imfile a little time to process + * the old file in case a process is still writing into it until the FILE_DELETE_DELAY + * is reached OR the inode has changed (see elseif below). In most cases, the + * delay will never be reached and the file will be closed when the inode has changed. + */ + if (act->time_to_delete + FILE_DELETE_DELAY < ttNow) { + DBGPRINTF("detect_updates obj gone away, unlinking: '%s', ttDelete: %ld/%ld\n", + act->name, act->time_to_delete, ttNow); + act_obj_unlink(act); + restart = 1; + } else { + DBGPRINTF("detect_updates obj gone away, keep '%s' open: %ld/%ld/%lds!\n", + act->name, act->time_to_delete, ttNow, ttNow - act->time_to_delete); + pollFile(act); + } + } break; } else if(fileInfo.st_ino != act->ino) { DBGPRINTF("file '%s' inode changed from %llu to %llu, unlinking from " "internal lists\n", act->name, (long long unsigned) act->ino, (long long unsigned) fileInfo.st_ino); - if(act->pStrm != NULL) { - /* we do no need to re-set later, as act_obj_unlink - * will destroy the strm obj */ - strmSet_checkRotation(act->pStrm, STRM_ROTATION_DO_NOT_CHECK); - } act_obj_unlink(act); restart = 1; break; } - } if (restart) { @@ -1103,7 +1129,8 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted) static void ATTR_NONNULL() act_obj_unlink(act_obj_t *act) { - DBGPRINTF("act_obj_unlink %p: %s, pStrm %p\n", act, act->name, act->pStrm); + DBGPRINTF("act_obj_unlink %p: %s, pStrm %p, ttDelete: %ld\n", + act, act->name, act->pStrm, act->time_to_delete); if(act->prev == NULL) { act->edge->active = act->next; } else { diff --git a/runtime/stream.c b/runtime/stream.c index 9cadcc6bab..e471a110f2 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -587,47 +587,6 @@ strmNextFile(strm_t *pThis) } -/* handle the eof case for monitored files. - * If we are monitoring a file, someone may have rotated it. In this case, we - * also need to close it and reopen it under the same name. - * rgerhards, 2008-02-13 - * The previous code also did a check for file truncation, in which case the - * file was considered rewritten. However, this potential border case turned - * out to be a big trouble spot on busy systems. It caused massive message - * duplication (I guess stat() can return a too-low number under some - * circumstances). So starting as of now, we only check the inode number and - * a file change is detected only if the inode changes. -- rgerhards, 2011-01-10 - */ -static rsRetVal ATTR_NONNULL() -strmHandleEOFMonitor(strm_t *const pThis) -{ - DEFiRet; - struct stat statName; - - ISOBJ_TYPE_assert(pThis, strm); - if(stat((char*) pThis->pszCurrFName, &statName) == -1) - ABORT_FINALIZE(RS_RET_IO_ERROR); - DBGPRINTF("strmHandleEOFMonitor: stream checking for file change on '%s', inode %u/%u size %llu/%llu\n", - pThis->pszCurrFName, (unsigned) pThis->inode, (unsigned) statName.st_ino, - (long long unsigned) pThis->iCurrOffs, (long long unsigned) statName.st_size); - - /* Inode unchanged but file size on disk is less than current offset - * means file was truncated, we also reopen if 'reopenOnTruncate' is on - */ - if (pThis->inode != statName.st_ino - || (pThis->bReopenOnTruncate && statName.st_size < pThis->iCurrOffs)) { - DBGPRINTF("we had a file change on '%s'\n", pThis->pszCurrFName); - CHKiRet(strmCloseFile(pThis)); - CHKiRet(strmOpenFile(pThis)); - } else { - ABORT_FINALIZE(RS_RET_EOF); - } - -finalize_it: - RETiRet; -} - - /* handle the EOF case of a stream * The EOF case is somewhat complicated, as the proper action depends on the * mode the stream is in. If there are multiple files (circular logs, most @@ -655,11 +614,8 @@ strmHandleEOF(strm_t *const pThis) case STREAMTYPE_FILE_MONITOR: DBGOPRINT((obj_t*) pThis, "file '%s' (%d) EOF, rotationCheck %d\n", pThis->pszCurrFName, pThis->fd, pThis->rotationCheck); - if(pThis->rotationCheck == STRM_ROTATION_DO_CHECK) { - CHKiRet(strmHandleEOFMonitor(pThis)); - } else { - ABORT_FINALIZE(RS_RET_EOF); - } +DBGPRINTF("RGER: EOF!\n"); + ABORT_FINALIZE(RS_RET_EOF); break; } @@ -776,6 +732,7 @@ strmReadBuf(strm_t *pThis, int *padBytes) } iLenRead = read(pThis->fd, pThis->pIOBuf, toRead); DBGOPRINT((obj_t*) pThis, "file %d read %ld bytes\n", pThis->fd, iLenRead); + DBGOPRINT((obj_t*) pThis, "file %d read %*s\n", pThis->fd, (unsigned) iLenRead, (char*) pThis->pIOBuf); /* end crypto */ if(iLenRead == 0) { CHKiRet(strmHandleEOF(pThis)); @@ -1025,6 +982,7 @@ strmReadLine(strm_t *const pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF } pThis->strtOffs = pThis->iCurrOffs; /* we are at begin of next line */ } else { +DBGPRINTF("RGER: strmReadLine iRet %d\n", iRet); if(*ppCStr != NULL) { if(cstrLen(*ppCStr) > 0) { /* we may have an empty string in an unsuccesfull poll or after restart! */ @@ -2161,14 +2119,6 @@ strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val) pThis->readTimeout = val; } -void ATTR_NONNULL() -strmSet_checkRotation(strm_t *const pThis, const int val) { - ISOBJ_TYPE_assert(pThis, strm); - assert(val == STRM_ROTATION_DO_CHECK || val == STRM_ROTATION_DO_NOT_CHECK); - pThis->rotationCheck = val; -} - - static rsRetVal ATTR_NONNULL() strmSetbDeleteOnClose(strm_t *const pThis, const int val) { diff --git a/runtime/stream.h b/runtime/stream.h index 4af3d3345b..38db1aba0c 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -97,10 +97,6 @@ typedef enum { STRM_COMPRESS_ZSTD = 1 } strm_compressionDriver_t; -/* settings for stream rotation (applies not to all processing modes!) */ -#define STRM_ROTATION_DO_CHECK 0 -#define STRM_ROTATION_DO_NOT_CHECK 1 - #define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */ /* The strm_t data structure */ struct strm_s { @@ -254,6 +250,5 @@ void strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val); const uchar * ATTR_NONNULL() strmGetPrevLineSegment(strm_t *const pThis); const uchar * ATTR_NONNULL() strmGetPrevMsgSegment(strm_t *const pThis); int ATTR_NONNULL() strmGetPrevWasNL(const strm_t *const pThis); -void ATTR_NONNULL() strmSet_checkRotation(strm_t *const pThis, const int val); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/tests/imfile-logrotate-async.sh b/tests/imfile-logrotate-async.sh index 1210b201c5..c15a69d24b 100755 --- a/tests/imfile-logrotate-async.sh +++ b/tests/imfile-logrotate-async.sh @@ -26,8 +26,8 @@ generate_conf add_conf ' $WorkDirectory '$RSYSLOG_DYNNAME'.spool -global( debug.whitelist="off" - debug.files=["rainerscript.c", "ratelimit.c", "ruleset.c", "main Q", "msg.c", "../action.c"] +global( debug.whitelist="on" + debug.files=["imfile.c", "stream.c"] ) module(load="../plugins/imfile/.libs/imfile" mode="inotify" PollingInterval="2") @@ -72,6 +72,9 @@ ls -li $RSYSLOG_DYNNAME.input* #wait_file_lines $RSYSLOG_OUT_LOG $msgcount $RETRIES wait_file_lines +touch $RSYSLOG_DYNNAME.input.log +./msleep 1000 + shutdown_when_empty wait_shutdown seq_check From 7897e76855d4c15b31e7a56087c7a04e456d1dbb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Aug 2022 15:17:08 +0200 Subject: [PATCH 129/134] maintain ChangeLog --- ChangeLog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0f7cbc8302..8b1c42d6da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,17 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-08-04: ksi bugfix: possible crash fixed when several log files are opened. + KSI module in async mode used to request aggregator conf. every time a log + file was opened. When several log files were opened simultaneously + corresponding amount of pointless concurrent conf. requests were posted. + Concurrent conf. requests lead to a bug in libksi, where internal count of + pending requests was not decremented correctly causing system to crash. + Fix for the issue is to optimize the frequency of conf. requests so that only + one conf. requests is handled at once. Instead of checking conf. every time + log file is opened, conf is requested periodically after conf timeout. This will + affect both sync and async mode. + New option for KSI module introduced - sig.confinterval="time, s". + Thanks to Taavi Valjaots for the patch. - 2022-08-04: openssl: add support to split tls commands by semicolon - Add support to split tls commands by semicolon. - Changed one test with multiple tls commands to use semicolon as From 9166e22f294cfb599598ba99608d16d3ca4280b7 Mon Sep 17 00:00:00 2001 From: "taavi.valjaots" Date: Thu, 28 Jul 2022 12:29:23 +0300 Subject: [PATCH 130/134] ksi bugfix: optimize processing of signer queue to fix delays. There is a worker queue where rsyslog KSI module collects events and signing requests. When queue is processed thread is periodically put to sleep. Previous implementation handles signature requests well but sleeps every time after handling new file open / close event. When several log files are opened or closed simultaneously process is significantly slowed down. Another issue is that thread always sleeps 1000ms that may be 2x longer than aggregation round. This slows down overall signing process. Fix for the issue is to simply not sleep after file open / close event if there are next items to be processed. To speed up the signing process, rsyslog uses KSI aggregator conf. to obtain the aggregation period that is used for the sleep time configuration. --- runtime/lib_ksils12.c | 87 ++++++++++++++++++++++++++++--------------- runtime/lib_ksils12.h | 1 + 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/runtime/lib_ksils12.c b/runtime/lib_ksils12.c index f5a11739a0..88722a0c58 100644 --- a/runtime/lib_ksils12.c +++ b/runtime/lib_ksils12.c @@ -692,44 +692,65 @@ ksiCreateFile(rsksictx ctx, const char *path, uid_t uid, gid_t gid, int mode, bo } static void handle_ksi_config(rsksictx ctx, KSI_AsyncService *as, KSI_Config *config) { - KSI_Integer *ksi_int = NULL; - uint64_t tmpInt, newLevel, res; + int res = KSI_UNKNOWN_ERROR; + KSI_Integer *intValue = NULL; - if (KSI_Config_getMaxRequests(config, &ksi_int) == KSI_OK && ksi_int != NULL) { - tmpInt = KSI_Integer_getUInt64(ksi_int); - ctx->max_requests=tmpInt; + if (KSI_Config_getMaxRequests(config, &intValue) == KSI_OK && intValue != NULL) { + ctx->max_requests = KSI_Integer_getUInt64(intValue); report(ctx, "KSI gateway has reported a max requests value of %llu", - (long long unsigned) tmpInt); + (long long unsigned) ctx->max_requests); if(as) { + /* libksi expects size_t. */ + size_t optValue = 0; + + optValue = ctx->max_requests; res = KSI_AsyncService_setOption(as, KSI_ASYNC_OPT_MAX_REQUEST_COUNT, - (void*) (ctx->max_requests)); + (void*)optValue); if(res != KSI_OK) reportKSIAPIErr(ctx, NULL, "KSI_AsyncService_setOption(max_request)", res); + optValue = 5 * ctx->max_requests; KSI_AsyncService_setOption(as, KSI_ASYNC_OPT_REQUEST_CACHE_SIZE, - (void*) (5*ctx->max_requests)); + (void*)optValue); } } - ksi_int = NULL; - if(KSI_Config_getMaxLevel(config, &ksi_int) == KSI_OK && ksi_int != NULL) { - tmpInt = KSI_Integer_getUInt64(ksi_int); + intValue = NULL; + if(KSI_Config_getMaxLevel(config, &intValue) == KSI_OK && intValue != NULL) { + uint64_t newLevel = 0; + newLevel = KSI_Integer_getUInt64(intValue); report(ctx, "KSI gateway has reported a max level value of %llu", - (long long unsigned) tmpInt); - newLevel=MIN(tmpInt, ctx->blockLevelLimit); + (long long unsigned) newLevel); + newLevel=MIN(newLevel, ctx->blockLevelLimit); if(ctx->effectiveBlockLevelLimit != newLevel) { report(ctx, "Changing the configured block level limit from %llu to %llu", (long long unsigned) ctx->effectiveBlockLevelLimit, (long long unsigned) newLevel); ctx->effectiveBlockLevelLimit = newLevel; } - else if(tmpInt < 2) { + else if(newLevel < 2) { report(ctx, "KSI gateway has reported an invalid level limit value (%llu), " - "plugin disabled", (long long unsigned) tmpInt); + "plugin disabled", (long long unsigned) newLevel); ctx->disabled = true; } } + + intValue = NULL; + if (KSI_Config_getAggrPeriod(config, &intValue) == KSI_OK && intValue != NULL) { + uint64_t newThreadSleep = 0; + newThreadSleep = KSI_Integer_getUInt64(intValue); + report(ctx, "KSI gateway has reported an aggregation period value of %llu", + (long long unsigned) newThreadSleep); + + newThreadSleep = MIN(newThreadSleep, ctx->threadSleepms); + if(ctx->threadSleepms != newThreadSleep) { + report(ctx, "Changing async signer thread sleep from %llu to %llu", + (long long unsigned) ctx->threadSleepms, + (long long unsigned) newThreadSleep); + ctx->threadSleepms = newThreadSleep; + } + } } static int @@ -873,6 +894,7 @@ rsksiCtxNew(void) { ctx->max_requests = (1 << 8); ctx->confInterval = 3600; ctx->tConfRequested = 0; + ctx->threadSleepms = 1000; ctx->errFunc = NULL; ctx->usrptr = NULL; ctx->fileUID = -1; @@ -1953,23 +1975,20 @@ request_async_config(rsksictx ctx, KSI_CTX *ksi_ctx, KSI_AsyncService *as) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" void *signer_thread(void *arg) { - + int res = KSI_UNKNOWN_ERROR; rsksictx ctx = (rsksictx) arg; - QueueItem *item = NULL; - size_t ksiFileCount = 0; - time_t timeout; - KSI_CTX *ksi_ctx; + KSI_CTX *ksi_ctx = NULL; KSI_AsyncService *as = NULL; - int res = 0; - bool ret = false; - int i = 0, endpoints = 0; + size_t ksiFileCount = 0; + int endpoints = 0; + bool bSleep = true; + CHECK_KSI_API(KSI_CTX_new(&ksi_ctx), ctx, "KSI_CTX_new"); CHECK_KSI_API(KSI_CTX_setAggregator(ksi_ctx, ctx->aggregatorUri, ctx->aggregatorId, ctx->aggregatorKey), ctx, "KSI_CTX_setAggregator"); - if(ctx->debugFile) { res = KSI_CTX_setLoggerCallback(ksi_ctx, rsksiStreamLogger, ctx->debugFile); if (res != KSI_OK) @@ -1987,6 +2006,7 @@ void *signer_thread(void *arg) { reportKSIAPIErr(ctx, NULL, "KSI_SigningAsyncService_new", res); } else { + int i = 0; for (i = 0; i < ctx->aggregatorEndpointCount; i++) { res = KSI_AsyncService_addEndpoint(as, ctx->aggregatorEndpoints[i], ctx->aggregatorId, ctx->aggregatorKey); @@ -2013,15 +2033,19 @@ void *signer_thread(void *arg) { ctx->signer_state = SIGNER_STARTED; while (true) { - timeout = 1; + QueueItem *item = NULL; if (isAggrConfNeeded(ctx)) { request_async_config(ctx, ksi_ctx, as); } /* Wait for a work item or timeout*/ - ProtectedQueue_waitForItem(ctx->signer_queue, NULL, timeout * 1000); - /* Check for block time limit*/ + if (bSleep) { + ProtectedQueue_waitForItem(ctx->signer_queue, NULL, ctx->threadSleepms); + } + bSleep = true; + + /* Check for block time limit. */ sigblkCheckTimeOut(ctx); /* in case there are no items go around*/ @@ -2033,8 +2057,7 @@ void *signer_thread(void *arg) { /* process signing requests only if there is an open signature file */ if(ksiFileCount > 0) { /* check for pending/unsent requests in asynchronous service */ - ret = process_requests_async(ctx, ksi_ctx, as); - if(!ret) { + if(!process_requests_async(ctx, ksi_ctx, as)) { // probably fatal error, disable signing, error should be already reported ctx->disabled = true; goto cleanup; @@ -2047,6 +2070,11 @@ void *signer_thread(void *arg) { /* Handle other types of work items */ if (ProtectedQueue_popFront(ctx->signer_queue, (void**) &item) != 0) { + /* There is no point to sleep after processing non request type item + * as there is great possibility that next item can already be + * processed. */ + bSleep = false; + if (item->type == QITEM_CLOSE_FILE) { if (item->file) { fclose(item->file); @@ -2070,6 +2098,7 @@ void *signer_thread(void *arg) { goto cleanup; } + free(item); } } diff --git a/runtime/lib_ksils12.h b/runtime/lib_ksils12.h index fb69fa95f5..a1c9011bac 100644 --- a/runtime/lib_ksils12.h +++ b/runtime/lib_ksils12.h @@ -95,6 +95,7 @@ struct rsksictx_s { uint64_t blockLevelLimit; uint32_t blockTimeLimit; uint32_t effectiveBlockLevelLimit; /* level limit adjusted by gateway settings */ + uint32_t threadSleepms; uint8_t syncMode; uid_t fileUID; /* IDs for creation */ uid_t dirUID; From 24a3f2dbd91dfb1fc780021643d21daa02b62eeb Mon Sep 17 00:00:00 2001 From: "taavi.valjaots" Date: Mon, 1 Aug 2022 13:46:25 +0300 Subject: [PATCH 131/134] ksi bugfix: request cache size and send timeout issue fixed. Async service send timeout is not configurable and request cache size is too small to handle large amount of signing requests with small amount of permitted requests per aggregation round. For example user with max_requests = 4 results cache size 5 * max_requests or at least 256. When signing 300 log files cache will be too small resulting several unsigned blocks. When signing 200 log file cache will be adequate, but with rate of 4 signatures per second, it is only possible to sign 4 * 10 blocks before all requests that are not sent out will timeout. Fix for the issue is to make send timeout configurable and make the size of the cache depend on the value of send timeout. New configuration value sig.block.signtimeout="time, s" introduced that defines the time window wherein the block has to be signed. The size of the request cache is increased to 3 * max_requests * sign_timeout or at least 256. --- runtime/lib_ksils12.c | 12 ++++++++++-- runtime/lib_ksils12.h | 2 ++ runtime/lmsig_ksi-ls12.c | 9 +++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/runtime/lib_ksils12.c b/runtime/lib_ksils12.c index 88722a0c58..5994994e7f 100644 --- a/runtime/lib_ksils12.c +++ b/runtime/lib_ksils12.c @@ -710,7 +710,7 @@ static void handle_ksi_config(rsksictx ctx, KSI_AsyncService *as, KSI_Config *co if(res != KSI_OK) reportKSIAPIErr(ctx, NULL, "KSI_AsyncService_setOption(max_request)", res); - optValue = 5 * ctx->max_requests; + optValue = 3 * ctx->max_requests * ctx->blockSigTimeout; KSI_AsyncService_setOption(as, KSI_ASYNC_OPT_REQUEST_CACHE_SIZE, (void*)optValue); } @@ -892,6 +892,7 @@ rsksiCtxNew(void) { ctx->bKeepTreeHashes = false; ctx->bKeepRecordHashes = true; ctx->max_requests = (1 << 8); + ctx->blockSigTimeout = 10; ctx->confInterval = 3600; ctx->tConfRequested = 0; ctx->threadSleepms = 1000; @@ -1979,6 +1980,7 @@ void *signer_thread(void *arg) { rsksictx ctx = (rsksictx) arg; KSI_CTX *ksi_ctx = NULL; KSI_AsyncService *as = NULL; + size_t size_t_value = 0; size_t ksiFileCount = 0; int endpoints = 0; bool bSleep = true; @@ -2028,8 +2030,14 @@ void *signer_thread(void *arg) { goto cleanup; } + /* Lets use buffer value, as libksi requires size_t. */ + size_t_value = ctx->max_requests; KSI_AsyncService_setOption(as, KSI_ASYNC_OPT_REQUEST_CACHE_SIZE, - (void*) (ctx->max_requests)); + (void*)size_t_value); + size_t_value = ctx->blockSigTimeout; + KSI_AsyncService_setOption(as, KSI_ASYNC_OPT_SND_TIMEOUT, + (void*)size_t_value); + ctx->signer_state = SIGNER_STARTED; while (true) { diff --git a/runtime/lib_ksils12.h b/runtime/lib_ksils12.h index a1c9011bac..72b65302d1 100644 --- a/runtime/lib_ksils12.h +++ b/runtime/lib_ksils12.h @@ -94,6 +94,7 @@ struct rsksictx_s { time_t tConfRequested; uint64_t blockLevelLimit; uint32_t blockTimeLimit; + uint32_t blockSigTimeout; uint32_t effectiveBlockLevelLimit; /* level limit adjusted by gateway settings */ uint32_t threadSleepms; uint8_t syncMode; @@ -204,6 +205,7 @@ struct rsksistatefile { #define getIVLenKSI(bh) (hashOutputLengthOctetsKSI((bh)->hashID)) #define rsksiSetBlockLevelLimit(ctx, limit) ((ctx)->blockLevelLimit = (ctx)->effectiveBlockLevelLimit = limit) #define rsksiSetBlockTimeLimit(ctx, limit) ((ctx)->blockTimeLimit = limit) +#define rsksiSetBlockSigTimeout(ctx, val) ((ctx)->blockSigTimeout = val) #define rsksiSetConfInterval(ctx, val) ((ctx)->confInterval = val) #define rsksiSetKeepRecordHashes(ctx, val) ((ctx)->bKeepRecordHashes = val) #define rsksiSetKeepTreeHashes(ctx, val) ((ctx)->bKeepTreeHashes = val) diff --git a/runtime/lmsig_ksi-ls12.c b/runtime/lmsig_ksi-ls12.c index 9ac57b7385..0336bc21d9 100644 --- a/runtime/lmsig_ksi-ls12.c +++ b/runtime/lmsig_ksi-ls12.c @@ -49,6 +49,7 @@ static struct cnfparamdescr cnfpdescr[] = { { "sig.aggregator.hmacAlg", eCmdHdlrGetWord, 0 }, { "sig.block.levelLimit", eCmdHdlrSize, CNFPARAM_REQUIRED}, { "sig.block.timeLimit", eCmdHdlrInt, 0}, + { "sig.block.signtimeout", eCmdHdlrInt, 0}, { "sig.confinterval", eCmdHdlrInt, 0}, { "sig.keeprecordhashes", eCmdHdlrBinary, 0 }, { "sig.keeptreehashes", eCmdHdlrBinary, 0}, @@ -179,6 +180,14 @@ SetCnfParam(void *pT, struct nvlst *lst) } } else if (!strcmp(pblk.descr[i].name, "sig.keeprecordhashes")) { rsksiSetKeepRecordHashes(pThis->ctx, pvals[i].val.d.n); + } else if (!strcmp(pblk.descr[i].name, "sig.block.signtimeout")) { + if (pvals[i].val.d.n < 0) { + LogError(0, RS_RET_ERR, "sig.block.signtimeout " + "%llu invalid - signing disabled", pvals[i].val.d.n); + pThis->ctx->disabled = true; + } else { + rsksiSetBlockSigTimeout(pThis->ctx, pvals[i].val.d.n); + } } else if(!strcmp(pblk.descr[i].name, "sig.keeptreehashes")) { rsksiSetKeepTreeHashes(pThis->ctx, pvals[i].val.d.n); } else if (!strcmp(pblk.descr[i].name, "sig.syncmode")) { From eab250be3701b6aa5ccc65f2f79fd5b41f08b2b2 Mon Sep 17 00:00:00 2001 From: "t.feng" Date: Mon, 8 Aug 2022 11:36:59 +0800 Subject: [PATCH 132/134] Fix Segmentation fault in close journal --- plugins/imjournal/imjournal.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c index 9b590cb8bf..f6c2885a53 100644 --- a/plugins/imjournal/imjournal.c +++ b/plugins/imjournal/imjournal.c @@ -991,7 +991,9 @@ CODESTARTafterRun persistJournalState(); } closeJournal(); - ratelimitDestruct(ratelimiter); + if (ratelimiter) { + ratelimitDestruct(ratelimiter); + } ENDafterRun From f901f113d2cc5bdda0551a9ce3a647503c189e7a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Aug 2022 09:07:27 +0200 Subject: [PATCH 133/134] maintain ChangeLog --- ChangeLog | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8b1c42d6da..ffa41d7ba5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,50 @@ ---------------------------------------------------------------------------------------- Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +- 2022-08-09: ksi bugfix: request cache size and send timeout issue fixed. + Async service send timeout is not configurable and request cache size is too + small to handle large amount of signing requests with small amount of permitted + requests per aggregation round. For example user with max_requests = 4 results + cache size 5 * max_requests or at least 256. When signing 300 log files cache + will be too small resulting several unsigned blocks. When signing 200 log file + cache will be adequate, but with rate of 4 signatures per second, it is only + possible to sign 4 * 10 blocks before all requests that are not sent out will + timeout. + Fix for the issue is to make send timeout configurable and make the size of the + cache depend on the value of send timeout. New configuration value + sig.block.signtimeout="time, s" introduced that defines the time window wherein + the block has to be signed. The size of the request cache is increased to + 3 * max_requests * sign_timeout or at least 256. + Thanks to Taavi Valjaots for the patch. +- 2022-08-09: imjournal bugfix: segmentation fault in close journal + Thanks to github user t-feng for the patch. +- 2022-08-09: net subsystem: support sha256 for StreamDriverAuthMode="x509/fingerprint" + Thanks to github user codemaker219 for the patch. +- 2022-08-05: imfile bugfix: message loss/duplication when monitored file is rotated + When a to-be-monitored file is being rotated, some messages may be lost or + duplicated. In case of duplication, many file lines may be duplicated + depending on actual timing. The whole bug was primarily timing depenedent + in general. It most often was visible in practice when the monitored + file was very frequently rotated (we had some report with every few + seconds). + Note that while we try hard to not lose any messages, input file + rotation always has some loss potential. This is inevitable if + the monitored file is being truncated. + Also note that this bugfix affects imfile, only. It has nothing to do + and no relation to rsyslog output files being rotated on HUP. + closes: https://github.com/rsyslog/rsyslog/issues/4797 +- 2022-08-05: ksi bugfix: optimize processing of signer queue to fix delays. + There is a worker queue where rsyslog KSI module collects events and signing + requests. When queue is processed thread is periodically put to sleep. Previous + implementation handles signature requests well but sleeps every time after + handling new file open / close event. When several log files are opened or + closed simultaneously process is significantly slowed down. Another issue is + that thread always sleeps 1000ms that may be 2x longer than aggregation round. + This slows down overall signing process. + Fix for the issue is to simply not sleep after file open / close event if there + are next items to be processed. To speed up the signing process, rsyslog uses + KSI aggregator conf. to obtain the aggregation period that is used for the sleep + time configuration. + Thanks to Taavi Valjaots for the patch. - 2022-08-04: ksi bugfix: possible crash fixed when several log files are opened. KSI module in async mode used to request aggregator conf. every time a log file was opened. When several log files were opened simultaneously From e8d1288ea1228b7484f19c382974749fc94ebbee Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Aug 2022 09:11:09 +0200 Subject: [PATCH 134/134] prepare for 8.2208.0 scheduled stable release --- ChangeLog | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ffa41d7ba5..d6e26bf192 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ ---------------------------------------------------------------------------------------- -Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-?? +Scheduled Release 8.2208.0 (aka 2022.08) 2022-08-09 - 2022-08-09: ksi bugfix: request cache size and send timeout issue fixed. Async service send timeout is not configurable and request cache size is too small to handle large amount of signing requests with small amount of permitted diff --git a/configure.ac b/configure.ac index 6da9ddbbed..f7973a0538 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[8.2208.0.master],[rsyslog@lists.adiscon.com]) # UPDATE on release +AC_INIT([rsyslog],[8.2208.0],[rsyslog@lists.adiscon.com]) # UPDATE on release AC_DEFINE(VERSION_YEAR, 22, [year part of real rsyslog version]) # UPDATE on release AC_DEFINE(VERSION_MONTH, 8, [month part of real rsyslog version]) # UPDATE on release