diff options
71 files changed, 2563 insertions, 1870 deletions
@@ -1,4 +1,63 @@ --------------------------------------------------------------------------- +Version 8.1.2 [devel] 2013-11-28 +- support for liblognorm1 added - results in performance improvements + Thanks to Pavel Levshin for his work in this regard. +- support for jemalloc added via --enable-jemalloc + Thanks to Pavel Levshin for suggesting jemalloc + Note that build system is experimental at this stage. +- queue defaults have changed + * high water mark is now dynamically 90% of queue size + * low water makr is now dynamically 70% of queue size + * queue.discardMark is now dynamically 98% of queue size + * queue.workerThreadMinimumMessage set to queue.size / num workers + For queues with very low queue.maxSize (< 100), "emergency" defaults + will be used. +- bugfix: disk queues created files in wrong working directory + if the $WorkDirectory was changed multiple times, all queues only + used the last value set. +- bugfix: legacy directive $ActionQueueWorkerThreads was not honored +- bugfix: mmrfc5424addhmac: "key" parameter was not properly processed +--------------------------------------------------------------------------- +Version 8.1.1 [devel] 2013-11-19 +- bugfix: STOP/discard(~) was mostly NOT honored + This lead to execution of config code that was not meant to be executed. +- bugfix: memory leak on worker thread termination +- bugfix: potential segfault in omfile under heavy load + Thanks to Pavel Levshin for alerting us. +- bugfix: mmsequence: instance mode did not work + Thanks to Pavel Levshin for the patch +- bugfix: segfault on startup when certain script constructs are used + e.g. "if not $msg ..." +- omhiredis: now supports v8 output module interface and works again + Thanks to Pavel Levshin for the patch +- mmaudit: now supports v8 output module interface and work again +- bugfix: potential abort on startup in debug mode + This depends on template type being used. The root cause was a + non-necessary debug output, which were at the wrong spot (leftover from + initial testing). + Thanks to Pavel Levshin for alerting us and providing a patch + proposal. +--------------------------------------------------------------------------- +Version 8.1.0 [devel] 2013-11-15 +- rewritten core engine for higher performance and new features + In detail: + * completely rewritten rule execution engine + * completely changed output module interface + * remodelled output module interface + * enabled important output modules to support full concurrent + operation + The core engine has been considerably changed and must be considered + experimental at this stage. Note that it does not yet include all + features planned for v8, but is close to this goal. In theory, the + engine should perform much better, especially on complex configurations + and busy servers. Most importantly, actions instances can now be called + concurrently from worker threads and many important output modules + support multiple concurrent action instances natively. +- module omruleset is no longer enabled by default. + Note that it has been deprecated in v7 and been replaced by the "call" + statement. Also, it can still be build without problems, the option must + just explicitely be given. +--------------------------------------------------------------------------- Version 7.5.8 [v7-devel] 2013-11-?? - actions now report if they suspend and resume themselves this is by default on and controllable by the action.reportSuspension diff --git a/Makefile.am b/Makefile.am index 6eb7b5b8..4b04dd9b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -284,8 +284,7 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ - --enable-imfile \ +DISTCHECK_CONFIGURE_FLAGS= --enable-imfile \ --enable-snmp \ --enable-libdbi \ --enable-mysql \ @@ -308,11 +307,12 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-pmaixforwardedfrom \ --enable-pmcisconames \ --enable-pmsnare \ - --enable-mmsnmptrapd \ --enable-elasticsearch \ --enable-valgrind \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) # temporarily disable these checks for make distcheck 2012-09-06 rgerhards +# --enable-mmsnmptrapd \ +# --enable-gssapi_krb5 \ # --enable-extended-tests \ # --enable-pgsql ACLOCAL_AMFLAGS = -I m4 @@ -13,18 +13,15 @@ * The different modes (and calling sequence) are: * * if set iExecEveryNthOccur > 1 || iSecsExecOnceInterval - * - doSubmitToActionQComplexBatch - * - helperSubmitToActionQComplexBatch - * - doActionCallAction + * - doSubmitToActionQComplex * handles mark message reduction, but in essence calls * - actionWriteToAction * - qqueueEnqObj * (now queue engine processing) - * if(pThis->bWriteAllMarkMsgs == RSFALSE) - this is the DEFAULT - * - doSubmitToActionQNotAllMarkBatch - * - doSubmitToActionQBatch (and from here like in the else case below!) + * if(pThis->bWriteAllMarkMsgs == RSFALSE) + * - doSubmitToActionQNotAllMark + * - doSubmitToActionQ (and from here like in the else case below!) * else - * - doSubmitToActionQBatch * - doSubmitToActionQ * - qqueueEnqObj * (now queue engine processing) @@ -36,9 +33,6 @@ * * After dequeue, processing is as follows: * - processBatchMain - * - processAction - * - submitBatch - * - tryDoAction * - ... * * MORE ON PROCESSING, QUEUES and FILTERING @@ -120,13 +114,12 @@ #define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ /* forward definitions */ -static rsRetVal processBatchMain(action_t *pAction, batch_t *pBatch, int*); -static rsRetVal doSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch); -static rsRetVal doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch); -static rsRetVal doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch); +static rsRetVal processBatchMain(void *pVoid, batch_t *pBatch, wti_t *pWti); +static rsRetVal doSubmitToActionQ(action_t *pAction, wti_t *pWti, msg_t*); +static rsRetVal doSubmitToActionQComplex(action_t *pAction, wti_t *pWti, msg_t*); +static rsRetVal doSubmitToActionQNotAllMark(action_t *pAction, wti_t *pWti, msg_t*); /* object static data (once for all instances) */ -/* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ DEFobjCurrIf(obj) DEFobjCurrIf(datetime) DEFobjCurrIf(module) @@ -176,10 +169,9 @@ configSettings_t cs_save; /* our saved (scope!) config settings */ /* the counter below counts actions created. It is used to obtain unique IDs for the action. They * should not be relied on for any long-term activity (e.g. disk queue names!), but they are nice * to have during one instance of an rsyslogd run. For example, I use them to name actions when there - * is no better name available. Note that I do NOT recover previous numbers on HUP - we simply keep - * counting. -- rgerhards, 2008-01-29 + * is no better name available. */ -static int iActionNbr = 0; +int iActionNbr = 0; int bActionReportSuspension = 1; /* tables for interfacing with the v6 config system */ @@ -310,7 +302,6 @@ rsRetVal actionDestruct(action_t *pThis) pThis->pMod->freeInstance(pThis->pModData); pthread_mutex_destroy(&pThis->mutAction); - pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); @@ -336,14 +327,15 @@ rsRetVal actionConstruct(action_t **ppThis) pThis->iResumeInterval = 30; pThis->iResumeRetryCount = 0; pThis->pszName = NULL; - pThis->bWriteAllMarkMsgs = RSFALSE; + pThis->bWriteAllMarkMsgs = 1; pThis->iExecEveryNthOccur = 0; pThis->iExecEveryNthOccurTO = 0; pThis->iSecsExecOnceInterval = 0; pThis->bExecWhenPrevSusp = 0; pThis->bRepMsgHasMsg = 0; + pThis->bDisabled = 0; pThis->tLastOccur = datetime.GetTime(NULL); /* done once per action on startup only */ - pthread_mutex_init(&pThis->mutActExec, NULL); + pThis->iActionNbr = iActionNbr; pthread_mutex_init(&pThis->mutAction, NULL); INIT_ATOMIC_HELPER_MUT(pThis->mutCAS); @@ -425,13 +417,13 @@ actionConstructFinalize(action_t *pThis, struct nvlst *lst) DBGPRINTF("info: firehose mode disabled for action because " "iExecEveryNthOccur=%d, iSecsExecOnceInterval=%d\n", pThis->iExecEveryNthOccur, pThis->iSecsExecOnceInterval); - pThis->submitToActQ = doSubmitToActionQComplexBatch; - } else if(pThis->bWriteAllMarkMsgs == RSFALSE) { - /* nearly full-speed submission mode, default case */ - pThis->submitToActQ = doSubmitToActionQNotAllMarkBatch; + pThis->submitToActQ = doSubmitToActionQComplex; + } else if(pThis->bWriteAllMarkMsgs) { + /* full firehose submission mode, default case*/ + pThis->submitToActQ = doSubmitToActionQ; } else { - /* full firehose submission mode */ - pThis->submitToActQ = doSubmitToActionQBatch; + /* nearly full-speed submission mode */ + pThis->submitToActQ = doSubmitToActionQNotAllMark; } /* create queue */ @@ -441,7 +433,7 @@ actionConstructFinalize(action_t *pThis, struct nvlst *lst) * spec. -- rgerhards, 2008-01-30 */ CHKiRet(qqueueConstruct(&pThis->pQueue, cs.ActionQueType, 1, cs.iActionQueueSize, - (rsRetVal (*)(void*, batch_t*, int*))processBatchMain)); + processBatchMain)); obj.SetName((obj_t*) pThis->pQueue, pszAName); qqueueSetpAction(pThis->pQueue, pThis); @@ -518,9 +510,9 @@ rsRetVal actionSetGlobalResumeInterval(int iNewVal) * returned string must not be modified. * rgerhards, 2009-05-07 */ -static uchar *getActStateName(action_t *pThis) +static uchar *getActStateName(action_t *pThis, wti_t *pWti) { - switch(pThis->eState) { + switch(getActionState(pWti, pThis)) { case ACT_STATE_RDY: return (uchar*) "rdy"; case ACT_STATE_ITX: @@ -529,8 +521,6 @@ static uchar *getActStateName(action_t *pThis) return (uchar*) "rtry"; case ACT_STATE_SUSP: return (uchar*) "susp"; - case ACT_STATE_DIED: - return (uchar*) "died"; case ACT_STATE_COMM: return (uchar*) "comm"; default: @@ -542,12 +532,12 @@ static uchar *getActStateName(action_t *pThis) /* returns a suitable return code based on action state * rgerhards, 2009-05-07 */ -static rsRetVal getReturnCode(action_t *pThis) +static rsRetVal getReturnCode(action_t *pThis, wti_t *pWti) { DEFiRet; ASSERT(pThis != NULL); - switch(pThis->eState) { + switch(getActionState(pWti, pThis)) { case ACT_STATE_RDY: iRet = RS_RET_OK; break; @@ -563,12 +553,11 @@ static rsRetVal getReturnCode(action_t *pThis) iRet = RS_RET_SUSPENDED; break; case ACT_STATE_SUSP: - case ACT_STATE_DIED: iRet = RS_RET_ACTION_FAILED; break; default: - DBGPRINTF("Invalid action engine state %d, program error\n", - (int) pThis->eState); + DBGPRINTF("Invalid action engine state %u, program error\n", + getActionState(pWti, pThis)); iRet = RS_RET_ERR; break; } @@ -580,29 +569,31 @@ static rsRetVal getReturnCode(action_t *pThis) /* set the action to a new state * rgerhards, 2007-08-02 */ -static inline void actionSetState(action_t *pThis, action_state_t newState) +static inline void +actionSetState(action_t *pThis, wti_t *pWti, uint8_t newState) { - pThis->eState = newState; - DBGPRINTF("Action %p transitioned to state: %s\n", pThis, getActStateName(pThis)); + setActionState(pWti, pThis, newState); + DBGPRINTF("Action %d transitioned to state: %s\n", + pThis->iActionNbr, getActStateName(pThis, pWti)); } /* Handles the transient commit state. So far, this is * mostly a dummy... * rgerhards, 2007-08-02 */ -static void actionCommitted(action_t *pThis) +static void actionCommitted(action_t *pThis, wti_t *pWti) { - actionSetState(pThis, ACT_STATE_RDY); + actionSetState(pThis, pWti, ACT_STATE_RDY); } /* set action to "rtry" state. * rgerhards, 2007-08-02 */ -static void actionRetry(action_t *pThis) +static void actionRetry(action_t *pThis, wti_t *pWti) { - actionSetState(pThis, ACT_STATE_RTRY); - pThis->iResumeOKinRow++; + actionSetState(pThis, pWti, ACT_STATE_RTRY); + incActionResumeInRow(pWti, pThis); } @@ -613,7 +604,7 @@ static void actionRetry(action_t *pThis) */ static void actionDisable(action_t *pThis) { - actionSetState(pThis, ACT_STATE_DIED); + pThis->bDisabled = 1; } @@ -626,7 +617,7 @@ static void actionDisable(action_t *pThis) * rgerhards, 2007-08-02 */ static inline void -actionSuspend(action_t * const pThis) +actionSuspend(action_t * const pThis, wti_t *pWti) { time_t ttNow; int suspendDuration; @@ -636,16 +627,17 @@ actionSuspend(action_t * const pThis) * since caching, and this would break logic (and it actually did so!) */ datetime.GetTime(&ttNow); - suspendDuration = pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + suspendDuration = pThis->iResumeInterval * (getActionNbrResRtry(pWti, pThis) / 10 + 1); pThis->ttResumeRtry = ttNow + suspendDuration; - actionSetState(pThis, ACT_STATE_SUSP); + actionSetState(pThis, pWti, ACT_STATE_SUSP); pThis->ctrSuspendDuration += suspendDuration; - if(pThis->iNbrResRtry == 0) { + if(getActionNbrResRtry(pWti, pThis) == 0) { STATSCOUNTER_INC(pThis->ctrSuspend, pThis->mutCtrSuspend); } - DBGPRINTF("action '%s' suspended, earliest retry=%lld (now %lld), iNbrResRtry %d\n", + DBGPRINTF("action '%s' suspended, earliest retry=%lld (now %lld), iNbrResRtry %d, " + "duration %d\n", pThis->pszName, (long long) pThis->ttResumeRtry, (long long) ttNow, - pThis->iNbrResRtry); + getActionNbrResRtry(pWti, pThis), suspendDuration); if(bActionReportSuspension) { ctime_r(&pThis->ttResumeRtry, timebuf); timebuf[strlen(timebuf)-1] = '\0'; /* strip LF */ @@ -664,15 +656,15 @@ actionSuspend(action_t * const pThis) * entry point. This is invalid, but has harsh consequences: it will cause the rsyslog * engine to go into a tight loop. That obviously is not acceptable. As such, we track the * count of iterations that a tryResume returning RS_RET_OK is immediately followed by - * an unsuccessful call to doAction(). If that happens more than 1,000 times, we assume + * an unsuccessful call to doAction(). If that happens more than 10 times, we assume * the return acutally is a RS_RET_SUSPENDED. In order to go through the various - * resumption stages, we do this for every 1000 requests. This magic number 1000 may + * resumption stages, we do this for every 10 requests. This magic number 10 may * not be the most appropriate, but it should be thought of a "if nothing else helps" * kind of facility: in the first place, the module should return a proper indication * of its inability to recover. -- rgerhards, 2010-04-26. */ -static inline rsRetVal -actionDoRetry(action_t *pThis, int *pbShutdownImmediate) +static rsRetVal +actionDoRetry(action_t *pThis, wti_t *pWti) { int iRetries; int iSleepPeriod; @@ -682,13 +674,13 @@ actionDoRetry(action_t *pThis, int *pbShutdownImmediate) ASSERT(pThis != NULL); iRetries = 0; - while((*pbShutdownImmediate == 0) && pThis->eState == ACT_STATE_RTRY) { + while((*pWti->pbShutdownImmediate == 0) && getActionState(pWti, pThis) == ACT_STATE_RTRY) { DBGPRINTF("actionDoRetry: %s enter loop, iRetries=%d\n", pThis->pszName, iRetries); - iRet = pThis->pMod->tryResume(pThis->pModData); + iRet = pThis->pMod->tryResume(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData); DBGPRINTF("actionDoRetry: %s action->tryResume returned %d\n", pThis->pszName, iRet); - if((pThis->iResumeOKinRow > 9) && (pThis->iResumeOKinRow % 10 == 0)) { + if((getActionResumeInRow(pWti, pThis) > 9) && (getActionResumeInRow(pWti, pThis) % 10 == 0)) { bTreatOKasSusp = 1; - pThis->iResumeOKinRow = 0; + setActionResumeInRow(pWti, pThis, 0); } else { bTreatOKasSusp = 0; } @@ -699,21 +691,21 @@ actionDoRetry(action_t *pThis, int *pbShutdownImmediate) errmsg.LogMsg(0, RS_RET_OK, LOG_INFO, "action '%s' resumed", pThis->pszName); } - actionSetState(pThis, ACT_STATE_RDY); + actionSetState(pThis, pWti, ACT_STATE_RDY); } else if(iRet == RS_RET_SUSPENDED || bTreatOKasSusp) { /* max retries reached? */ DBGPRINTF("actionDoRetry: %s check for max retries, iResumeRetryCount " "%d, iRetries %d\n", pThis->pszName, pThis->iResumeRetryCount, iRetries); if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { - actionSuspend(pThis); - if(pThis->iNbrResRtry < 20) - ++pThis->iNbrResRtry; + actionSuspend(pThis, pWti); + if(getActionNbrResRtry(pWti, pThis) < 20) + incActionNbrResRtry(pWti, pThis); } else { ++iRetries; iSleepPeriod = pThis->iResumeInterval; srSleep(iSleepPeriod, 0); - if(*pbShutdownImmediate) { + if(*pWti->pbShutdownImmediate) { ABORT_FINALIZE(RS_RET_FORCE_TERM); } } @@ -722,8 +714,8 @@ actionDoRetry(action_t *pThis, int *pbShutdownImmediate) } } - if(pThis->eState == ACT_STATE_RDY) { - pThis->iNbrResRtry = 0; + if(getActionState(pWti, pThis) == ACT_STATE_RDY) { + setActionNbrResRtry(pWti, pThis, 0); } finalize_it: @@ -731,17 +723,33 @@ finalize_it: } +static rsRetVal +actionCheckAndCreateWrkrInstance(action_t *pThis, wti_t *pWti) +{ + DEFiRet; + if(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData == NULL) { +dbgprintf("DDDD: wti %p create new worker instance for action %d\n", pWti, pThis->iActionNbr); + DBGPRINTF("we need to create a new action worker instance for " + "action %d\n", pThis->iActionNbr); + CHKiRet(pThis->pMod->mod.om.createWrkrInstance(&(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData), + pThis->pModData)); + pWti->actWrkrInfo[pThis->iActionNbr].pAction = pThis; + setActionState(pWti, pThis, ACT_STATE_RDY); /* action is enabled */ + } +finalize_it: + RETiRet; +} + /* try to resume an action -- rgerhards, 2007-08-02 * changed to new action state engine -- rgerhards, 2009-05-07 */ -static rsRetVal actionTryResume(action_t *pThis, int *pbShutdownImmediate) +static rsRetVal +actionTryResume(action_t *pThis, wti_t *pWti) { DEFiRet; time_t ttNow = NO_TIME_PROVIDED; - ASSERT(pThis != NULL); - - if(pThis->eState == ACT_STATE_SUSP) { + if(getActionState(pWti, pThis) == ACT_STATE_SUSP) { /* if we are suspended, we need to check if the timeout expired. * for this handling, we must always obtain a fresh timestamp. We used * to use the action timestamp, but in this case we will never reach a @@ -751,19 +759,19 @@ static rsRetVal actionTryResume(action_t *pThis, int *pbShutdownImmediate) */ datetime.GetTime(&ttNow); /* cache "now" */ if(ttNow >= pThis->ttResumeRtry) { - actionSetState(pThis, ACT_STATE_RTRY); /* back to retries */ + actionSetState(pThis, pWti, ACT_STATE_RTRY); /* back to retries */ } } - if(pThis->eState == ACT_STATE_RTRY) { + if(getActionState(pWti, pThis) == ACT_STATE_RTRY) { if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */ datetime.GetTime(&ttNow); - CHKiRet(actionDoRetry(pThis, pbShutdownImmediate)); + CHKiRet(actionDoRetry(pThis, pWti)); } - if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) { + if(Debug && (getActionState(pWti, pThis) == ACT_STATE_RTRY ||getActionState(pWti, pThis) == ACT_STATE_SUSP)) { DBGPRINTF("actionTryResume: action %p state: %s, next retry (if applicable): %u [now %u]\n", - pThis, getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + pThis, getActStateName(pThis, pWti), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); } finalize_it: @@ -775,24 +783,27 @@ finalize_it: * depending on its current state. * rgerhards, 2009-05-07 */ -static inline rsRetVal actionPrepare(action_t *pThis, int *pbShutdownImmediate) +static inline rsRetVal +actionPrepare(action_t *pThis, wti_t *pWti) { DEFiRet; assert(pThis != NULL); - CHKiRet(actionTryResume(pThis, pbShutdownImmediate)); + CHKiRet(actionCheckAndCreateWrkrInstance(pThis, pWti)); + CHKiRet(actionTryResume(pThis, pWti)); /* if we are now ready, we initialize the transaction and advance * action state accordingly */ - if(pThis->eState == ACT_STATE_RDY) { - iRet = pThis->pMod->mod.om.beginTransaction(pThis->pModData); + if(getActionState(pWti, pThis) == ACT_STATE_RDY) { +dbgprintf("DDDDD: calling beginTransaction for action %d\n", pThis->iActionNbr); + iRet = pThis->pMod->mod.om.beginTransaction(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData); switch(iRet) { case RS_RET_OK: - actionSetState(pThis, ACT_STATE_ITX); + actionSetState(pThis, pWti, ACT_STATE_ITX); break; case RS_RET_SUSPENDED: - actionRetry(pThis); + actionRetry(pThis, pWti); break; case RS_RET_DISABLE_ACTION: actionDisable(pThis); @@ -806,10 +817,11 @@ finalize_it: } +#if 0 // TODO: remove? /* debug-print the contents of an action object * rgerhards, 2007-08-02 */ -rsRetVal actionDbgPrint(action_t *pThis) +static rsRetVal actionDbgPrint(action_t *pThis) { DEFiRet; char *sz; @@ -819,11 +831,12 @@ rsRetVal actionDbgPrint(action_t *pThis) dbgprintf("\n"); dbgprintf("\tInstance data: 0x%lx\n", (unsigned long) pThis->pModData); dbgprintf("\tResume Interval: %d\n", pThis->iResumeInterval); - if(pThis->eState == ACT_STATE_SUSP) { +#if 0 // do we need this ??? + if(getActionState(pWti, pThis) == ACT_STATE_SUSP) { dbgprintf("\tresume next retry: %u, number retries: %d", (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); } - dbgprintf("\tState: %s\n", getActStateName(pThis)); +#endif dbgprintf("\tExec only when previous is suspended: %d\n", pThis->bExecWhenPrevSusp); if(pThis->submitToActQ == doSubmitToActionQComplexBatch) { sz = "slow, but feature-rich"; @@ -839,40 +852,45 @@ rsRetVal actionDbgPrint(action_t *pThis) RETiRet; } +#endif /* prepare the calling parameters for doAction() * rgerhards, 2009-05-07 */ static rsRetVal -prepareDoActionParams(action_t *pAction, batch_obj_t *pElem, struct syslogTime *ttNow) +prepareDoActionParams(action_t *pAction, wti_t *pWti, msg_t *pMsg, struct syslogTime *ttNow) { int i; - msg_t *pMsg; struct json_object *json; + actWrkrIParams_t *iparams; + actWrkrInfo_t *pWrkrInfo; DEFiRet; - ASSERT(pAction != NULL); - ASSERT(pElem != NULL); + pWrkrInfo = &(pWti->actWrkrInfo[pAction->iActionNbr]); + if(pAction->eParamPassing == ACT_STRING_PASSING) { + CHKiRet(wtiNewIParam(pWti, pAction, &iparams)); + } - pMsg = pElem->pMsg; /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { +dbgprintf("DDDDD: generating template #%d\n", i); switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pElem->staticActStrings[i]), - &pElem->staticLenStrings[i], ttNow)); - pElem->staticActParams[i] = pElem->staticActStrings[i]; + iparams->msgFlags = pMsg->msgFlags; + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(iparams->staticActStrings[i]), + &iparams->staticLenStrings[i], ttNow)); + iparams->staticActParams[i] = iparams->staticActStrings[i]; break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pElem->staticActParams[i]), ttNow)); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pWrkrInfo->staticActParams[i]), ttNow)); break; case ACT_MSG_PASSING: - pElem->staticActParams[i] = (void*) pMsg; + pWrkrInfo->staticActParams[i] = (void*) pMsg; break; case ACT_JSON_PASSING: CHKiRet(tplToJSON(pAction->ppTpl[i], pMsg, &json, ttNow)); - pElem->staticActParams[i] = (void*) json; + pWrkrInfo->staticActParams[i] = (void*) json; break; default:dbgprintf("software bug/error: unknown pAction->eParamPassing %d in prepareDoActionParams\n", (int) pAction->eParamPassing); @@ -886,96 +904,85 @@ finalize_it: } -/* free a batches ressources, but not string buffers (because they will - * most probably be reused). String buffers are only deleted upon final - * destruction of the batch. - * This function here must be called only when the batch is actually no - * longer used, also not for retrying actions or such. It invalidates - * buffers. - * rgerhards, 2010-12-17 - */ -static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch) +static void +releaseDoActionParams(action_t *pAction, wti_t *pWti) { int jArr; - int i, j; - batch_obj_t *pElem; + int j; + actWrkrInfo_t *pWrkrInfo; uchar ***ppMsgs; - DEFiRet; - - ASSERT(pAction != NULL); if(pAction->eParamPassing == ACT_STRING_PASSING || pAction->eParamPassing == ACT_MSG_PASSING) goto done; /* we need to do nothing with these types! */ - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - pElem = &(pBatch->pElem[i]); - if(batchIsValidElem(pBatch, i)) { - switch(pAction->eParamPassing) { - case ACT_ARRAY_PASSING: - ppMsgs = (uchar***) pElem->staticActParams; - for(j = 0 ; j < pAction->iNumTpls ; ++j) { - if(((uchar**)ppMsgs)[j] != NULL) { - jArr = 0; - while(ppMsgs[j][jArr] != NULL) { - d_free(ppMsgs[j][jArr]); - ppMsgs[j][jArr] = NULL; - ++jArr; - } - d_free(((uchar**)ppMsgs)[j]); - ((uchar**)ppMsgs)[j] = NULL; - } - } - break; - case ACT_JSON_PASSING: - for(j = 0 ; j < pAction->iNumTpls ; ++j) { - json_object_put((struct json_object*) - pElem->staticActParams[j]); - pElem->staticActParams[j] = NULL; + pWrkrInfo = &(pWti->actWrkrInfo[pAction->iActionNbr]); + switch(pAction->eParamPassing) { + case ACT_ARRAY_PASSING: + ppMsgs = (uchar***) pWrkrInfo->staticActParams; + for(j = 0 ; j < pAction->iNumTpls ; ++j) { + if(((uchar**)ppMsgs)[j] != NULL) { + jArr = 0; + while(ppMsgs[j][jArr] != NULL) { + free(ppMsgs[j][jArr]); + ppMsgs[j][jArr] = NULL; + ++jArr; } - break; - case ACT_STRING_PASSING: - case ACT_MSG_PASSING: - /* can never happen, just to keep compiler happy! */ - break; + free(((uchar**)ppMsgs)[j]); + ((uchar**)ppMsgs)[j] = NULL; } } + break; + case ACT_JSON_PASSING: + for(j = 0 ; j < pAction->iNumTpls ; ++j) { + json_object_put((struct json_object*) + pWrkrInfo->staticActParams[j]); + pWrkrInfo->staticActParams[j] = NULL; + } + break; + case ACT_STRING_PASSING: + case ACT_MSG_PASSING: + /* can never happen, just to keep compiler happy! */ + break; } -done: RETiRet; +done: return; } /* call the DoAction output plugin entry point * rgerhards, 2008-01-28 */ -rsRetVal -actionCallDoAction(action_t *pThis, msg_t *pMsg, void *actParams) +static rsRetVal +actionCallDoAction(action_t *pThis, int msgFlags, void *actParams, wti_t *pWti) { DEFiRet; ASSERT(pThis != NULL); - ISOBJ_TYPE_assert(pMsg, msg); - DBGPRINTF("entering actionCalldoAction(), state: %s\n", getActStateName(pThis)); + DBGPRINTF("entering actionCalldoAction(), state: %s, actionNbr %d\n", + getActStateName(pThis, pWti), pThis->iActionNbr); + + CHKiRet(actionCheckAndCreateWrkrInstance(pThis, pWti)); pThis->bHadAutoCommit = 0; - iRet = pThis->pMod->mod.om.doAction(actParams, pMsg->msgFlags, pThis->pModData); + iRet = pThis->pMod->mod.om.doAction(actParams, msgFlags, + pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData); switch(iRet) { case RS_RET_OK: - actionCommitted(pThis); - pThis->iResumeOKinRow = 0; /* we had a successful call! */ + actionCommitted(pThis, pWti); + setActionResumeInRow(pWti, pThis, 0); break; case RS_RET_DEFER_COMMIT: - pThis->iResumeOKinRow = 0; /* we had a successful call! */ + setActionResumeInRow(pWti, pThis, 0); /* we are done, action state remains the same */ break; case RS_RET_PREVIOUS_COMMITTED: /* action state remains the same, but we had a commit. */ pThis->bHadAutoCommit = 1; - pThis->iResumeOKinRow = 0; /* we had a successful call! */ + setActionResumeInRow(pWti, pThis, 0); break; case RS_RET_SUSPENDED: - actionRetry(pThis); + actionRetry(pThis, pWti); break; case RS_RET_DISABLE_ACTION: actionDisable(pThis); @@ -985,7 +992,7 @@ actionCallDoAction(action_t *pThis, msg_t *pMsg, void *actParams) */ FINALIZE; } - iRet = getReturnCode(pThis); + iRet = getReturnCode(pThis, pWti); finalize_it: RETiRet; @@ -996,57 +1003,90 @@ finalize_it: * this readies the action and then calls doAction() * rgerhards, 2008-01-28 */ -static inline rsRetVal -actionProcessMessage(action_t *pThis, msg_t *pMsg, void *actParams, int *pbShutdownImmediate) +rsRetVal +actionProcessMessage(action_t *pThis, int msgFlags, void *actParams, wti_t *pWti) { DEFiRet; - ASSERT(pThis != NULL); - ISOBJ_TYPE_assert(pMsg, msg); - - CHKiRet(actionPrepare(pThis, pbShutdownImmediate)); + CHKiRet(actionPrepare(pThis, pWti)); if(pThis->pMod->mod.om.SetShutdownImmdtPtr != NULL) - pThis->pMod->mod.om.SetShutdownImmdtPtr(pThis->pModData, pbShutdownImmediate); - if(pThis->eState == ACT_STATE_ITX) - CHKiRet(actionCallDoAction(pThis, pMsg, actParams)); + pThis->pMod->mod.om.SetShutdownImmdtPtr(pThis->pModData, pWti->pbShutdownImmediate); + if(getActionState(pWti, pThis) == ACT_STATE_ITX) + CHKiRet(actionCallDoAction(pThis, msgFlags, actParams, pWti)); - iRet = getReturnCode(pThis); + iRet = getReturnCode(pThis, pWti); finalize_it: RETiRet; } -/* finish processing a batch. Most importantly, that means we commit if we - * need to do so. - * rgerhards, 2008-01-28 - */ +/* the following functions simulates a potential future new omo callback */ static rsRetVal -finishBatch(action_t *pThis, batch_t *pBatch) +doTransaction(action_t *pThis, wti_t *pWti) { + actWrkrInfo_t *wrkrInfo; + actWrkrIParams_t *iparamCurr; int i; DEFiRet; - ASSERT(pThis != NULL); + wrkrInfo = &(pWti->actWrkrInfo[pThis->iActionNbr]); + dbgprintf("DDDD: doTransaction: action %d, currIParams %d\n", pThis->iActionNbr, wrkrInfo->currIParam); + for(i = 0 ; i < wrkrInfo->currIParam ; ++i) { + iparamCurr = wrkrInfo->iparams + i; + iRet = actionProcessMessage(pThis, iparamCurr->msgFlags, + iparamCurr->staticActParams, + pWti); + dbgprintf("DDDD: doTransaction loop, iRet %d\n", iRet); + } + RETiRet; +} + - if(pThis->eState == ACT_STATE_RDY) { - /* we just need to flag the batch as commited */ - FINALIZE; /* nothing to do */ +static void +actionFreeParams(action_t *pThis, wti_t *pWti) +{ + actWrkrInfo_t *wrkrInfo; + actWrkrIParams_t *iparamCurr; + int i; + int j; + + wrkrInfo = &(pWti->actWrkrInfo[pThis->iActionNbr]); + dbgprintf("DDDD: actionFreeParams: action %d, currIParam %d\n", pThis->iActionNbr, wrkrInfo->currIParam); + for(i = 0 ; i < wrkrInfo->currIParam ; ++i) { + iparamCurr = wrkrInfo->iparams + i; + for(j = 0 ; j < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++j) { + /* TODO: we can save time by not freeing everything, + * but that's left for a later optimization. + */ + free(iparamCurr->staticActStrings[j]); + iparamCurr->staticActStrings[j] = NULL; + iparamCurr->staticLenStrings[j] = 0; + } } + wrkrInfo->currIParam = 0; /* reset to beginning */ + releaseDoActionParams(pThis, pWti); +} + - CHKiRet(actionPrepare(pThis, pBatch->pbShutdownImmediate)); - if(pThis->eState == ACT_STATE_ITX) { - iRet = pThis->pMod->mod.om.endTransaction(pThis->pModData); +/* Commit try committing (do not handle retry processing and such) */ +static rsRetVal +actionTryCommit(action_t *pThis, wti_t *pWti) +{ + //actWrkrInfo_t *wrkrInfo; + DEFiRet; + + doTransaction(pThis, pWti); + + CHKiRet(actionPrepare(pThis, pWti)); + if(getActionState(pWti, pThis) == ACT_STATE_ITX) { +dbgprintf("DDDDD: calling endTransaction for action %d\n", pThis->iActionNbr); + iRet = pThis->pMod->mod.om.endTransaction(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData); switch(iRet) { case RS_RET_OK: - actionCommitted(pThis); - /* flag messages as committed */ - for(i = 0 ; i < pBatch->nElem ; ++i) { - batchSetElemState(pBatch, i, BATCH_STATE_COMM); - pBatch->pElem[i].bPrevWasSuspended = 0; /* we had success! */ - } + actionCommitted(pThis, pWti); break; case RS_RET_SUSPENDED: - actionRetry(pThis); + actionRetry(pThis, pWti); break; case RS_RET_DISABLE_ACTION: actionDisable(pThis); @@ -1054,12 +1094,12 @@ finishBatch(action_t *pThis, batch_t *pBatch) case RS_RET_DEFER_COMMIT: DBGPRINTF("output plugin error: endTransaction() returns RS_RET_DEFER_COMMIT " "- ignored\n"); - actionCommitted(pThis); + actionCommitted(pThis, pWti); break; case RS_RET_PREVIOUS_COMMITTED: DBGPRINTF("output plugin error: endTransaction() returns RS_RET_PREVIOUS_COMMITTED " "- ignored\n"); - actionCommitted(pThis); + actionCommitted(pThis, pWti); break; default:/* permanent failure of this message - no sense in retrying. This is * not yet handled (but easy TODO) @@ -1067,304 +1107,139 @@ finishBatch(action_t *pThis, batch_t *pBatch) FINALIZE; } } - iRet = getReturnCode(pThis); + iRet = getReturnCode(pThis, pWti); finalize_it: + actionFreeParams(pThis, pWti); RETiRet; } - -/* try to submit a partial batch of elements. - * rgerhards, 2009-05-12 - */ -static inline rsRetVal -tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) -{ - int i; - int iElemProcessed; - int iCommittedUpTo; - msg_t *pMsg; - rsRetVal localRet; - DEFiRet; - - assert(pBatch != NULL); - assert(pnElem != NULL); - - i = pBatch->iDoneUpTo; /* all messages below that index are processed */ - iElemProcessed = 0; - iCommittedUpTo = i; - DBGPRINTF("tryDoAction %p, pnElem %d, nElem %d\n", pAction, *pnElem, pBatch->nElem); - while(iElemProcessed <= *pnElem && i < pBatch->nElem) { - if(*(pBatch->pbShutdownImmediate)) - ABORT_FINALIZE(RS_RET_FORCE_TERM); - /* NOTE: do NOT extend the filter below! Anything else must be done on the - * enq side of the queue (see file header comment)! -- rgerhards, 2011-06-15 - */ - if(batchIsValidElem(pBatch, i)) { - pMsg = pBatch->pElem[i].pMsg; - localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams, - pBatch->pbShutdownImmediate); - DBGPRINTF("action %p call returned %d\n", pAction, localRet); - /* Note: we directly modify the batch object state, because we know that - * wo do not overwrite BATCH_STATE_DISC indicators! - */ - if(localRet == RS_RET_OK) { - /* mark messages as committed */ - while(iCommittedUpTo <= i) { - pBatch->pElem[iCommittedUpTo].bPrevWasSuspended = 0; /* we had success! */ - batchSetElemState(pBatch, iCommittedUpTo, BATCH_STATE_COMM); - ++iCommittedUpTo; - //pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; - } - } else if(localRet == RS_RET_PREVIOUS_COMMITTED) { - /* mark messages as committed */ - while(iCommittedUpTo < i) { - pBatch->pElem[iCommittedUpTo].bPrevWasSuspended = 0; /* we had success! */ - batchSetElemState(pBatch, iCommittedUpTo, BATCH_STATE_COMM); - ++iCommittedUpTo; - //pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; - } - pBatch->eltState[i] = BATCH_STATE_SUB; - } else if(localRet == RS_RET_DEFER_COMMIT) { - pBatch->eltState[i] = BATCH_STATE_SUB; - } else if(localRet == RS_RET_DISCARDMSG) { - pBatch->eltState[i] = BATCH_STATE_DISC; - } else { - dbgprintf("tryDoAction: unexpected error code %d[nElem %d, Commited UpTo %d], finalizing\n", - localRet, *pnElem, iCommittedUpTo); - iRet = localRet; - FINALIZE; - } - } - ++i; - ++iElemProcessed; - } - -finalize_it: - if(pBatch->iDoneUpTo != iCommittedUpTo) { - pBatch->iDoneUpTo = iCommittedUpTo; - } - RETiRet; -} - -/* submit a batch for actual action processing. - * The first nElem elements are processed. This function calls itself - * recursively if it needs to handle errors. - * Note: we don't need the number of the first message to be processed as a parameter, - * because this is kept track of inside the batch itself (iDoneUpTo). - * rgerhards, 2009-05-12 +/* Note: we currently need to return an iRet, as this is used in + * direct mode. TODO: However, it may be worth further investigating this, + * as it looks like there is no ultimate consumer of this code. + * rgerhards, 2013-11-06 */ static rsRetVal -submitBatch(action_t *pAction, batch_t *pBatch, int nElem) +actionCommit(action_t *pThis, wti_t *pWti) { - int i; - int bDone; - rsRetVal localRet; - int wasDoneTo; + sbool bDone; DEFiRet; - assert(pBatch != NULL); - - DBGPRINTF("submitBatch: enter, nElem %d\n", nElem); - wasDoneTo = pBatch->iDoneUpTo; + /* even more TODO: + This is the place where retry processing needs to go in. If the action + permanently fails, we should - as a new feature - add the capability to + write an error file. This is already done be omelasticsearch, and IMHO + pretty useful. + For the time being, I do NOT implement all of this (not even retry!) + as I want to get the rest of the engine to SISD (non-SIMD ;)) so that + I know any potential suprises and complications that arise out of this. + When this is done, I can come back here and complete this work. Obviously, + many features do not work in the mean time (but it is not planned to release + any of these partial implementations). + rgerhards, 2013-11-04 + */ bDone = 0; do { - localRet = tryDoAction(pAction, pBatch, &nElem); - if(localRet == RS_RET_FORCE_TERM) { + iRet = actionTryCommit(pThis, pWti); + DBGPRINTF("actionCommit, in retry loop, iRet %d\n", iRet); + if(iRet == RS_RET_FORCE_TERM) { ABORT_FINALIZE(RS_RET_FORCE_TERM); - } - if( localRet == RS_RET_OK - || localRet == RS_RET_PREVIOUS_COMMITTED - || localRet == RS_RET_DEFER_COMMIT) { - /* try commit transaction, once done, we can simply do so as if - * that return state was returned from tryDoAction(). - */ - localRet = finishBatch(pAction, pBatch); - } - - if( localRet == RS_RET_OK - || localRet == RS_RET_PREVIOUS_COMMITTED - || localRet == RS_RET_DEFER_COMMIT) { + } else if(iRet == RS_RET_OK || iRet == RS_RET_ACTION_FAILED) { bDone = 1; - } else if(localRet == RS_RET_SUSPENDED) { - DBGPRINTF("action ret RS_RET_SUSPENDED - retry full batch\n"); - /* do nothing, this will retry the full batch */ - } else if(localRet == RS_RET_ACTION_FAILED) { - /* in this case, everything not yet committed is BAD */ - for(i = pBatch->iDoneUpTo ; i < wasDoneTo + nElem ; ++i) { - if( pBatch->eltState[i] != BATCH_STATE_DISC - && pBatch->eltState[i] != BATCH_STATE_COMM ) { - pBatch->eltState[i] = BATCH_STATE_BAD; - pBatch->pElem[i].bPrevWasSuspended = 1; - STATSCOUNTER_INC(pAction->ctrFail, pAction->mutCtrFail); - } - } - bDone = 1; - } else { - if(nElem == 1) { - batchSetElemState(pBatch, pBatch->iDoneUpTo, BATCH_STATE_BAD); - bDone = 1; - } else { - /* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */ - DBGPRINTF("submitBatch recursing trying to find and exclude the culprit " - "for iRet %d\n", localRet); - submitBatch(pAction, pBatch, nElem / 2); - submitBatch(pAction, pBatch, nElem - (nElem / 2)); - bDone = 1; - } } - } while(!bDone && !*(pBatch->pbShutdownImmediate)); /* do .. while()! */ - - if(*(pBatch->pbShutdownImmediate)) - ABORT_FINALIZE(RS_RET_FORCE_TERM); - -finalize_it: - RETiRet; -} - - -/* copy "active" array of batch, as we need to modify it. The caller - * must make sure the new array is freed and the orginal batch - * pointer is restored (thus the caller must save it). If active - * is currently NULL, this is properly handled. - * Note: the batches active pointer is modified, so it must be - * saved BEFORE calling this function! - * rgerhards, 2012-09-12 - */ -static rsRetVal -copyActive(batch_t *pBatch) -{ - sbool *active; - DEFiRet; - - CHKmalloc(active = malloc(batchNumMsgs(pBatch) * sizeof(sbool))); - if(pBatch->active == NULL) - memset(active, 1, batchNumMsgs(pBatch)); - else - memcpy(active, pBatch->active, batchNumMsgs(pBatch)); - pBatch->active = active; + } while(!bDone); finalize_it: RETiRet; } -/* The following function prepares a batch for processing, that it is - * reinitializes batch states, generates strings and does everything else - * that needs to be done in order to make the batch ready for submission to - * the actual output module. Note that we look at the precomputed - * filter OK condition and process only those messages, that actually matched - * the filter. - * rgerhards, 2010-06-14 - */ -static inline rsRetVal -prepareBatch(action_t *pAction, batch_t *pBatch, sbool **activeSave, int *bMustRestoreActivePtr) +/* Commit all active transactions in *DIRECT mode* */ +void +actionCommitAllDirect(wti_t *pWti) { int i; - batch_obj_t *pElem; - struct syslogTime ttNow; - DEFiRet; - - /* indicate we have not yet read the date */ - ttNow.year = 0; + action_t *pAction; - pBatch->iDoneUpTo = 0; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - pElem = &(pBatch->pElem[i]); - if(batchIsValidElem(pBatch, i)) { - pBatch->eltState[i] = BATCH_STATE_RDY; - if(prepareDoActionParams(pAction, pElem, &ttNow) != RS_RET_OK) { - /* make sure we have our copy of "active" array */ - if(!*bMustRestoreActivePtr) { - *activeSave = pBatch->active; - copyActive(pBatch); - } - pBatch->active[i] = RSFALSE; - } - } + for(i = 0 ; i < iActionNbr ; ++i) { + dbgprintf("DDDD: actionCommitAll: action %d, state %u, nbr to commit %d\n", + i, getActionStateByNbr(pWti, i), pWti->actWrkrInfo->currIParam); + pAction = pWti->actWrkrInfo[i].pAction; + if(pAction != NULL && pAction->pQueue->qType == QUEUETYPE_DIRECT) + actionCommit(pAction, pWti); } - RETiRet; } - -/* receive a batch and process it. This includes retry handling. - * rgerhards, 2009-05-12 +/* process a single message. This is both called if we run from the + * cosumer side of an action queue as well as directly from the main + * queue thread if the action queue is set to "direct". */ -static inline rsRetVal -processAction(action_t *pAction, batch_t *pBatch) +static rsRetVal +processMsgMain(action_t *pAction, wti_t *pWti, msg_t *pMsg, struct syslogTime *ttNow) { DEFiRet; - assert(pBatch != NULL); - CHKiRet(submitBatch(pAction, pBatch, pBatch->nElem)); - iRet = finishBatch(pAction, pBatch); + if(pAction->bExecWhenPrevSusp && !pWti->execState.bPrevWasSuspended) { + DBGPRINTF("action %d: NOT executing, as previous action was " + "not suspended\n", pAction->iActionNbr); + FINALIZE; + } + +dbgprintf("DDDD: processMsgMain[act %d], %s\n", pAction->iActionNbr, pMsg->pszRawMsg); + iRet = prepareDoActionParams(pAction, pWti, pMsg, ttNow); + if(pAction->eParamPassing == ACT_STRING_PASSING) { + pWti->actWrkrInfo[pAction->iActionNbr].pAction = pAction; + dbgprintf("DDDD: action %d is string passing - executing in commit phase\n", pAction->iActionNbr); + actionPrepare(pAction, pWti); + iRet = getReturnCode(pAction, pWti); + FINALIZE; + } + iRet = actionProcessMessage(pAction, pMsg->msgFlags, + pWti->actWrkrInfo[pAction->iActionNbr].staticActParams, + pWti); + releaseDoActionParams(pAction, pWti); finalize_it: + if(pWti->execState.bDoAutoCommit) + iRet = actionCommit(pAction, pWti); + pWti->execState.bPrevWasSuspended = (iRet == RS_RET_SUSPENDED || iRet == RS_RET_ACTION_FAILED); +dbgprintf("DDDD: bPrevWasSuspended now %d, action state %d\n", (int)pWti->execState.bPrevWasSuspended, getActionState(pWti, pAction)); RETiRet; } - -#pragma GCC diagnostic ignored "-Wempty-body" -/* receive an array of to-process user pointers and submit them - * for processing. - * rgerhards, 2009-04-22 +/* This entry point is called by the ACTION queue (not main queue!) */ static rsRetVal -processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) +processBatchMain(void *pVoid, batch_t *pBatch, wti_t *pWti) { - int *pbShutdownImmdtSave; - sbool *activeSave; - int bMustRestoreActivePtr = 0; - rsRetVal localRet; + action_t *pAction = (action_t*) pVoid; + msg_t *pMsg; + int i; + struct syslogTime ttNow; DEFiRet; - assert(pBatch != NULL); - - if(pbShutdownImmediate != NULL) { - pbShutdownImmdtSave = pBatch->pbShutdownImmediate; - pBatch->pbShutdownImmediate = pbShutdownImmediate; - } - CHKiRet(prepareBatch(pAction, pBatch, &activeSave, &bMustRestoreActivePtr)); - - /* We now must guard the output module against execution by multiple threads. The - * plugin interface specifies that output modules must not be thread-safe (except - * if they notify us they are - functionality not yet implemented...). - * rgerhards, 2008-01-30 - */ - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - - iRet = processAction(pAction, pBatch); - - pthread_cleanup_pop(1); /* unlock mutex */ - - /* even if processAction failed, we need to release the batch (else we - * have a memory leak). So we do this first, and then check if we need to - * return an error code. If so, the code from processAction has priority. - * rgerhards, 2010-12-17 - */ - localRet = releaseBatch(pAction, pBatch); + wtiResetExecState(pWti, pBatch); + /* indicate we have not yet read the date */ + ttNow.year = 0; - if(iRet == RS_RET_OK) - iRet = localRet; - - if(bMustRestoreActivePtr) { - free(pBatch->active); - pBatch->active = activeSave; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*pWti->pbShutdownImmediate ; ++i) { + if(batchIsValidElem(pBatch, i)) { + pMsg = pBatch->pElem[i].pMsg; + iRet = processMsgMain(pAction, pWti, pMsg, &ttNow); + // TODO: we must refactor this! flag messages as committed + batchSetElemState(pBatch, i, BATCH_STATE_COMM); + } } -finalize_it: - if(pbShutdownImmediate != NULL) - pBatch->pbShutdownImmediate = pbShutdownImmdtSave; + if(!pWti->execState.bDoAutoCommit) + iRet = actionCommit(pAction, pWti); +dbgprintf("DDDD: processBatchMain - end\n"); RETiRet; } -#pragma GCC diagnostic warning "-Wempty-body" -/* call the HUP handler for a given action, if such a handler is defined. The - * action mutex is locked, because the HUP handler most probably needs to modify - * some internal state information. - * rgerhards, 2008-10-22 +/* call the HUP handler for a given action, if such a handler is defined. + * Note that the action must be able to service HUP requests concurrently + * to any current doAction() processing. */ -#pragma GCC diagnostic ignored "-Wempty-body" rsRetVal actionCallHUPHdlr(action_t *pAction) { @@ -1373,19 +1248,13 @@ actionCallHUPHdlr(action_t *pAction) ASSERT(pAction != NULL); DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP); - if(pAction->pMod->doHUP == NULL) { - FINALIZE; /* no HUP handler, so we are done ;) */ + if(pAction->pMod->doHUP != NULL) { + CHKiRet(pAction->pMod->doHUP(pAction->pModData)); } - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - CHKiRet(pAction->pMod->doHUP(pAction->pModData)); - pthread_cleanup_pop(1); /* unlock mutex */ - finalize_it: RETiRet; } -#pragma GCC diagnostic warning "-Wempty-body" /* set the action message queue mode @@ -1420,27 +1289,28 @@ static rsRetVal setActionQueType(void __attribute__((unused)) *pVal, uchar *pszT /* This submits the message to the action queue in case we do NOT need to handle repeat * message processing. That case permits us to gain lots of freedom during processing - * and thus speed. This is also utilized to submit messages in complex case once + * and thus speed. This is also utilized to submit messages in more complex cases once * the complex logic has been applied ;) * rgerhards, 2010-06-08 */ -static inline rsRetVal -doSubmitToActionQ(action_t *pAction, msg_t *pMsg) +static rsRetVal +doSubmitToActionQ(action_t *pAction, wti_t *pWti, msg_t *pMsg) { + struct syslogTime ttNow; // TODO: think if we can buffer this in pWti DEFiRet; - if(pAction->eState == ACT_STATE_DIED) { - DBGPRINTF("action %p died, do NOT execute\n", pAction); - FINALIZE; - } + DBGPRINTF("Called action, logging to %s\n", module.GetStateName(pAction->pMod)); STATSCOUNTER_INC(pAction->ctrProcessed, pAction->mutCtrProcessed); - if(pAction->pQueue->qType == QUEUETYPE_DIRECT) - iRet = qqueueEnqMsgDirect(pAction->pQueue, MsgAddRef(pMsg)); - else + if(pAction->pQueue->qType == QUEUETYPE_DIRECT) { + ttNow.year = 0; + iRet = processMsgMain(pAction, pWti, pMsg, &ttNow); + } else {/* in this case, we do single submits to the queue. + * TODO: optimize this, we may do at least a multi-submit! + */ iRet = qqueueEnqMsg(pAction->pQueue, eFLOWCTL_NO_DELAY, MsgAddRef(pMsg)); + } -finalize_it: RETiRet; } @@ -1453,7 +1323,7 @@ finalize_it: * be filtered out before calling us (what is done currently!). */ rsRetVal -actionWriteToAction(action_t *pAction, msg_t *pMsg) +actionWriteToAction(action_t *pAction, msg_t *pMsg, wti_t *pWti) { DEFiRet; @@ -1508,44 +1378,46 @@ actionWriteToAction(action_t *pAction, msg_t *pMsg) /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 */ - iRet = doSubmitToActionQ(pAction, pMsg); + iRet = doSubmitToActionQ(pAction, pWti, pMsg); finalize_it: RETiRet; } -/* helper to actonCallAction, mostly needed because of this damn - * pthread_cleanup_push() POSIX macro... +/* Call configured action, most complex case with all features supported (and thus slow). + * rgerhards, 2010-06-08 */ -static inline rsRetVal -doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch) +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +doSubmitToActionQComplex(action_t *pAction, wti_t *pWti, msg_t *pMsg) { - msg_t *pMsg; DEFiRet; - pMsg = pBatch->pElem[idxBtch].pMsg; + d_pthread_mutex_lock(&pAction->mutAction); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutAction); + DBGPRINTF("Called action %p (complex case), logging to %s\n", + pAction, module.GetStateName(pAction->pMod)); + pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */ + // TODO: can we optimize the "now" handling again (was batch, I guess...)? /* don't output marks to recently written outputs */ - if(pAction->bWriteAllMarkMsgs == RSFALSE + if(pAction->bWriteAllMarkMsgs == 0 && (pMsg->msgFlags & MARK) && (getActNow(pAction) - pAction->f_time) < MarkInterval / 2) { ABORT_FINALIZE(RS_RET_OK); } /* call the output driver */ - iRet = actionWriteToAction(pAction, pMsg); + iRet = actionWriteToAction(pAction, pMsg, pWti); finalize_it: - /* we need to update the batch to handle failover processing correctly */ - if(iRet == RS_RET_OK) { - pBatch->pElem[idxBtch].bPrevWasSuspended = 0; - } else if(iRet == RS_RET_ACTION_FAILED) { - pBatch->pElem[idxBtch].bPrevWasSuspended = 1; - } + d_pthread_mutex_unlock(&pAction->mutAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ RETiRet; } +#pragma GCC diagnostic warning "-Wempty-body" /* helper to activateActions, it activates a specific action. @@ -1594,194 +1466,41 @@ activateActions(void) * rgerhards, 2010-06-08 */ static rsRetVal -doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch) +doSubmitToActionQNotAllMark(action_t *pAction, wti_t *pWti, msg_t *pMsg) { - time_t now = 0; + int doProcess = 1; time_t lastAct; - int i; - sbool *activeSave; DEFiRet; - activeSave = pBatch->active; - copyActive(pBatch); - - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if((pBatch->eltState[i] == BATCH_STATE_DISC) || !pBatch->active[i]) - continue; - if(now == 0) { - now = datetime.GetTime(NULL); /* good time call - the only one done */ - } - /* CAS loop, we write back a bit early, but that's OK... */ - /* we use reception time, not dequeue time - this is considered more appropriate and - * also faster ;) -- rgerhards, 2008-09-17 */ - do { - lastAct = pAction->f_time; - if(pBatch->pElem[i].pMsg->msgFlags & MARK) { - if((now - lastAct) < MarkInterval / 2) { - pBatch->active[i] = 0; - DBGPRINTF("batch item %d: action was recently called, ignoring " - "mark message\n", i); - break; /* do not update timestamp for non-written mark messages */ - } - } - } while(ATOMIC_CAS_time_t(&pAction->f_time, lastAct, - pBatch->pElem[i].pMsg->ttGenTime, &pAction->mutCAS) == 0); - if(pBatch->active[i]) { - DBGPRINTF("Called action(NotAllMark), processing batch[%d] via '%s'\n", - i, module.GetStateName(pAction->pMod)); - } - } - - iRet = doSubmitToActionQBatch(pAction, pBatch); - - free(pBatch->active); - pBatch->active = activeSave; - - RETiRet; -} - -static inline void -countStatsBatchEnq(action_t *pAction, batch_t *pBatch) -{ - int i; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if( batchIsValidElem(pBatch, i)) { - STATSCOUNTER_INC(pAction->ctrProcessed, pAction->mutCtrProcessed); - } - } -} - - -/* enqueue a batch in direct mode. We have put this into its own function just to avoid - * cluttering the actual submit function. - * rgerhards, 2011-06-16 - */ -static inline rsRetVal -doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch) -{ - sbool bNeedSubmit; - sbool *activeSave; - int i; - DEFiRet; - - activeSave = pBatch->active; - copyActive(pBatch); - - /* note: for direct mode, we need to adjust the filter property. For non-direct - * this is not necessary, because in that case we enqueue only what actually needs - * to be processed. + /* TODO: think about the whole logic. If messages come in out of order, things + * tend to become a bit unreliable. On the other hand, this only happens if we have + * very high traffic, in which this use case here is not really affected (as the + * MarkInterval is pretty corase). */ - if(pAction->bExecWhenPrevSusp) { - bNeedSubmit = 0; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if(!pBatch->pElem[i].bPrevWasSuspended) { - DBGPRINTF("action enq stage: change active to 0 due to " - "failover case in elem %d\n", i); - pBatch->active[i] = 0; - } - if(batchIsValidElem(pBatch, i)) { - STATSCOUNTER_INC(pAction->ctrProcessed, pAction->mutCtrProcessed); - bNeedSubmit = 1; - } - DBGPRINTF("action %p[%d]: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, i, batchIsValidElem(pBatch, i), pBatch->eltState[i], - pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - } - if(bNeedSubmit) { - /* note: stats were already computed above */ - iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch); - } else { - DBGPRINTF("no need to submit batch, all invalid\n"); - } - } else { - if(GatherStats) - countStatsBatchEnq(pAction, pBatch); - iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch); - } - - free(pBatch->active); - pBatch->active = activeSave; - RETiRet; -} - -/* This submits the message to the action queue in case we do NOT need to handle repeat - * message processing. That case permits us to gain lots of freedom during processing - * and thus speed. - * rgerhards, 2010-06-08 - */ -static rsRetVal -doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch) -{ - int i; - DEFiRet; - - DBGPRINTF("Called action(Batch), logging to %s\n", module.GetStateName(pAction->pMod)); - - if(pAction->pQueue->qType == QUEUETYPE_DIRECT) { - iRet = doQueueEnqObjDirectBatch(pAction, pBatch); - } else {/* in this case, we do single submits to the queue. - * TODO: optimize this, we may do at least a multi-submit! - */ - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - DBGPRINTF("action %p: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, batchIsValidElem(pBatch, i), pBatch->eltState[i], - pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - if( batchIsValidElem(pBatch, i) - && (pAction->bExecWhenPrevSusp == 0 || pBatch->pElem[i].bPrevWasSuspended == 1)) { - doSubmitToActionQ(pAction, pBatch->pElem[i].pMsg); + /* CAS loop, we write back a bit early, but that's OK... */ + /* we use reception time, not dequeue time - this is considered more appropriate and + * also faster ;) -- rgerhards, 2008-09-17 */ + do { + lastAct = pAction->f_time; + if(pMsg->msgFlags & MARK) { + if((pMsg->ttGenTime - lastAct) < MarkInterval / 2) { + doProcess = 0; + DBGPRINTF("action was recently called, ignoring mark message\n"); + break; /* do not update timestamp for non-written mark messages */ } } - } + } while(ATOMIC_CAS_time_t(&pAction->f_time, lastAct, + pMsg->ttGenTime, &pAction->mutCAS) == 0); - RETiRet; -} - - - -/* Helper to submit a batch of actions to the engine. Note that we have rather - * complicated processing here, so we need to do this one message after another. - * rgerhards, 2010-06-23 - */ -static inline rsRetVal -helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) -{ - int i; - DEFiRet; - - DBGPRINTF("Called action %p (complex case), logging to %s\n", - pAction, module.GetStateName(pAction->pMod)); - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - DBGPRINTF("action %p: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, batchIsValidElem(pBatch, i), pBatch->eltState[i], - pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - if( batchIsValidElem(pBatch, i) - && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { - doActionCallAction(pAction, pBatch, i); - } + if(doProcess) { + DBGPRINTF("Called action(NotAllMark), processing via '%s'\n", + module.GetStateName(pAction->pMod)); + iRet = doSubmitToActionQ(pAction, pWti, pMsg); } RETiRet; } -/* Call configured action, most complex case with all features supported (and thus slow). - * rgerhards, 2010-06-08 - */ -#pragma GCC diagnostic ignored "-Wempty-body" -static rsRetVal -doSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) -{ - DEFiRet; - - d_pthread_mutex_lock(&pAction->mutAction); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutAction); - iRet = helperSubmitToActionQComplexBatch(pAction, pBatch); - d_pthread_mutex_unlock(&pAction->mutAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - - RETiRet; -} -#pragma GCC diagnostic warning "-Wempty-body" - /* apply all params from param block to action. This supports the v6 config system. * Defaults must have been set appropriately during action construct! @@ -1832,7 +1551,7 @@ actionApplyCnfParam(action_t *pAction, struct cnfparamvals *pvals) rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, - struct nvlst *lst, int bSuspended) + struct nvlst *lst) { DEFiRet; int i; @@ -1861,7 +1580,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, pAction->bRepMsgHasMsg = cs.bActionRepMsgHasMsg; cs.iActExecEveryNthOccur = 0; /* auto-reset */ cs.iActExecEveryNthOccurTO = 0; /* auto-reset */ - cs.bActionWriteAllMarkMsgs = RSFALSE; /* auto-reset */ + cs.bActionWriteAllMarkMsgs = 1; /* auto-reset */ cs.pszActionName = NULL; /* free again! */ } else { actionApplyCnfParam(pAction, actParams); @@ -1919,15 +1638,10 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, pAction->pMod = pMod; pAction->pModData = pModData; - /* check if the module is compatible with select features (currently no such features exist) */ - pAction->eState = ACT_STATE_RDY; /* action is enabled */ - - if(bSuspended) - actionSuspend(pAction); CHKiRet(actionConstructFinalize(pAction, lst)); - /* TODO: if we exit here, we have a memory leak... */ + /* TODO: if we exit here, we have a (quite acceptable...) memory leak */ *ppAction = pAction; /* finally store the action pointer */ @@ -1963,7 +1677,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus static inline void initConfigVariables(void) { - cs.bActionWriteAllMarkMsgs = RSFALSE; + cs.bActionWriteAllMarkMsgs = 1; cs.glbliActionResumeRetryCount = 0; cs.bActExecWhenPrevSusp = 0; cs.iActExecOnceInterval = 0; @@ -2002,17 +1716,11 @@ actionNewInst(struct nvlst *lst, action_t **ppAction) errmsg.LogError(0, RS_RET_MOD_UNKNOWN, "module name '%s' is unknown", cnfModName); ABORT_FINALIZE(RS_RET_MOD_UNKNOWN); } - iRet = pMod->mod.om.newActInst(cnfModName, lst, &pModData, &pOMSR); - // TODO: check if RS_RET_SUSPENDED is still valid in v6! - if(iRet != RS_RET_OK && iRet != RS_RET_SUSPENDED) { - FINALIZE; /* iRet is already set to error state */ - } + CHKiRet(pMod->mod.om.newActInst(cnfModName, lst, &pModData, &pOMSR)); - if((iRet = addAction(&pAction, pMod, pModData, pOMSR, paramvals, lst, - (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { + if((iRet = addAction(&pAction, pMod, pModData, pOMSR, paramvals, lst)) == RS_RET_OK) { /* check if the module is compatible with select features * (currently no such features exist) */ - pAction->eState = ACT_STATE_RDY; /* action is enabled */ loadConf->actions.nbrActions++; /* one more active action! */ } *ppAction = pAction; @@ -2023,8 +1731,6 @@ finalize_it: RETiRet; } -/* TODO: we are not yet a real object, the ClassInit here just looks like it is.. - */ rsRetVal actionClassInit(void) { DEFiRet; @@ -33,15 +33,6 @@ extern int glbliActionResumeRetryCount; extern int bActionReportSuspension; -typedef enum { - ACT_STATE_DIED = 0, /* action permanently failed and now disabled - MUST BE ZERO! */ - ACT_STATE_RDY = 1, /* action ready, waiting for new transaction */ - ACT_STATE_ITX = 2, /* transaction active, waiting for new data or commit */ - ACT_STATE_COMM = 3, /* transaction finished (a transient state) */ - ACT_STATE_RTRY = 4, /* failure occured, trying to restablish ready state */ - ACT_STATE_SUSP = 5 /* suspended due to failure (return fail until timeout expired) */ -} action_state_t; - /* the following struct defines the action object data structure */ struct action_s { @@ -49,16 +40,15 @@ struct action_s { time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ + int iActionNbr; /* this action's number (ID) */ sbool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ sbool bWriteAllMarkMsgs;/* should all mark msgs be written (not matter how recent the action was executed)? */ - int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ - action_state_t eState; /* current state of action */ sbool bHadAutoCommit; /* did an auto-commit happen during doAction()? */ + sbool bDisabled; + int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ time_t ttResumeRtry; /* when is it time to retry the resume? */ - int iResumeOKinRow; /* number of times in a row that resume said OK with an immediate failure following */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ - int iNbrResRtry; /* number of retries since last suspend */ int iNbrNoExec; /* number of matches that did not yet yield to an exec */ int iExecEveryNthOccur;/* execute this action only every n-th occurence (with n=0,1 -> always) */ int iExecEveryNthOccurTO;/* timeout for n-th occurence feature */ @@ -66,7 +56,7 @@ struct action_s { struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ sbool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ - rsRetVal (*submitToActQ)(action_t *, batch_t *);/* function submit message to action queue */ + rsRetVal (*submitToActQ)(action_t *, wti_t*, msg_t*);/* function submit message to action queue */ rsRetVal (*qConstruct)(struct queue_s *pThis); enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1, ACT_MSG_PASSING = 2, ACT_JSON_PASSING = 3} @@ -76,8 +66,7 @@ struct action_s { * in this order. */ qqueue_t *pQueue; /* action queue */ pthread_mutex_t mutAction; /* primary action mutex */ - pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ - uchar *pszName; /* action name (for documentation) */ + uchar *pszName; /* action name */ DEF_ATOMIC_HELPER_MUT(mutCAS); /* for statistics subsystem */ statsobj_t *statsobj; @@ -94,15 +83,19 @@ struct action_s { rsRetVal actionConstruct(action_t **ppThis); rsRetVal actionConstructFinalize(action_t *pThis, struct nvlst *lst); rsRetVal actionDestruct(action_t *pThis); -rsRetVal actionDbgPrint(action_t *pThis); +//rsRetVal actionDbgPrint(action_t *pThis); rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); -rsRetVal actionWriteToAction(action_t *pAction, msg_t *pMsg); +rsRetVal actionWriteToAction(action_t *pAction, msg_t *pMsg, wti_t*); rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); -rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, struct nvlst *lst, int bSuspended); +rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, struct cnfparamvals *actParams, struct nvlst *lst); rsRetVal activateActions(void); rsRetVal actionNewInst(struct nvlst *lst, action_t **ppAction); rsRetVal actionProcessCnf(struct cnfobj *o); +void actionCommitAllDirect(wti_t *pWti); + +/* external data */ +extern int iActionNbr; #endif /* #ifndef ACTION_H_INCLUDED */ diff --git a/configure.ac b/configure.ac index 83784278..d086c6fc 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[7.5.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[8.1.2],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -379,6 +379,29 @@ if test "$enable_inet" = "yes"; then AC_DEFINE(SYSLOG_INET, 1, [network support is integrated.]) fi +# jemalloc +AC_ARG_ENABLE(jemalloc, + [AS_HELP_STRING([--enable-jemalloc],[Enable jemalloc support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_jemalloc="yes" ;; + no) enable_jemalloc="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-jemalloc) ;; + esac], + [enable_jemalloc="no"] +) +AM_CONDITIONAL(ENABLE_JEMALLOC, test x$enable_jemalloc = xyes) +if test "$enable_jemalloc" = "yes"; then + AC_CHECK_LIB( + [jemalloc], + [malloc_stats_print], + [RT_LIBS="$RT_LIBS -ljemalloc" + AC_DEFINE(HAVE_JEMALLOC, 1, [jemalloc support is integrated.]) + ], + [AC_MSG_FAILURE([jemalloc library is missing])], + [] + ) +fi + # # The following define determines whether the package adheres to the @@ -931,8 +954,7 @@ AC_ARG_ENABLE(mmnormalize, [enable_mmnormalize=no] ) if test "x$enable_mmnormalize" = "xyes"; then - PKG_CHECK_MODULES(LIBEE, libee >= 0.4.0) - PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 0.3.1 lognorm < 1.0.0) + PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 1.0.0) fi AM_CONDITIONAL(ENABLE_MMNORMALIZE, test x$enable_mmnormalize = xyes) @@ -1353,7 +1375,7 @@ AC_ARG_ENABLE(omruleset, no) enable_omruleset="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-omruleset) ;; esac], - [enable_omruleset=yes] + [enable_omruleset=no] ) AM_CONDITIONAL(ENABLE_OMRULESET, test x$enable_omruleset = xyes) @@ -36,6 +36,7 @@ rsRetVal logmsgInternal(const int iErr, const int pri, const uchar *msg, int fla rsRetVal __attribute__((deprecated)) parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime, ruleset_t *pRuleset); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst *lst); +rsRetVal startMainQueue(qqueue_t *pQueue); extern int MarkInterval; extern qqueue_t *pMsgQueue; /* the main message queue */ diff --git a/doc/Makefile.am b/doc/Makefile.am index a7417e44..9a7ed162 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -122,6 +122,9 @@ html_files = \ v3compatibility.html \ v4compatibility.html \ v5compatibility.html \ + v6compatibility.html \ + v7compatibility.html \ + v8compatibility.html \ im3195.html \ netstream.html \ ns_gtls.html \ @@ -158,8 +161,6 @@ html_files = \ rsconf1_omfileforcechown.html \ rsyslog_queue_pointers.jpeg \ rsyslog_queue_pointers2.jpeg \ - v6compatibility.html \ - v7compatibility.html \ rsyslog_conf_basic_structure.html \ rsyslog_conf_sysklogd_compatibility.html \ imkmsg.html \ diff --git a/doc/manual.html b/doc/manual.html index 7e24ea7d..017ece24 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ professional services</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 7.5.7 (devel branch) of rsyslog.</b> +<p><b>This documentation is for version 8.1.2 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -27,16 +27,18 @@ want to lend us a helping hand. </b>It doesn't require a lot of time - even a single mouse click helps. Learn <a href="how2help.html">how to help the rsyslog project</a>. Due to popular demand, there is now a <a href="rsyslog_ng_comparison.html">side-by-side comparison between rsyslog and syslog-ng</a>.</p> -<p>If you are upgrading from rsyslog v2 or stock sysklogd, -<a href="v3compatibility.html">be sure to read the rsyslog v3 compatibility notes</a>, -and if you are upgrading from v3, read the -<a href="v4compatibility.html">rsyslog v4 compatibility notes</a>, -if you upgrade from v4, read the -<a href="v5compatibility.html">rsyslog v5 compatibility notes</a>, and -if you upgrade from v5, read the -<a href="v6compatibility.html">rsyslog v6 compatibility notes</a>. +If you upgrade from v7, read the +<a href="v8compatibility.html">rsyslog v8 compatibility notes</a>. if you upgrade from v6, read the <a href="v7compatibility.html">rsyslog v7 compatibility notes</a>. +if you upgrade from v5, read the +<a href="v6compatibility.html">rsyslog v6 compatibility notes</a>. +if you upgrade from v4, read the +<a href="v5compatibility.html">rsyslog v5 compatibility notes</a>. +if you upgrade from v3, read the +<a href="v4compatibility.html">rsyslog v4 compatibility notes</a>. +<p>If you are upgrading from rsyslog v2 or stock sysklogd, +<a href="v3compatibility.html">be sure to read the rsyslog v3 compatibility notes</a>, <p>Rsyslog will work even if you do not read the doc, but doing so will definitely improve your experience.</p> <p><b>Follow the links below for the</b></p> @@ -44,7 +46,7 @@ if you do not read the doc, but doing so will definitely improve your experience <li><a href="troubleshoot.html">troubleshooting rsyslog problems</a></li> <li><a href="rsyslog_conf.html">configuration file format (rsyslog.conf)</a></li> <li><a href="http://www.rsyslog.com/tool-regex">a regular expression checker/generator tool for rsyslog</a></li> -<li> <a href="property_replacer.html">property replacer, an important core component</a></li> +<li><a href="property_replacer.html">property replacer, an important core component</a></li> <li><a href="bugs.html">rsyslog bug list</a></li> <li><a href="messageparser.html">understanding rsyslog message parsers</a></li> <li><a href="generic_design.html">backgrounder on generic syslog application design</a></li> diff --git a/doc/mmnormalize.html b/doc/mmnormalize.html index fc6ec6d2..f09d0993 100644 --- a/doc/mmnormalize.html +++ b/doc/mmnormalize.html @@ -38,6 +38,11 @@ can be found in the <a href="http://www.liblognorm.com/files/manual/index.html"> <li><b>useRawMsg</b> [boolean]<br> Specifies if the raw message should be used for normalization (on) or just the MSG part of the message (off). Default is "off". +<li><b>path</b> [word], defaults to "$!"<br> +Specifies the JSON path under which parsed elements should be placed. By default, +all parsed properties are merged into root of message properties. You can place them +under a subtree, instead. You can place them in local variables, also, by setting +path="$.". </ul> <p><b>Legacy Configuration Directives</b>:</p> <ul> diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html index 14a761c5..1be2c81e 100644 --- a/doc/multi_ruleset.html +++ b/doc/multi_ruleset.html @@ -89,6 +89,15 @@ modify the ruleset to which the next input is bound but rather provides a system default rule set for those inputs that did not explicitly bind to one. As such, the directive can not be used as a work-around to bind inputs to non-default rulesets that do not support ruleset binding. + +<h2>Rulesets and Main Queues</h2> +By default, rulesets do not have their own queue. It must be activated via +the $RulesetCreateMainQueue directive, or if using rainerscript format, by +specifying queue parameters on the ruleset directive, e.g. +ruleset(name="whatever" queue.type="fixedArray" queue. ...) +See <a href="http://www.rsyslog.com/doc/queue_parameters.html">http://www.rsyslog.com/doc/queue_parameters.html</a> +for more details. + <h2>Examples</h2> <h3>Split local and remote logging</h3> <p>Let's say you have a pretty standard system that logs its local messages to the usual diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html index adca540b..260d1f2b 100644 --- a/doc/rsyslog_conf_actions.html +++ b/doc/rsyslog_conf_actions.html @@ -31,8 +31,14 @@ implemented via <a href="rsyslog_conf_modules.html#om">output modules</a>. <br>used for statistics gathering and documentation <li><b>type</b> string <br>Mandatory parameter for every action. The name of the module that should be used. </li> - <li><b>action.writeAllMarkMessages</b> on/off - <br>Normally, mark messages are written to actions only if the action was not recently executed (by default, recently means within the past 20 minutes). If this setting is switched to "on", mark messages are always sent to actions, no matter how recently they have been executed. In this mode, mark messages can be used as a kind of heartbeat.</li> + <li><b>action.writeAllMarkMessages</b> <i>on</i>/off + <br>This setting tells if mark messages are always written ("on", the default) or only + if the action was not recently executed ("off"). By default, recently means within the + past 20 minutes. If this setting is "on", mark messages are always sent to actions, no + matter how recently they have been executed. In this mode, mark messages can be used as + a kind of heartbeat. This mode also enables faster processing inside the rule engine. So + it should be set to "off" only when there is a good reason to do so. + </li> <li><b>action.execOnlyEveryNthTime</b> integer <br>If configured, the next action will only be executed every n-th time. For example, if configured to 3, the first two messages that go into the action will be dropped, the 3rd will actually cause the action to execute, the 4th and 5th will be dropped, the 6th executed under the action, ... and so on.</li> <li><b>action.execOnlyEveryNthTimeout</b> integer diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 8dc3ed56..99557215 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -53,28 +53,28 @@ to message generators. and messages be transmitted to various different targets. <ul> <li><a href="omfile.html">omfile</a> - file output module</li> -<li><a href="omfwd.html">omfwd</a> - syslog forwarding output module</li> +<li><a href="omfwd.html">omfwd</a> (does NOT yet work in v8) - syslog forwarding output module</li> <li><a href="omjournal.html">omjournal</a> - Linux journal output module</li> <li><a href="ompipe.html">ompipe</a> - named pipe output module</li> <li><a href="omusrmsg.html">omusrmsg</a> - user message output module</li> -<li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> +<li><a href="omsnmp.html">omsnmp</a> (does NOT yet work in v8) - SNMP trap output module</li> <li><a href="omstdout.html">omtdout</a> - stdout output module (mainly a test tool)</li> -<li><a href="omrelp.html">omrelp</a> - RELP output module</li> +<li><a href="omrelp.html">omrelp</a> (does NOT yet work in v8) - RELP output module</li> <li><a href="omruleset.html">omruleset</a> - forward message to another ruleset</li> -<li>omgssapi - output module for GSS-enabled syslog</li> +<li>omgssapi (does NOT yet work in v8) - output module for GSS-enabled syslog</li> <li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> -<li>ompgsql - output module for PostgreSQL</li> -<li><a href="omlibdbi.html">omlibdbi</a> - +<li>ompgsql (does NOT yet work in v8) - output module for PostgreSQL</li> +<li><a href="omlibdbi.html">omlibdbi</a> (does NOT yet work in v8) - generic database output module (Firebird/Interbase, MS SQL, Sybase, SQLLite, Ingres, Oracle, mSQL)</li> -<li><a href="ommail.html">ommail</a> - +<li><a href="ommail.html">ommail</a> (does NOT yet work in v8) - permits rsyslog to alert folks by mail if something important happens</li> -<li><a href="omprog.html">omprog</a> - permits sending messages to a program for custom processing</li> -<li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> -<li><a href="omudpspoof.html">omudpspoof</a> - output module sending UDP syslog messages with a spoofed address</li> -<li><a href="omuxsock.html">omuxsock</a> - output module Unix domain sockets</li> -<li><a href="omhdfs.html">omhdfs</a> - output module for Hadoop's HDFS file system</li> -<li><a href="ommongodb.html">ommongodb</a> - output module for MongoDB</li> +<li><a href="omprog.html">omprog</a> (does NOT yet work in v8) - permits sending messages to a program for custom processing</li> +<li><a href="omoracle.html">omoracle</a> (orphaned) - output module for Oracle (native OCI interface)</li> +<li><a href="omudpspoof.html">omudpspoof</a> (does NOT yet work in v8) - output module sending UDP syslog messages with a spoofed address</li> +<li><a href="omuxsock.html">omuxsock</a> (does NOT yet work in v8) - output module Unix domain sockets</li> +<li><a href="omhdfs.html">omhdfs</a> (does NOT yet work in v8) - output module for Hadoop's HDFS file system</li> +<li><a href="ommongodb.html">ommongodb</a> (does NOT yet work in v8) - output module for MongoDB</li> <li><a href="omelasticsearch.html">omelasticsearch</a> - output module for ElasticSearch</li> </ul> diff --git a/doc/v8compatibility.html b/doc/v8compatibility.html new file mode 100644 index 00000000..549a7fef --- /dev/null +++ b/doc/v8compatibility.html @@ -0,0 +1,119 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Compatibility notes for rsyslog v8</title> +</head> +<body> +<h1>Compatibility Notes for rsyslog v8</h1> +This document describes things to keep in mind when moving from v7 to v8. It +does not list enhancements nor does it talk about compatibility concerns introduced +by earlier versions (for this, see their respective compatibility documents). Its focus +is primarily on what you need to know if you used v7 and want to use v8 without hassle. +<p>Version 8 offers a completely rewritten core rsyslog engine. This resulted in +a number of changes that are visible to users and (plugin) developers. +Most importantly, pre-v8 plugins <b>do not longer work</b> and need to +be updated to support the new calling interfaces. If you developed a plugin, +be sure to review the developer section below. +</p> +<h2>Mark Messages</h2> +<p>In previous versions, mark messages were by default only processed if an +action was not executed for some time. The default has now changed, and mark +messages are now always processed. Note that this enables faster processing +inside rsyslog. To change to previous behaviour, you need to add +action.writeAllMarkMessages="off" to the actions in question. + +<h2>Modules WITHOUT v8 Support</h2> +<p>The following modules will not properly build under rsyslog v8 and +need to be updated: +<ul> +<li>plugins/omgssapi +<li>plugins/omhdfs +<li>plugins/omrabbitmq - not project supported +<li>plugins/omzmq3 - not project supported +<li>plugins/mmrfc5424addhmac - was a custom project, requires sponsoring + for conversion +<li>plugins/omoracle - orphaned since a while -- use omlibdbi instead +<li>plugins/mmsnmptrapd - waiting for demand +<li>plugins/mmcount - scheduled to be updated soon +</ul> + +<h2>Untested Modules</h2> +<p>The following modules have been updated and successfully +build, but no "real" test were conducted. Users of these modules +should use extra care. +<ul> +<li>mmsequence +<li>omprog +<li>omsnmp +<li>mmfields +<li>mmpstrucdata +<li>plugins/mmaudit +<li>omlibdbi - will be tested soon +<li>ommongodb - larger changes still outstanding +<li>ompgsql - larger chages still outstanding +<li>omuxsock +</ul> +<p>In addition to bug reports, success reports are also appreciated for +these modules (this may save us testing). + +<h2>What Developers need to Know</h2> +<h3>output plugin interface</h3> +<p>To support the new core engine, the output interface has been considerably +changed. It is suggested to review some of the project-provided plugins for +full details. In this doc, we describe the most important changes from a high +level perspective. +<p><b>NOTE: the v8 output module interface is not yet stable.</b> It is highly +likely that additional changes will be made within the next weeks. So if you +convert a module now, be prepared for additional mandatory changes. + +<p><b>Multi-thread awareness required</b></p> +<p>The new engine activates one <b>worker</b>instance of output actions on +each worker thread. This means an action has now three types of data: +<ul> +<li>global +<li>action-instance - previously known pData, one for each action inside the config +<li>worker-action-instance - one for each worker thread (called pWrkrData), note +that this is specific to exactly one pData +</ul> +The plugin <b>must</b> now by multi-threading aware. It may be called by multiple +threads concurrently, but it is guaranteed that each call is for a unique +pWrkrData structure. This still permits to write plugins easily, but enables the +engine to work with much higher performance. Note that plugin developers +should assume it is the norm that multiple concurrent worker action instances +are active a the some time. + +<p><b>New required entry points</b></p> +<p>In order to support the new threading model, new entry points are required. +Most importantly, only the plugin knows which data must be present in pData and +pWrkrData, so it must created and destroy these data structures on request of +the engine. Note that pWrkrData may be destroyed at any time and new ones +re-created later. Depending on workload structure and configuration, this can happen +frequently. +<p>New entry points are: +<ul> +<li>createWrkrInstance +<li>freeWrkrInstance +</ul> + +The calling interface for these entry points has changed. Basically, +they now receive a pWrkrData object instead pData. It is assumed that +createWrkrInstance populates pWrkrData->pData appropriately. +<ul> +<li>beginTransaction +<li>doAction +<li>endTransaction +</ul> + +<p><b>RS_RET_SUSPENDED is no longer supported when creating an action instance</b> +<p>This means a plugin must not try to establish any connections or the like +before any of its processing entry points (like beginTransaction or doAction) +is called. This was generally also the case von v7, but was not enforced in +all cases. In v8, creating action creation fails if anything but RS_RET_OK +is returned. + +<p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> + +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2013 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body></html> diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index e4f85860..427e1abb 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -2428,7 +2428,7 @@ cnfstmtPrintOnly(struct cnfstmt *stmt, int indent, sbool subtree) free(cstr); break; case S_ACT: - doIndent(indent); dbgprintf("ACTION %p [%s:%s]\n", stmt->d.act, + doIndent(indent); dbgprintf("ACTION %d [%s:%s]\n", stmt->d.act->iActionNbr, modGetName(stmt->d.act->pMod), stmt->printable); break; case S_IF: diff --git a/plugins/impstats/impstats.c b/plugins/impstats/impstats.c index 694c07c4..a883ef1b 100644 --- a/plugins/impstats/impstats.c +++ b/plugins/impstats/impstats.c @@ -377,7 +377,6 @@ checkRuleset(modConfData_t *modConf) DEFiRet; modConf->pBindRuleset = NULL; /* assume default ruleset */ -dbgprintf("DDDD: impstats ruleset %s\n", modConf->pszBindRuleset); if(modConf->pszBindRuleset == NULL) FINALIZE; @@ -390,7 +389,6 @@ dbgprintf("DDDD: impstats ruleset %s\n", modConf->pszBindRuleset); CHKiRet(localRet); modConf->pBindRuleset = pRuleset; finalize_it: -dbgprintf("DDDD: impstats ruleset ptr %p\n", modConf->pBindRuleset); RETiRet; } diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 4bce0c1c..a53ce159 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -398,7 +398,7 @@ addListner(instanceConf_t *inst) listeners[nfd].flags = inst->bIgnoreTimestamp ? IGNDATE : NOFLAG; listeners[nfd].bCreatePath = inst->bCreatePath; listeners[nfd].sockName = ustrdup(inst->sockName); - listeners[nfd].bUseCreds = (inst->bDiscardOwnMsgs || inst->bWritePid || inst->ratelimitInterval || inst->bAnnotate) ? 1 : 0; + listeners[nfd].bUseCreds = (inst->bDiscardOwnMsgs || inst->bWritePid || inst->ratelimitInterval || inst->bAnnotate || inst->bUseSysTimeStamp) ? 1 : 0; listeners[nfd].bAnnotate = inst->bAnnotate; listeners[nfd].bParseTrusted = inst->bParseTrusted; listeners[nfd].bDiscardOwnMsgs = inst->bDiscardOwnMsgs; diff --git a/plugins/mmanon/mmanon.c b/plugins/mmanon/mmanon.c index 16a4f34b..28797807 100644 --- a/plugins/mmanon/mmanon.c +++ b/plugins/mmanon/mmanon.c @@ -71,6 +71,10 @@ typedef struct _instanceData { } ipv4; } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -119,6 +123,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -130,6 +138,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + static inline void setInstParamDefaults(instanceData *pData) { @@ -358,7 +371,7 @@ CODESTARTdoAction lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); for(i = 0 ; i < lenMsg ; ++i) { - anonip(pData, msg, &lenMsg, &i); + anonip(pWrkrData->pData, msg, &lenMsg, &i); } if(lenMsg != getMSGLen(pMsg)) setMSGLen(pMsg, lenMsg); @@ -387,6 +400,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/mmaudit/mmaudit.c b/plugins/mmaudit/mmaudit.c index c7cff2cb..75f8dd4b 100644 --- a/plugins/mmaudit/mmaudit.c +++ b/plugins/mmaudit/mmaudit.c @@ -14,7 +14,7 @@ * * File begun on 2012-02-23 by RGerhards * - * Copyright 2012 Adiscon GmbH. + * Copyright 2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -69,6 +69,11 @@ typedef struct _instanceData { int dummy; /* remove when the first real parameter is needed */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + + BEGINinitConfVars /* (re)set config variables to default values */ CODESTARTinitConfVars resetConfigVariables(NULL, NULL); @@ -79,6 +84,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -89,6 +98,10 @@ BEGINfreeInstance CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -302,6 +315,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES ENDqueryEtryPt diff --git a/plugins/mmfields/mmfields.c b/plugins/mmfields/mmfields.c index fa7fa100..c408a6c9 100644 --- a/plugins/mmfields/mmfields.c +++ b/plugins/mmfields/mmfields.c @@ -56,6 +56,10 @@ typedef struct _instanceData { uchar *jsonRoot; /**< container where to store fields */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -103,6 +107,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -114,6 +122,10 @@ CODESTARTfreeInstance free(pData->jsonRoot); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + static inline void setInstParamDefaults(instanceData *pData) @@ -232,7 +244,7 @@ CODESTARTdoAction pMsg = (msg_t*) ppString[0]; lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); - CHKiRet(parse_fields(pData, pMsg, msg, lenMsg)); + CHKiRet(parse_fields(pWrkrData->pData, pMsg, msg, lenMsg)); finalize_it: ENDdoAction @@ -259,6 +271,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/mmjsonparse/mmjsonparse.c b/plugins/mmjsonparse/mmjsonparse.c index b16aef0e..9c0ab88f 100644 --- a/plugins/mmjsonparse/mmjsonparse.c +++ b/plugins/mmjsonparse/mmjsonparse.c @@ -58,9 +58,15 @@ DEFobjCurrIf(errmsg); DEF_OMOD_STATIC_DATA typedef struct _instanceData { - struct json_tokener *tokener; + int dummy; /* not needed, but some compilers do not support empty structs */ + /* REMOVE dummy when real data items are to be added! */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; + struct json_tokener *tokener; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -94,14 +100,18 @@ ENDfreeCnf BEGINcreateInstance CODESTARTcreateInstance - pData->tokener = json_tokener_new(); - if(pData->tokener == NULL) { +ENDcreateInstance + +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->tokener = json_tokener_new(); + if(pWrkrData->tokener == NULL) { errmsg.LogError(0, RS_RET_ERR, "error: could not create json " - "tokener, cannot activate action"); + "tokener, cannot activate instance"); ABORT_FINALIZE(RS_RET_ERR); } finalize_it: -ENDcreateInstance +ENDcreateWrkrInstance BEGINisCompatibleWithFeature @@ -111,10 +121,14 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - if(pData->tokener != NULL) - json_tokener_free(pData->tokener); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + if(pWrkrData->tokener != NULL) + json_tokener_free(pWrkrData->tokener); +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -128,28 +142,28 @@ ENDtryResume static rsRetVal -processJSON(instanceData *pData, msg_t *pMsg, char *buf, size_t lenBuf) +processJSON(wrkrInstanceData_t *pWrkrData, msg_t *pMsg, char *buf, size_t lenBuf) { struct json_object *json; const char *errMsg; DEFiRet; - assert(pData->tokener != NULL); + assert(pWrkrData->tokener != NULL); DBGPRINTF("mmjsonparse: toParse: '%s'\n", buf); - json_tokener_reset(pData->tokener); + json_tokener_reset(pWrkrData->tokener); - json = json_tokener_parse_ex(pData->tokener, buf, lenBuf); + json = json_tokener_parse_ex(pWrkrData->tokener, buf, lenBuf); if(Debug) { errMsg = NULL; if(json == NULL) { enum json_tokener_error err; - err = pData->tokener->err; + err = pWrkrData->tokener->err; if(err != json_tokener_continue) errMsg = json_tokener_errors[err]; else errMsg = "Unterminated input"; - } else if((size_t)pData->tokener->char_offset < lenBuf) + } else if((size_t)pWrkrData->tokener->char_offset < lenBuf) errMsg = "Extra characters after JSON object"; else if(!json_object_is_type(json, json_type_object)) errMsg = "JSON value is not an object"; @@ -159,7 +173,7 @@ processJSON(instanceData *pData, msg_t *pMsg, char *buf, size_t lenBuf) } } if(json == NULL - || ((size_t)pData->tokener->char_offset < lenBuf) + || ((size_t)pWrkrData->tokener->char_offset < lenBuf) || (!json_object_is_type(json, json_type_object))) { ABORT_FINALIZE(RS_RET_NO_CEE_MSG); } @@ -194,7 +208,7 @@ CODESTARTdoAction ABORT_FINALIZE(RS_RET_NO_CEE_MSG); } buf += LEN_COOKIE; - CHKiRet(processJSON(pData, pMsg, (char*) buf, strlen((char*)buf))); + CHKiRet(processJSON(pWrkrData, pMsg, (char*) buf, strlen((char*)buf))); bSuccess = 1; finalize_it: if(iRet == RS_RET_NO_CEE_MSG) { @@ -257,6 +271,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/mmnormalize/Makefile.am b/plugins/mmnormalize/Makefile.am index 0a3b5ba5..6a50264d 100644 --- a/plugins/mmnormalize/Makefile.am +++ b/plugins/mmnormalize/Makefile.am @@ -1,8 +1,8 @@ pkglib_LTLIBRARIES = mmnormalize.la mmnormalize_la_SOURCES = mmnormalize.c -mmnormalize_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBLOGNORM_CFLAGS) $(LIBEE_CFLAGS) -mmnormalize_la_LDFLAGS = -module -avoid-version $(LIBLOGNORM_LIBS) $(LIBEE_LIBS) +mmnormalize_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBLOGNORM_CFLAGS) +mmnormalize_la_LDFLAGS = -module -avoid-version $(LIBLOGNORM_LIBS) mmnormalize_la_LIBADD = EXTRA_DIST = diff --git a/plugins/mmnormalize/mmnormalize.c b/plugins/mmnormalize/mmnormalize.c index 7e25824a..ba2e730d 100644 --- a/plugins/mmnormalize/mmnormalize.c +++ b/plugins/mmnormalize/mmnormalize.c @@ -1,15 +1,12 @@ /* mmnormalize.c * This is a message modification module. It normalizes the input message with - * the help of liblognorm. The messages EE event structure is updated. + * the help of liblognorm. The message's JSON variables are updated. * * NOTE: read comments in module-template.h for details on the calling interface! * - * TODO: check if we can replace libee via JSON system - currently that part - * is pretty inefficient... rgerhards, 2012-08-27 - * * File begun on 2010-01-01 by RGerhards * - * Copyright 2010-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -39,7 +36,6 @@ #include <errno.h> #include <unistd.h> #include <libestr.h> -#include <libee/libee.h> #include <json.h> #include <liblognorm.h> #include "conf.h" @@ -67,9 +63,13 @@ typedef struct _instanceData { sbool bUseRawMsg; /**< use %rawmsg% instead of %msg% */ uchar *rulebase; /**< name of rulebase to use */ ln_ctx ctxln; /**< context to be used for liblognorm */ - ee_ctx ctxee; /**< context to be used for libee */ + char *pszPath; /**< path of normalized data */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { uchar *rulebase; /**< name of normalization rulebase to use */ int bUseRawMsg; /**< use %rawmsg% instead of %msg% */ @@ -80,6 +80,7 @@ static configSettings_t cs; /* action (instance) parameters */ static struct cnfparamdescr actpdescr[] = { { "rulebase", eCmdHdlrGetWord, 1 }, + { "path", eCmdHdlrGetWord, 0 }, { "userawmsg", eCmdHdlrBinary, 0 } }; static struct cnfparamblk actpblk = @@ -96,30 +97,21 @@ static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current l static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ -/* to be called to build the libee part of the instance ONCE ALL PARAMETERS ARE CORRECT +/* to be called to build the liblognorm part of the instance ONCE ALL PARAMETERS ARE CORRECT * (and set within pData!). */ static rsRetVal buildInstance(instanceData *pData) { DEFiRet; - if((pData->ctxee = ee_initCtx()) == NULL) { - errmsg.LogError(0, RS_RET_ERR_LIBEE_INIT, "error: could not initialize libee " - "ctx, cannot activate action"); - ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT); - } - if((pData->ctxln = ln_initCtx()) == NULL) { errmsg.LogError(0, RS_RET_ERR_LIBLOGNORM_INIT, "error: could not initialize " "liblognorm ctx, cannot activate action"); - ee_exitCtx(pData->ctxee); ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_INIT); } - ln_setEECtx(pData->ctxln, pData->ctxee); if(ln_loadSamples(pData->ctxln, (char*) pData->rulebase) != 0) { errmsg.LogError(0, RS_RET_NO_RULEBASE, "error: normalization rulebase '%s' " - "could not be loaded cannot activate action", cs.rulebase); - ee_exitCtx(pData->ctxee); + "could not be loaded cannot activate action", pData->rulebase); ln_exitCtx(pData->ctxln); ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_SAMPDB_LOAD); } @@ -139,6 +131,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINbeginCnfLoad CODESTARTbeginCnfLoad loadModConf = pModConf; @@ -176,11 +173,16 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance free(pData->rulebase); - ee_exitCtx(pData->ctxee); ln_exitCtx(pData->ctxln); + free(pData->pszPath); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo dbgprintf("mmnormalize\n"); @@ -193,50 +195,28 @@ ENDtryResume BEGINdoAction msg_t *pMsg; - es_str_t *str; uchar *buf; - char *cstrJSON; int len; int r; - struct ee_event *event = NULL; - struct json_tokener *tokener; - struct json_object *json; + struct json_object *json = NULL; CODESTARTdoAction pMsg = (msg_t*) ppString[0]; - /* note that we can performance-optimize the interface, but this also - * requires changes to the libraries. For now, we accept message - * duplication. -- rgerhards, 2010-12-01 - */ - if(pData->bUseRawMsg) { + if(pWrkrData->pData->bUseRawMsg) { getRawMsg(pMsg, &buf, &len); } else { buf = getMSG(pMsg); len = getMSGLen(pMsg); } - str = es_newStrFromCStr((char*)buf, len); - r = ln_normalize(pData->ctxln, str, &event); + r = ln_normalize(pWrkrData->pData->ctxln, (char*)buf, len, &json); if(r != 0) { DBGPRINTF("error %d during ln_normalize\n", r); MsgSetParseSuccess(pMsg, 0); } else { MsgSetParseSuccess(pMsg, 1); } - es_deleteStr(str); - - /* reformat to our json data struct */ - /* TODO: this is all extremly ineffcient! */ - ee_fmtEventToJSON(event, &str); - cstrJSON = es_str2cstr(str, NULL); - ee_deleteEvent(event); - dbgprintf("mmnormalize generated: %s\n", cstrJSON); - - tokener = json_tokener_new(); - json = json_tokener_parse_ex(tokener, cstrJSON, strlen((char*)cstrJSON)); - json_tokener_free(tokener); - msgAddJSON(pMsg, (uchar*)"!", json); - - free(cstrJSON); - es_deleteStr(str); + + msgAddJSON(pMsg, (uchar*)pWrkrData->pData->pszPath + 1, json); + ENDdoAction @@ -245,12 +225,14 @@ setInstParamDefaults(instanceData *pData) { pData->rulebase = NULL; pData->bUseRawMsg = 0; + pData->pszPath = strdup("$!"); } BEGINnewActInst struct cnfparamvals *pvals; int i; int bDestructPValsOnExit; + char *cstr; CODESTARTnewActInst DBGPRINTF("newActInst (mmnormalize)\n"); @@ -278,6 +260,23 @@ CODESTARTnewActInst pData->rulebase = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "userawmsg")) { pData->bUseRawMsg = (int) pvals[i].val.d.n; + } else if(!strcmp(actpblk.descr[i].name, "path")) { + cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + if (strlen(cstr) < 2) { + errmsg.LogError(0, RS_RET_VALUE_NOT_SUPPORTED, + "mmnormalize: valid path name should be at least " + "2 symbols long, got %s", cstr); + free(cstr); + } else if (cstr[0] != '$') { + errmsg.LogError(0, RS_RET_VALUE_NOT_SUPPORTED, + "mmnormalize: valid path name should start with $," + "got %s", cstr); + free(cstr); + } else { + free(pData->pszPath); + pData->pszPath = cstr; + } + continue; } else { DBGPRINTF("mmnormalize: program error, non-handled " "param '%s'\n", actpblk.descr[i].name); @@ -313,6 +312,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) pData->rulebase = cs.rulebase; pData->bUseRawMsg = cs.bUseRawMsg; + pData->pszPath = strdup("$!"); /* old interface does not support this feature */ /* all config vars auto-reset! */ cs.bUseRawMsg = 0; cs.rulebase = NULL; /* we used it up! */ @@ -338,6 +338,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/mmpstrucdata/mmpstrucdata.c b/plugins/mmpstrucdata/mmpstrucdata.c index 123363bc..680ba92b 100644 --- a/plugins/mmpstrucdata/mmpstrucdata.c +++ b/plugins/mmpstrucdata/mmpstrucdata.c @@ -53,6 +53,10 @@ typedef struct _instanceData { uchar *jsonRoot; /**< container where to store fields */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -99,6 +103,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -110,6 +118,10 @@ CODESTARTfreeInstance free(pData->jsonRoot); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + static inline void setInstParamDefaults(instanceData *pData) @@ -359,7 +371,7 @@ dbgprintf("DDDD: parse mmpstrucdata\n"); /* don't check return code - we never want rsyslog to retry * or suspend this action! */ - parse_sd(pData, pMsg); + parse_sd(pWrkrData->pData, pMsg); dbgprintf("DDDD: done parse mmpstrucdata\n"); finalize_it: ENDdoAction @@ -387,6 +399,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/mmsequence/mmsequence.c b/plugins/mmsequence/mmsequence.c index 20a85370..d1ea85b6 100644 --- a/plugins/mmsequence/mmsequence.c +++ b/plugins/mmsequence/mmsequence.c @@ -74,6 +74,10 @@ typedef struct _instanceData { char *pszVar; } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -100,6 +104,8 @@ static struct cnfparamblk actpblk = /* table for key-counter pairs */ static struct hashtable *ght; static pthread_mutex_t ght_mutex = PTHREAD_MUTEX_INITIALIZER; + +static pthread_mutex_t inst_mutex = PTHREAD_MUTEX_INITIALIZER; BEGINbeginCnfLoad CODESTARTbeginCnfLoad @@ -129,6 +135,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -139,6 +149,10 @@ BEGINfreeInstance CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + static inline void setInstParamDefaults(instanceData *pData) @@ -243,7 +257,7 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_ERR); } } - pthread_mutex_unlock(&ght_mutex); + pthread_mutex_unlock(&ght_mutex); break; default: errmsg.LogError(0, RS_RET_INVLD_MODE, @@ -303,7 +317,9 @@ BEGINdoAction struct json_object *json; int val = 0; int *pCounter; + instanceData *pData; CODESTARTdoAction + pData = pWrkrData->pData; pMsg = (msg_t*) ppString[0]; switch(pData->mode) { @@ -312,12 +328,18 @@ CODESTARTdoAction (pData->valueTo - pData->valueFrom)); break; case mmSequencePerInstance: - if (pData->value >= pData->valueTo - pData->step) { - pData->value = pData->valueFrom; + if (!pthread_mutex_lock(&inst_mutex)) { + if (pData->value >= pData->valueTo - pData->step) { + pData->value = pData->valueFrom; + } else { + pData->value += pData->step; + } + val = pData->value; + pthread_mutex_unlock(&inst_mutex); } else { - pData->value += pData->step; + errmsg.LogError(0, RS_RET_ERR, + "mmsequence: mutex lock has failed!"); } - val = pData->value; break; case mmSequencePerKey: if (!pthread_mutex_lock(&ght_mutex)) { @@ -381,6 +403,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/mmutf8fix/mmutf8fix.c b/plugins/mmutf8fix/mmutf8fix.c index e2077950..351bb129 100644 --- a/plugins/mmutf8fix/mmutf8fix.c +++ b/plugins/mmutf8fix/mmutf8fix.c @@ -60,6 +60,10 @@ typedef struct _instanceData { uint8_t mode; /* operations mode */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -108,6 +112,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature @@ -118,6 +127,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + static inline void setInstParamDefaults(instanceData *pData) { @@ -274,10 +288,10 @@ CODESTARTdoAction pMsg = (msg_t*) ppString[0]; lenMsg = getMSGLen(pMsg); msg = getMSG(pMsg); - if(pData->mode == MODE_CC) { - doCC(pData, msg, lenMsg); + if(pWrkrData->pData->mode == MODE_CC) { + doCC(pWrkrData->pData, msg, lenMsg); } else { - doUTF8(pData, msg, lenMsg); + doUTF8(pWrkrData->pData, msg, lenMsg); } ENDdoAction @@ -304,6 +318,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/omelasticsearch/README b/plugins/omelasticsearch/README index 9021bc0e..b8bf4151 100644 --- a/plugins/omelasticsearch/README +++ b/plugins/omelasticsearch/README @@ -1,3 +1,7 @@ +How to access ElasticSearch on local machine (for testing): +=========================================================== +see: https://github.com/mobz/elasticsearch-head + How to produce an error: ======================== It's quite easy to get 400, if you put a wrong mapping to your diff --git a/plugins/omelasticsearch/omelasticsearch.c b/plugins/omelasticsearch/omelasticsearch.c index eb82c35f..b878050d 100644 --- a/plugins/omelasticsearch/omelasticsearch.c +++ b/plugins/omelasticsearch/omelasticsearch.c @@ -69,8 +69,8 @@ STATSCOUNTER_DEF(indexESFail, mutIndexESFail) typedef struct curl_slist HEADER; typedef struct _instanceData { int port; - int replyLen; int fdErrFile; /* error file fd or -1 if not open */ + pthread_mutex_t mutErrFile; uchar *server; uchar *uid; uchar *pwd; @@ -80,24 +80,29 @@ typedef struct _instanceData { uchar *tplName; uchar *timeout; uchar *bulkId; - uchar *restURL; /* last used URL for error reporting */ uchar *errorFile; - char *reply; sbool dynSrchIdx; sbool dynSrchType; sbool dynParent; sbool dynBulkId; sbool bulkmode; sbool asyncRepl; +} instanceData; + +typedef struct wrkrInstanceData { + instanceData *pData; + int replyLen; + char *reply; + CURL *curlHandle; /* libcurl session handle */ + HEADER *postHeader; /* json POST request info */ + uchar *restURL; /* last used URL for error reporting */ struct { es_str_t *data; int nmemb; /* number of messages in batch (for statistics counting) */ uchar *currTpl1; uchar *currTpl2; } batch; - CURL *curlHandle; /* libcurl session handle */ - HEADER *postHeader; /* json POST request info */ -} instanceData; +} wrkrInstanceData_t; /* tables for interfacing with the v6 config system */ @@ -117,7 +122,7 @@ static struct cnfparamdescr actpdescr[] = { { "asyncrepl", eCmdHdlrBinary, 0 }, { "timeout", eCmdHdlrGetWord, 0 }, { "errorfile", eCmdHdlrGetWord, 0 }, - { "template", eCmdHdlrGetWord, 1 }, + { "template", eCmdHdlrGetWord, 0 }, { "dynbulkid", eCmdHdlrBinary, 0 }, { "bulkid", eCmdHdlrGetWord, 0 }, }; @@ -127,12 +132,32 @@ static struct cnfparamblk actpblk = actpdescr }; +static rsRetVal curlSetup(wrkrInstanceData_t *pWrkrData, instanceData *pData); + BEGINcreateInstance CODESTARTcreateInstance - pData->restURL = NULL; pData->fdErrFile = -1; + pthread_mutex_init(&pData->mutErrFile, NULL); ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +dbgprintf("omelasticsearch: createWrkrInstance\n"); + pWrkrData->restURL = NULL; + if(pData->bulkmode) { + pWrkrData->batch.currTpl1 = NULL; + pWrkrData->batch.currTpl2 = NULL; + if((pWrkrData->batch.data = es_newStr(1024)) == NULL) { + DBGPRINTF("omelasticsearch: error creating batch string " + "turned off bulk mode\n"); + pData->bulkmode = 0; /* at least it works */ + } + } + CHKiRet(curlSetup(pWrkrData, pWrkrData->pData)); +finalize_it: +dbgprintf("DDDD: createWrkrInstance,pData %p/%p, pWrkrData %p\n", pData, pWrkrData->pData, pWrkrData); +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -141,16 +166,9 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - if (pData->postHeader) { - curl_slist_free_all(pData->postHeader); - pData->postHeader = NULL; - } - if (pData->curlHandle) { - curl_easy_cleanup(pData->curlHandle); - pData->curlHandle = NULL; - } if(pData->fdErrFile != -1) close(pData->fdErrFile); + pthread_mutex_destroy(&pData->mutErrFile); free(pData->server); free(pData->uid); free(pData->pwd); @@ -159,11 +177,23 @@ CODESTARTfreeInstance free(pData->parent); free(pData->tplName); free(pData->timeout); - free(pData->restURL); free(pData->errorFile); free(pData->bulkId); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + if(pWrkrData->postHeader) { + curl_slist_free_all(pWrkrData->postHeader); + pWrkrData->postHeader = NULL; + } + if(pWrkrData->curlHandle) { + curl_easy_cleanup(pWrkrData->curlHandle); + pWrkrData->curlHandle = NULL; + } + free(pWrkrData->restURL); +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo dbgprintf("omelasticsearch\n"); @@ -211,7 +241,7 @@ setBaseURL(instanceData *pData, es_str_t **url) static inline rsRetVal -checkConn(instanceData *pData) +checkConn(wrkrInstanceData_t *pWrkrData) { es_str_t *url; CURL *curl = NULL; @@ -219,7 +249,7 @@ checkConn(instanceData *pData) char *cstr; DEFiRet; - setBaseURL(pData, &url); + setBaseURL(pWrkrData->pData, &url); curl = curl_easy_init(); if(curl == NULL) { DBGPRINTF("omelasticsearch: checkConn() curl_easy_init() failed\n"); @@ -235,16 +265,16 @@ checkConn(instanceData *pData) curl_easy_setopt(curl, CURLOPT_URL, cstr); free(cstr); - pData->reply = NULL; - pData->replyLen = 0; - curl_easy_setopt(curl, CURLOPT_WRITEDATA, pData); + pWrkrData->reply = NULL; + pWrkrData->replyLen = 0; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, pWrkrData); res = curl_easy_perform(curl); if(res != CURLE_OK) { DBGPRINTF("omelasticsearch: checkConn() curl_easy_perform() " "failed: %s\n", curl_easy_strerror(res)); ABORT_FINALIZE(RS_RET_SUSPENDED); } - free(pData->reply); + free(pWrkrData->reply); DBGPRINTF("omelasticsearch: checkConn() completed with success\n"); finalize_it: @@ -257,7 +287,7 @@ finalize_it: BEGINtryResume CODESTARTtryResume DBGPRINTF("omelasticsearch: tryResume called\n"); - iRet = checkConn(pData); + iRet = checkConn(pWrkrData); ENDtryResume @@ -330,7 +360,7 @@ getIndexTypeAndParent(instanceData *pData, uchar **tpls, static rsRetVal -setCurlURL(instanceData *pData, uchar **tpls) +setCurlURL(wrkrInstanceData_t *pWrkrData, instanceData *pData, uchar **tpls) { char authBuf[1024]; uchar *searchIndex; @@ -368,11 +398,11 @@ setCurlURL(instanceData *pData, uchar **tpls) if(r == 0) r = es_addBuf(&url, (char*)parent, ustrlen(parent)); } - free(pData->restURL); - pData->restURL = (uchar*)es_str2cstr(url, NULL); - curl_easy_setopt(pData->curlHandle, CURLOPT_URL, pData->restURL); + free(pWrkrData->restURL); + pWrkrData->restURL = (uchar*)es_str2cstr(url, NULL); + curl_easy_setopt(pWrkrData->curlHandle, CURLOPT_URL, pWrkrData->restURL); es_deleteStr(url); - DBGPRINTF("omelasticsearch: using REST URL: '%s'\n", pData->restURL); + DBGPRINTF("omelasticsearch: using REST URL: '%s'\n", pWrkrData->restURL); if(pData->uid != NULL) { rLocal = snprintf(authBuf, sizeof(authBuf), "%s:%s", pData->uid, @@ -383,8 +413,8 @@ setCurlURL(instanceData *pData, uchar **tpls) rLocal); ABORT_FINALIZE(RS_RET_ERR); } - curl_easy_setopt(pData->curlHandle, CURLOPT_USERPWD, authBuf); - curl_easy_setopt(pData->curlHandle, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curl_easy_setopt(pWrkrData->curlHandle, CURLOPT_USERPWD, authBuf); + curl_easy_setopt(pWrkrData->curlHandle, CURLOPT_PROXYAUTH, CURLAUTH_ANY); } finalize_it: RETiRet; @@ -396,7 +426,7 @@ finalize_it: * index changes. */ static rsRetVal -buildBatch(instanceData *pData, uchar *message, uchar **tpls) +buildBatch(wrkrInstanceData_t *pWrkrData, uchar *message, uchar **tpls) { int length = strlen((char *)message); int r; @@ -411,29 +441,29 @@ buildBatch(instanceData *pData, uchar *message, uchar **tpls) # define META_ID "\", \"_id\":\"" # define META_END "\"}}\n" - getIndexTypeAndParent(pData, tpls, &searchIndex, &searchType, &parent, &bulkId); - r = es_addBuf(&pData->batch.data, META_STRT, sizeof(META_STRT)-1); - if(r == 0) r = es_addBuf(&pData->batch.data, (char*)searchIndex, + getIndexTypeAndParent(pWrkrData->pData, tpls, &searchIndex, &searchType, &parent, &bulkId); + r = es_addBuf(&pWrkrData->batch.data, META_STRT, sizeof(META_STRT)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchIndex, ustrlen(searchIndex)); - if(r == 0) r = es_addBuf(&pData->batch.data, META_TYPE, sizeof(META_TYPE)-1); - if(r == 0) r = es_addBuf(&pData->batch.data, (char*)searchType, + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_TYPE, sizeof(META_TYPE)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)searchType, ustrlen(searchType)); if(parent != NULL) { - if(r == 0) r = es_addBuf(&pData->batch.data, META_PARENT, sizeof(META_PARENT)-1); - if(r == 0) r = es_addBuf(&pData->batch.data, (char*)parent, ustrlen(parent)); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_PARENT, sizeof(META_PARENT)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)parent, ustrlen(parent)); } if(bulkId != NULL) { - if(r == 0) r = es_addBuf(&pData->batch.data, META_ID, sizeof(META_ID)-1); - if(r == 0) r = es_addBuf(&pData->batch.data, (char*)bulkId, ustrlen(bulkId)); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_ID, sizeof(META_ID)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)bulkId, ustrlen(bulkId)); } - if(r == 0) r = es_addBuf(&pData->batch.data, META_END, sizeof(META_END)-1); - if(r == 0) r = es_addBuf(&pData->batch.data, (char*)message, length); - if(r == 0) r = es_addBuf(&pData->batch.data, "\n", sizeof("\n")-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, META_END, sizeof(META_END)-1); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, (char*)message, length); + if(r == 0) r = es_addBuf(&pWrkrData->batch.data, "\n", sizeof("\n")-1); if(r != 0) { DBGPRINTF("omelasticsearch: growing batch failed with code %d\n", r); ABORT_FINALIZE(RS_RET_ERR); } - ++pData->batch.nmemb; + ++pWrkrData->batch.nmemb; iRet = RS_RET_DEFER_COMMIT; finalize_it: @@ -446,7 +476,7 @@ finalize_it: * needs to be closed, HUP must be sent. */ static inline rsRetVal -writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) +writeDataError(wrkrInstanceData_t *pWrkrData, instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) { char *rendered = NULL; cJSON *errRoot; @@ -454,6 +484,7 @@ writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) cJSON *replyRoot = *pReplyRoot; size_t toWrite; ssize_t wrRet; + sbool bMutLocked = 0; char errStr[1024]; DEFiRet; @@ -463,6 +494,9 @@ writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) FINALIZE; } + pthread_mutex_lock(&pData->mutErrFile); + bMutLocked = 1; + if(pData->fdErrFile == -1) { pData->fdErrFile = open((char*)pData->errorFile, O_WRONLY|O_CREAT|O_APPEND|O_LARGEFILE|O_CLOEXEC, @@ -474,7 +508,7 @@ writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) } } if((req=cJSON_CreateObject()) == NULL) ABORT_FINALIZE(RS_RET_ERR); - cJSON_AddItemToObject(req, "url", cJSON_CreateString((char*)pData->restURL)); + cJSON_AddItemToObject(req, "url", cJSON_CreateString((char*)pWrkrData->restURL)); cJSON_AddItemToObject(req, "postdata", cJSON_CreateString((char*)reqmsg)); if((errRoot=cJSON_CreateObject()) == NULL) ABORT_FINALIZE(RS_RET_ERR); @@ -495,13 +529,15 @@ writeDataError(instanceData *pData, cJSON **pReplyRoot, uchar *reqmsg) *pReplyRoot = NULL; /* tell caller not to delete once again! */ finalize_it: + if(bMutLocked) + pthread_mutex_unlock(&pData->mutErrFile); free(rendered); RETiRet; } static inline rsRetVal -checkResultBulkmode(instanceData *pData, cJSON *root) +checkResultBulkmode(wrkrInstanceData_t *pWrkrData, cJSON *root) { int i; int numitems; @@ -515,7 +551,7 @@ checkResultBulkmode(instanceData *pData, cJSON *root) if(items == NULL || items->type != cJSON_Array) { DBGPRINTF("omelasticsearch: error in elasticsearch reply: " "bulkmode insert does not return array, reply is: %s\n", - pData->reply); + pWrkrData->reply); ABORT_FINALIZE(RS_RET_DATAFAIL); } numitems = cJSON_GetArraySize(items); @@ -547,20 +583,20 @@ finalize_it: static inline rsRetVal -checkResult(instanceData *pData, uchar *reqmsg) +checkResult(wrkrInstanceData_t *pWrkrData, uchar *reqmsg) { cJSON *root; cJSON *ok; DEFiRet; - root = cJSON_Parse(pData->reply); + root = cJSON_Parse(pWrkrData->reply); if(root == NULL) { DBGPRINTF("omelasticsearch: could not parse JSON result \n"); ABORT_FINALIZE(RS_RET_ERR); } - if(pData->bulkmode) { - iRet = checkResultBulkmode(pData, root); + if(pWrkrData->pData->bulkmode) { + iRet = checkResultBulkmode(pWrkrData, root); } else { ok = cJSON_GetObjectItem(root, "ok"); if(ok == NULL || ok->type != cJSON_True) { @@ -572,7 +608,7 @@ checkResult(instanceData *pData, uchar *reqmsg) * these in any case. */ if(iRet == RS_RET_DATAFAIL) { - writeDataError(pData, &root, reqmsg); + writeDataError(pWrkrData, pWrkrData->pData, &root, reqmsg); iRet = RS_RET_OK; /* we have handled the problem! */ } @@ -587,19 +623,19 @@ finalize_it: static rsRetVal -curlPost(instanceData *pData, uchar *message, int msglen, uchar **tpls, int nmsgs) +curlPost(wrkrInstanceData_t *pWrkrData, uchar *message, int msglen, uchar **tpls, int nmsgs) { CURLcode code; - CURL *curl = pData->curlHandle; + CURL *curl = pWrkrData->curlHandle; DEFiRet; - pData->reply = NULL; - pData->replyLen = 0; + pWrkrData->reply = NULL; + pWrkrData->replyLen = 0; - if(pData->dynSrchIdx || pData->dynSrchType || pData->dynParent) - CHKiRet(setCurlURL(pData, tpls)); + if(pWrkrData->pData->dynSrchIdx || pWrkrData->pData->dynSrchType || pWrkrData->pData->dynParent) + CHKiRet(setCurlURL(pWrkrData, pWrkrData->pData, tpls)); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, pData); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, pWrkrData); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, (char *)message); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, msglen); code = curl_easy_perform(curl); @@ -618,27 +654,27 @@ curlPost(instanceData *pData, uchar *message, int msglen, uchar **tpls, int nmsg break; } - DBGPRINTF("omelasticsearch: pData replyLen = '%d'\n", pData->replyLen); - if (pData->replyLen > 0) { - pData->reply[pData->replyLen] = '\0'; /* Append 0 Byte if replyLen is above 0 - byte has been reserved in malloc */ + DBGPRINTF("omelasticsearch: pWrkrData replyLen = '%d'\n", pWrkrData->replyLen); + if(pWrkrData->replyLen > 0) { + pWrkrData->reply[pWrkrData->replyLen] = '\0'; /* Append 0 Byte if replyLen is above 0 - byte has been reserved in malloc */ } - DBGPRINTF("omelasticsearch: pData reply: '%s'\n", pData->reply); + DBGPRINTF("omelasticsearch: pWrkrData reply: '%s'\n", pWrkrData->reply); - CHKiRet(checkResult(pData, message)); + CHKiRet(checkResult(pWrkrData, message)); finalize_it: - free(pData->reply); + free(pWrkrData->reply); RETiRet; } BEGINbeginTransaction CODESTARTbeginTransaction -dbgprintf("omelasticsearch: beginTransaction\n"); - if(!pData->bulkmode) { +dbgprintf("omelasticsearch: beginTransaction, pWrkrData %p, pData %p\n", pWrkrData, pWrkrData->pData); + if(!pWrkrData->pData->bulkmode) { FINALIZE; } - es_emptyStr(pData->batch.data); - pData->batch.nmemb = 0; + es_emptyStr(pWrkrData->batch.data); + pWrkrData->batch.nmemb = 0; finalize_it: ENDbeginTransaction @@ -646,14 +682,14 @@ ENDbeginTransaction BEGINdoAction CODESTARTdoAction STATSCOUNTER_INC(indexSubmit, mutIndexSubmit); - if(pData->bulkmode) { - CHKiRet(buildBatch(pData, ppString[0], ppString)); + if(pWrkrData->pData->bulkmode) { + CHKiRet(buildBatch(pWrkrData, ppString[0], ppString)); } else { - CHKiRet(curlPost(pData, ppString[0], strlen((char*)ppString[0]), + CHKiRet(curlPost(pWrkrData, ppString[0], strlen((char*)ppString[0]), ppString, 1)); } finalize_it: -dbgprintf("omelasticsearch: result doAction: %d (bulkmode %d)\n", iRet, pData->bulkmode); +dbgprintf("omelasticsearch: result doAction: %d (bulkmode %d)\n", iRet, pWrkrData->pData->bulkmode); ENDdoAction @@ -662,13 +698,13 @@ BEGINendTransaction CODESTARTendTransaction dbgprintf("omelasticsearch: endTransaction init\n"); /* End Transaction only if batch data is not empty */ - if (pData->batch.data != NULL ) { - cstr = es_str2cstr(pData->batch.data, NULL); + if (pWrkrData->batch.data != NULL ) { + cstr = es_str2cstr(pWrkrData->batch.data, NULL); dbgprintf("omelasticsearch: endTransaction, batch: '%s'\n", cstr); - CHKiRet(curlPost(pData, (uchar*) cstr, strlen(cstr), NULL, pData->batch.nmemb)); + CHKiRet(curlPost(pWrkrData, (uchar*) cstr, strlen(cstr), NULL, pWrkrData->batch.nmemb)); } else - dbgprintf("omelasticsearch: endTransaction, pData->batch.data is NULL, nothing to send. \n"); + dbgprintf("omelasticsearch: endTransaction, pWrkrData->batch.data is NULL, nothing to send. \n"); finalize_it: free(cstr); dbgprintf("omelasticsearch: endTransaction done with %d\n", iRet); @@ -679,24 +715,24 @@ size_t curlResult(void *ptr, size_t size, size_t nmemb, void *userdata) { char *p = (char *)ptr; - instanceData *pData = (instanceData*) userdata; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t*) userdata; char *buf; size_t newlen; - newlen = pData->replyLen + size*nmemb; - if((buf = realloc(pData->reply, newlen + 1)) == NULL) { + newlen = pWrkrData->replyLen + size*nmemb; + if((buf = realloc(pWrkrData->reply, newlen + 1)) == NULL) { DBGPRINTF("omelasticsearch: realloc failed in curlResult\n"); return 0; /* abort due to failure */ } - memcpy(buf+pData->replyLen, p, size*nmemb); - pData->replyLen = newlen; - pData->reply = buf; + memcpy(buf+pWrkrData->replyLen, p, size*nmemb); + pWrkrData->replyLen = newlen; + pWrkrData->reply = buf; return size*nmemb; } static rsRetVal -curlSetup(instanceData *pData) +curlSetup(wrkrInstanceData_t *pWrkrData, instanceData *pData) { HEADER *header; CURL *handle; @@ -712,13 +748,13 @@ curlSetup(instanceData *pData) curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curlResult); curl_easy_setopt(handle, CURLOPT_POST, 1); - pData->curlHandle = handle; - pData->postHeader = header; + pWrkrData->curlHandle = handle; + pWrkrData->postHeader = header; if( pData->bulkmode || (pData->dynSrchIdx == 0 && pData->dynSrchType == 0 && pData->dynParent == 0)) { /* in this case, we know no tpls are involved in the request-->NULL OK! */ - setCurlURL(pData, NULL); + setCurlURL(pWrkrData, pData, NULL); } if(Debug) { @@ -838,16 +874,6 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } - if(pData->bulkmode) { - pData->batch.currTpl1 = NULL; - pData->batch.currTpl2 = NULL; - if((pData->batch.data = es_newStr(1024)) == NULL) { - DBGPRINTF("omelasticsearch: error creating batch string " - "turned off bulk mode\n"); - pData->bulkmode = 0; /* at least it works */ - } - } - iNumTpls = 1; if(pData->dynSrchIdx) ++iNumTpls; if(pData->dynSrchType) ++iNumTpls; @@ -939,9 +965,6 @@ CODESTARTnewActInst pData->searchIndex = (uchar*) strdup("system"); if(pData->searchType == NULL) pData->searchType = (uchar*) strdup("events"); - - CHKiRet(curlSetup(pData)); - CODE_STD_FINALIZERnewActInst cnfparamvalsDestruct(pvals, &actpblk); ENDnewActInst @@ -979,6 +1002,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_doHUP diff --git a/plugins/omhiredis/omhiredis.c b/plugins/omhiredis/omhiredis.c index 757d5eb2..28a1b9cd 100644 --- a/plugins/omhiredis/omhiredis.c +++ b/plugins/omhiredis/omhiredis.c @@ -52,14 +52,17 @@ DEFobjCurrIf(errmsg) * this will be accessable * via pData */ typedef struct _instanceData { - redisContext *conn; /* redis connection */ uchar *server; /* redis server address */ int port; /* redis port */ uchar *tplName; /* template name */ - redisReply **replies; /* array to hold replies from redis */ - int count; /* count of command sent for current batch */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; + redisContext *conn; /* redis connection */ + redisReply **replies; /* array to hold replies from redis */ + int count; /* count of command sent for current batch */ +} wrkrInstanceData_t; static struct cnfparamdescr actpdescr[] = { { "server", eCmdHdlrGetWord, 0 }, @@ -76,6 +79,11 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->conn = NULL; /* Connect later */ +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -83,11 +91,11 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature /* called when closing */ -static void closeHiredis(instanceData *pData) +static void closeHiredis(wrkrInstanceData_t *pWrkrData) { - if(pData->conn != NULL) { - redisFree(pData->conn); - pData->conn = NULL; + if(pWrkrData->conn != NULL) { + redisFree(pWrkrData->conn); + pWrkrData->conn = NULL; } } @@ -95,10 +103,15 @@ static void closeHiredis(instanceData *pData) * TODO: free **replies */ BEGINfreeInstance CODESTARTfreeInstance - closeHiredis(pData); - free(pData->server); + if (pData->server != NULL) { + free(pData->server); + } ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + closeHiredis(pWrkrData); +ENDfreeWrkrInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -106,17 +119,20 @@ CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo /* establish our connection to redis */ -static rsRetVal initHiredis(instanceData *pData, int bSilent) +static rsRetVal initHiredis(wrkrInstanceData_t *pWrkrData, int bSilent) { char *server; DEFiRet; - server = (pData->server == NULL) ? "127.0.0.1" : (char*) pData->server; - DBGPRINTF("omhiredis: trying connect to '%s' at port %d\n", server, pData->port); - + server = (pWrkrData->pData->server == NULL) ? "127.0.0.1" : + (char*) pWrkrData->pData->server; + DBGPRINTF("omhiredis: trying connect to '%s' at port %d\n", server, + pWrkrData->pData->port); + struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */ - pData->conn = redisConnectWithTimeout(server, pData->port, timeout); - if (pData->conn->err) { + pWrkrData->conn = redisConnectWithTimeout(server, pWrkrData->pData->port, + timeout); + if (pWrkrData->conn->err) { if(!bSilent) errmsg.LogError(0, RS_RET_SUSPENDED, "can not initialize redis handle"); @@ -126,29 +142,29 @@ finalize_it: RETiRet; } -rsRetVal writeHiredis(uchar *message, instanceData *pData) +rsRetVal writeHiredis(uchar *message, wrkrInstanceData_t *pWrkrData) { DEFiRet; - /* if we do not have a redis connection, call - * initHiredis and try to establish one */ - if(pData->conn == NULL) - CHKiRet(initHiredis(pData, 0)); + /* if we do not have a redis connection, call + * initHiredis and try to establish one */ + if(pWrkrData->conn == NULL) + CHKiRet(initHiredis(pWrkrData, 0)); - /* try to append the command to the pipeline. - * REDIS_ERR reply indicates something bad - * happened, in which case abort. otherwise - * increase our current pipeline count - * by 1 and continue. */ + /* try to append the command to the pipeline. + * REDIS_ERR reply indicates something bad + * happened, in which case abort. otherwise + * increase our current pipeline count + * by 1 and continue. */ int rc; - rc = redisAppendCommand(pData->conn, (char*)message); + rc = redisAppendCommand(pWrkrData->conn, (char*)message); if (rc == REDIS_ERR) { - errmsg.LogError(0, NO_ERRCODE, "omhiredis: %s", pData->conn->errstr); - dbgprintf("omhiredis: %s\n", pData->conn->errstr); + errmsg.LogError(0, NO_ERRCODE, "omhiredis: %s", pWrkrData->conn->errstr); + dbgprintf("omhiredis: %s\n", pWrkrData->conn->errstr); ABORT_FINALIZE(RS_RET_ERR); } else { - pData->count++; - } + pWrkrData->count++; + } finalize_it: RETiRet; @@ -158,17 +174,18 @@ finalize_it: * try to restablish our connection to redis */ BEGINtryResume CODESTARTtryResume - if(pData->conn == NULL) - iRet = initHiredis(pData, 0); + if(pWrkrData->conn == NULL) + iRet = initHiredis(pWrkrData, 0); ENDtryResume -/* begin a transaction. for now does nothing. +/* begin a transaction. * if I decide to use MULTI ... EXEC in the - * fture, this block should send the + * future, this block should send the * MULTI command to redis. */ BEGINbeginTransaction CODESTARTbeginTransaction - dbgprintf("omhiredis: beginTransaction called\n"); + dbgprintf("omhiredis: beginTransaction called\n"); + pWrkrData->count = 0; ENDbeginTransaction /* call writeHiredis for this log line, @@ -176,8 +193,8 @@ ENDbeginTransaction * current pipeline */ BEGINdoAction CODESTARTdoAction - CHKiRet(writeHiredis(ppString[0], pData)); - iRet = RS_RET_DEFER_COMMIT; + CHKiRet(writeHiredis(ppString[0], pWrkrData)); + iRet = RS_RET_DEFER_COMMIT; finalize_it: ENDdoAction @@ -189,16 +206,15 @@ ENDdoAction * which should be fixed */ BEGINendTransaction CODESTARTendTransaction - dbgprintf("omhiredis: endTransaction called\n"); - int i; - pData->replies = malloc ( sizeof ( redisReply* ) * pData->count ); - for ( i = 0; i < pData->count; i++ ) { - redisGetReply ( pData->conn, (void *)&pData->replies[i] ); - /* TODO: add error checking here! */ - freeReplyObject ( pData->replies[i] ); - } - free ( pData->replies ); - pData->count = 0; + dbgprintf("omhiredis: endTransaction called\n"); + int i; + pWrkrData->replies = malloc ( sizeof ( redisReply* ) * pWrkrData->count ); + for ( i = 0; i < pWrkrData->count; i++ ) { + redisGetReply ( pWrkrData->conn, (void *)&pWrkrData->replies[i] ); + /* TODO: add error checking here! */ + freeReplyObject ( pWrkrData->replies[i] ); + } + free ( pWrkrData->replies ); ENDendTransaction /* set defaults. note server is set to NULL @@ -211,7 +227,6 @@ setInstParamDefaults(instanceData *pData) pData->server = NULL; pData->port = 6379; pData->tplName = NULL; - pData->count = 0; } /* here is where the work to set up a new instance @@ -281,6 +296,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_TXIF_OMOD_QUERIES /* supports transaction interface */ ENDqueryEtryPt @@ -292,9 +308,9 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); - if (!bCoreSupportsBatching) { - errmsg.LogError(0, NO_ERRCODE, "omhiredis: rsyslog core does not support batching - abort"); - ABORT_FINALIZE(RS_RET_ERR); - } + if (!bCoreSupportsBatching) { + errmsg.LogError(0, NO_ERRCODE, "omhiredis: rsyslog core does not support batching - abort"); + ABORT_FINALIZE(RS_RET_ERR); + } DBGPRINTF("omhiredis: module compiled with rsyslog version %s.\n", VERSION); ENDmodInit diff --git a/plugins/omjournal/omjournal.c b/plugins/omjournal/omjournal.c index 160c369d..82fd7bfb 100644 --- a/plugins/omjournal/omjournal.c +++ b/plugins/omjournal/omjournal.c @@ -56,6 +56,10 @@ DEF_OMOD_STATIC_DATA typedef struct _instanceData { } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + struct modConfData_s { rsconf_t *pConf; /* our overall config object */ }; @@ -91,6 +95,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature @@ -101,6 +110,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINnewActInst CODESTARTnewActInst /* Note: we currently do not have any parameters, so we do not need @@ -172,6 +186,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES ENDqueryEtryPt diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 3beba4f0..c203b4aa 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -50,6 +50,9 @@ #include "errmsg.h" #include "conf.h" +#undef HAVE_DBI_TXSUPP +#warning transaction support disabled in v8 -- TODO: reenable + MODULE_TYPE_OUTPUT MODULE_TYPE_NOKEEP MODULE_CNFNAME("omlibdbi") @@ -73,6 +76,10 @@ typedef struct _instanceData { int txSupport; /* transaction support */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { uchar *dbiDrvrDir; /* global: where do the dbi drivers reside? */ uchar *drvrName; /* driver to use */ @@ -94,6 +101,8 @@ static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current l static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ static int bLegacyCnfModGlobalsPermitted;/* are legacy module-global config parameters permitted? */ +static pthread_mutex_t mutDoAct = PTHREAD_MUTEX_INITIALIZER; + /* tables for interfacing with the v6 config system */ /* module-global parameters */ @@ -157,6 +166,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -187,6 +200,9 @@ CODESTARTfreeInstance free(pData->dbName); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -326,16 +342,16 @@ finalize_it: BEGINtryResume CODESTARTtryResume - if(pData->conn == NULL) { - iRet = initConn(pData, 1); + if(pWrkrData->pData->conn == NULL) { + iRet = initConn(pWrkrData->pData, 1); } ENDtryResume /* transaction support 2013-03 */ BEGINbeginTransaction CODESTARTbeginTransaction - if(pData->conn == NULL) { - CHKiRet(initConn(pData, 0)); + if(pWrkrData->pData->conn == NULL) { + CHKiRet(initConn(pWrkrData->pData, 0)); } # if HAVE_DBI_TXSUPP if (pData->txSupport == 1) { @@ -355,13 +371,15 @@ ENDbeginTransaction BEGINdoAction CODESTARTdoAction - CHKiRet(writeDB(ppString[0], pData)); + pthread_mutex_lock(&mutDoAct); + CHKiRet(writeDB(ppString[0], pWrkrData->pData)); # if HAVE_DBI_TXSUPP if (pData->txSupport == 1) { iRet = RS_RET_DEFER_COMMIT; } # endif finalize_it: + pthread_mutex_unlock(&mutDoAct); ENDdoAction /* transaction support 2013-03 */ @@ -552,6 +570,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 0a781e10..910b371a 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -13,7 +13,7 @@ * * File begun on 2008-04-04 by RGerhards * - * Copyright 2008-2012 Adiscon GmbH. + * Copyright 2008-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -82,13 +82,21 @@ typedef struct _instanceData { uchar *pszSrvPort; uchar *pszFrom; toRcpt_t *lstRcpt; + } smtp; + } md; /* mode-specific data */ +} instanceData; + +typedef struct wrkrInstanceData { + instanceData *pData; + union { + struct { char RcvBuf[1024]; /* buffer for receiving server responses */ size_t lenRcvBuf; size_t iRcvBuf; /* current index into the rcvBuf (buf empty if iRcvBuf == lenRcvBuf) */ int sock; /* socket to this server (most important when we do multiple msgs per mail) */ } smtp; } md; /* mode-specific data */ -} instanceData; +} wrkrInstanceData_t; typedef struct configSettings_s { toRcpt_t *lstRcpt; @@ -112,7 +120,7 @@ ENDinitConfVars /* forward definitions (as few as possible) */ static rsRetVal Send(int sock, char *msg, size_t len); -static rsRetVal readResponse(instanceData *pData, int *piState, int iExpected); +static rsRetVal readResponse(wrkrInstanceData_t *pWrkrData, int *piState, int iExpected); /* helpers for handling the recipient lists */ @@ -163,24 +171,22 @@ finalize_it: * iStatusToCheck < 0 means no checking should happen */ static rsRetVal -WriteRcpts(instanceData *pData, uchar *pszOp, size_t lenOp, int iStatusToCheck) +WriteRcpts(wrkrInstanceData_t *pWrkrData, uchar *pszOp, size_t lenOp, int iStatusToCheck) { toRcpt_t *pRcpt; int iState; DEFiRet; - assert(pData != NULL); - assert(pszOp != NULL); assert(lenOp != 0); - for(pRcpt = pData->md.smtp.lstRcpt ; pRcpt != NULL ; pRcpt = pRcpt->pNext) { + for(pRcpt = pWrkrData->pData->md.smtp.lstRcpt ; pRcpt != NULL ; pRcpt = pRcpt->pNext) { dbgprintf("Sending '%s: <%s>'\n", pszOp, pRcpt->pszTo); - CHKiRet(Send(pData->md.smtp.sock, (char*)pszOp, lenOp)); - CHKiRet(Send(pData->md.smtp.sock, ":<", sizeof(":<") - 1)); - CHKiRet(Send(pData->md.smtp.sock, (char*)pRcpt->pszTo, strlen((char*)pRcpt->pszTo))); - CHKiRet(Send(pData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)pszOp, lenOp)); + CHKiRet(Send(pWrkrData->md.smtp.sock, ":<", sizeof(":<") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)pRcpt->pszTo, strlen((char*)pRcpt->pszTo))); + CHKiRet(Send(pWrkrData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); if(iStatusToCheck >= 0) - CHKiRet(readResponse(pData, &iState, iStatusToCheck)); + CHKiRet(readResponse(pWrkrData, &iState, iStatusToCheck)); } finalize_it: @@ -193,6 +199,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -203,17 +214,19 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance if(pData->iMode == 0) { - if(pData->md.smtp.pszSrv != NULL) - free(pData->md.smtp.pszSrv); - if(pData->md.smtp.pszSrvPort != NULL) - free(pData->md.smtp.pszSrvPort); - if(pData->md.smtp.pszFrom != NULL) - free(pData->md.smtp.pszFrom); + free(pData->md.smtp.pszSrv); + free(pData->md.smtp.pszSrvPort); + free(pData->md.smtp.pszFrom); lstRcptDestruct(pData->md.smtp.lstRcpt); } ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo printf("mail"); /* TODO: extend! */ @@ -229,16 +242,16 @@ ENDdbgPrintInstInfo * rgerhards, 2008-04-04 */ static rsRetVal -getRcvChar(instanceData *pData, char *pC) +getRcvChar(wrkrInstanceData_t *pWrkrData, char *pC) { DEFiRet; ssize_t lenBuf; - assert(pData != NULL); - if(pData->md.smtp.iRcvBuf == pData->md.smtp.lenRcvBuf) { /* buffer empty? */ + if(pWrkrData->md.smtp.iRcvBuf == pWrkrData->md.smtp.lenRcvBuf) { /* buffer empty? */ /* yes, we need to read the next server response */ do { - lenBuf = recv(pData->md.smtp.sock, pData->md.smtp.RcvBuf, sizeof(pData->md.smtp.RcvBuf), 0); + lenBuf = recv(pWrkrData->md.smtp.sock, pWrkrData->md.smtp.RcvBuf, + sizeof(pWrkrData->md.smtp.RcvBuf), 0); if(lenBuf == 0) { ABORT_FINALIZE(RS_RET_NO_MORE_DATA); } else if(lenBuf < 0) { @@ -247,15 +260,15 @@ getRcvChar(instanceData *pData, char *pC) } } else { /* good read */ - pData->md.smtp.iRcvBuf = 0; - pData->md.smtp.lenRcvBuf = lenBuf; + pWrkrData->md.smtp.iRcvBuf = 0; + pWrkrData->md.smtp.lenRcvBuf = lenBuf; } } while(lenBuf < 1); } /* when we reach this point, we have a non-empty buffer */ - *pC = pData->md.smtp.RcvBuf[pData->md.smtp.iRcvBuf++]; + *pC = pWrkrData->md.smtp.RcvBuf[pWrkrData->md.smtp.iRcvBuf++]; finalize_it: RETiRet; @@ -266,14 +279,14 @@ finalize_it: * rgerhards, 2008-04-08 */ static rsRetVal -serverDisconnect(instanceData *pData) +serverDisconnect(wrkrInstanceData_t *pWrkrData) { DEFiRet; - assert(pData != NULL); + assert(pWrkrData != NULL); - if(pData->md.smtp.sock != -1) { - close(pData->md.smtp.sock); - pData->md.smtp.sock = -1; + if(pWrkrData->md.smtp.sock != -1) { + close(pWrkrData->md.smtp.sock); + pWrkrData->md.smtp.sock = -1; } RETiRet; @@ -284,16 +297,17 @@ serverDisconnect(instanceData *pData) * rgerhards, 2008-04-04 */ static rsRetVal -serverConnect(instanceData *pData) +serverConnect(wrkrInstanceData_t *pWrkrData) { struct addrinfo *res = NULL; struct addrinfo hints; char *smtpPort; char *smtpSrv; char errStr[1024]; - + instanceData *pData; DEFiRet; - assert(pData != NULL); + + pData = pWrkrData->pData; if(pData->md.smtp.pszSrv == NULL) smtpSrv = "127.0.0.1"; @@ -313,12 +327,12 @@ serverConnect(instanceData *pData) ABORT_FINALIZE(RS_RET_IO_ERROR); } - if((pData->md.smtp.sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) { + if((pWrkrData->md.smtp.sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) { dbgprintf("couldn't create send socket, reason %s", rs_strerror_r(errno, errStr, sizeof(errStr))); ABORT_FINALIZE(RS_RET_IO_ERROR); } - if(connect(pData->md.smtp.sock, res->ai_addr, res->ai_addrlen) != 0) { + if(connect(pWrkrData->md.smtp.sock, res->ai_addr, res->ai_addrlen) != 0) { dbgprintf("create tcp connection failed, reason %s", rs_strerror_r(errno, errStr, sizeof(errStr))); ABORT_FINALIZE(RS_RET_IO_ERROR); } @@ -328,9 +342,9 @@ finalize_it: freeaddrinfo(res); if(iRet != RS_RET_OK) { - if(pData->md.smtp.sock != -1) { - close(pData->md.smtp.sock); - pData->md.smtp.sock = -1; + if(pWrkrData->md.smtp.sock != -1) { + close(pWrkrData->md.smtp.sock); + pWrkrData->md.smtp.sock = -1; } } @@ -374,7 +388,7 @@ finalize_it: * The body is special in that we must escape a leading dot inside a line */ static rsRetVal -bodySend(instanceData *pData, char *msg, size_t len) +bodySend(wrkrInstanceData_t *pWrkrData, char *msg, size_t len) { DEFiRet; char szBuf[2048]; @@ -383,12 +397,12 @@ bodySend(instanceData *pData, char *msg, size_t len) int bHadCR = 0; int bInStartOfLine = 1; - assert(pData != NULL); + assert(pWrkrData != NULL); assert(msg != NULL); for(iSrc = 0 ; iSrc < len ; ++iSrc) { if(iBuf >= sizeof(szBuf) - 1) { /* one is reserved for our extra dot */ - CHKiRet(Send(pData->md.smtp.sock, szBuf, iBuf)); + CHKiRet(Send(pWrkrData->md.smtp.sock, szBuf, iBuf)); iBuf = 0; } szBuf[iBuf++] = msg[iSrc]; @@ -413,7 +427,7 @@ bodySend(instanceData *pData, char *msg, size_t len) } if(iBuf > 0) { /* incomplete buffer to send (the *usual* case)? */ - CHKiRet(Send(pData->md.smtp.sock, szBuf, iBuf)); + CHKiRet(Send(pWrkrData->md.smtp.sock, szBuf, iBuf)); } finalize_it: @@ -424,17 +438,17 @@ finalize_it: /* read response line from server */ static rsRetVal -readResponseLn(instanceData *pData, char *pLn, size_t lenLn) +readResponseLn(wrkrInstanceData_t *pWrkrData, char *pLn, size_t lenLn) { DEFiRet; size_t i = 0; char c; - assert(pData != NULL); + assert(pWrkrData != NULL); assert(pLn != NULL); do { - CHKiRet(getRcvChar(pData, &c)); + CHKiRet(getRcvChar(pWrkrData, &c)); if(c == '\n') break; if(i < (lenLn - 1)) /* if line is too long, we simply discard the rest */ @@ -453,18 +467,18 @@ finalize_it: * rgerhards, 2008-04-07 */ static rsRetVal -readResponse(instanceData *pData, int *piState, int iExpected) +readResponse(wrkrInstanceData_t *pWrkrData, int *piState, int iExpected) { DEFiRet; int bCont; char buf[128]; - assert(pData != NULL); + assert(pWrkrData != NULL); assert(piState != NULL); bCont = 1; do { - CHKiRet(readResponseLn(pData, buf, sizeof(buf))); + CHKiRet(readResponseLn(pWrkrData, buf, sizeof(buf))); /* note: the code below is not 100% clean as we may have received less than 4 characters. * However, as we have a fixed size this will not create a vulnerability. An error will * also most likely be generated, so it is quite acceptable IMHO -- rgerhards, 2008-04-08 @@ -506,64 +520,65 @@ mkSMTPTimestamp(uchar *pszBuf, size_t lenBuf) * rgerhards, 2008-04-04 */ static rsRetVal -sendSMTP(instanceData *pData, uchar *body, uchar *subject) +sendSMTP(wrkrInstanceData_t *pWrkrData, uchar *body, uchar *subject) { DEFiRet; int iState; /* SMTP state */ + instanceData *pData; uchar szDateBuf[64]; - assert(pData != NULL); + pData = pWrkrData->pData; - CHKiRet(serverConnect(pData)); - CHKiRet(readResponse(pData, &iState, 220)); + CHKiRet(serverConnect(pWrkrData)); + CHKiRet(readResponse(pWrkrData, &iState, 220)); - CHKiRet(Send(pData->md.smtp.sock, "HELO ", 5)); - CHKiRet(Send(pData->md.smtp.sock, (char*)glbl.GetLocalHostName(), strlen((char*)glbl.GetLocalHostName()))); - CHKiRet(Send(pData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); - CHKiRet(readResponse(pData, &iState, 250)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "HELO ", 5)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)glbl.GetLocalHostName(), strlen((char*)glbl.GetLocalHostName()))); + CHKiRet(Send(pWrkrData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); + CHKiRet(readResponse(pWrkrData, &iState, 250)); - CHKiRet(Send(pData->md.smtp.sock, "MAIL FROM:<", sizeof("MAIL FROM:<") - 1)); - CHKiRet(Send(pData->md.smtp.sock, (char*)pData->md.smtp.pszFrom, strlen((char*)pData->md.smtp.pszFrom))); - CHKiRet(Send(pData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); - CHKiRet(readResponse(pData, &iState, 250)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "MAIL FROM:<", sizeof("MAIL FROM:<") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)pData->md.smtp.pszFrom, strlen((char*)pData->md.smtp.pszFrom))); + CHKiRet(Send(pWrkrData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); + CHKiRet(readResponse(pWrkrData, &iState, 250)); - CHKiRet(WriteRcpts(pData, (uchar*)"RCPT TO", sizeof("RCPT TO") - 1, 250)); + CHKiRet(WriteRcpts(pWrkrData, (uchar*)"RCPT TO", sizeof("RCPT TO") - 1, 250)); - CHKiRet(Send(pData->md.smtp.sock, "DATA\r\n", sizeof("DATA\r\n") - 1)); - CHKiRet(readResponse(pData, &iState, 354)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "DATA\r\n", sizeof("DATA\r\n") - 1)); + CHKiRet(readResponse(pWrkrData, &iState, 354)); /* now come the data part */ /* header */ mkSMTPTimestamp(szDateBuf, sizeof(szDateBuf)); - CHKiRet(Send(pData->md.smtp.sock, (char*)szDateBuf, strlen((char*)szDateBuf))); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)szDateBuf, strlen((char*)szDateBuf))); - CHKiRet(Send(pData->md.smtp.sock, "From: <", sizeof("From: <") - 1)); - CHKiRet(Send(pData->md.smtp.sock, (char*)pData->md.smtp.pszFrom, strlen((char*)pData->md.smtp.pszFrom))); - CHKiRet(Send(pData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "From: <", sizeof("From: <") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)pData->md.smtp.pszFrom, strlen((char*)pData->md.smtp.pszFrom))); + CHKiRet(Send(pWrkrData->md.smtp.sock, ">\r\n", sizeof(">\r\n") - 1)); - CHKiRet(WriteRcpts(pData, (uchar*)"To", sizeof("To") - 1, -1)); + CHKiRet(WriteRcpts(pWrkrData, (uchar*)"To", sizeof("To") - 1, -1)); - CHKiRet(Send(pData->md.smtp.sock, "Subject: ", sizeof("Subject: ") - 1)); - CHKiRet(Send(pData->md.smtp.sock, (char*)subject, strlen((char*)subject))); - CHKiRet(Send(pData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "Subject: ", sizeof("Subject: ") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, (char*)subject, strlen((char*)subject))); + CHKiRet(Send(pWrkrData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); - CHKiRet(Send(pData->md.smtp.sock, "X-Mailer: rsyslog-immail\r\n", sizeof("x-mailer: rsyslog-immail\r\n") - 1)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "X-Mailer: rsyslog-immail\r\n", sizeof("x-mailer: rsyslog-immail\r\n") - 1)); - CHKiRet(Send(pData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); /* indicate end of header */ + CHKiRet(Send(pWrkrData->md.smtp.sock, "\r\n", sizeof("\r\n") - 1)); /* indicate end of header */ /* body */ if(pData->bEnableBody) - CHKiRet(bodySend(pData, (char*)body, strlen((char*) body))); + CHKiRet(bodySend(pWrkrData, (char*)body, strlen((char*) body))); /* end of data, back to envelope transaction */ - CHKiRet(Send(pData->md.smtp.sock, "\r\n.\r\n", sizeof("\r\n.\r\n") - 1)); - CHKiRet(readResponse(pData, &iState, 250)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "\r\n.\r\n", sizeof("\r\n.\r\n") - 1)); + CHKiRet(readResponse(pWrkrData, &iState, 250)); - CHKiRet(Send(pData->md.smtp.sock, "QUIT\r\n", sizeof("QUIT\r\n") - 1)); - CHKiRet(readResponse(pData, &iState, 221)); + CHKiRet(Send(pWrkrData->md.smtp.sock, "QUIT\r\n", sizeof("QUIT\r\n") - 1)); + CHKiRet(readResponse(pWrkrData, &iState, 221)); /* we are finished, a new connection is created for each request, so let's close it now */ - CHKiRet(serverDisconnect(pData)); + CHKiRet(serverDisconnect(pWrkrData)); finalize_it: RETiRet; @@ -583,8 +598,8 @@ finalize_it: */ BEGINtryResume CODESTARTtryResume - CHKiRet(serverConnect(pData)); - CHKiRet(serverDisconnect(pData)); /* if we fail, we will never reach this line */ + CHKiRet(serverConnect(pWrkrData)); + CHKiRet(serverDisconnect(pWrkrData)); /* if we fail, we will never reach this line */ finalize_it: if(iRet == RS_RET_IO_ERROR) iRet = RS_RET_SUSPENDED; @@ -593,17 +608,14 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" Mail\n"); + DBGPRINTF(" Mail\n"); - /* forward */ - if(pData->bHaveSubject) - iRet = sendSMTP(pData, ppString[0], ppString[1]); - else - iRet = sendSMTP(pData, ppString[0], (uchar*)"message from rsyslog"); + iRet = sendSMTP(pWrkrData, ppString[0], + (pWrkrData->pData->bHaveSubject) ? + ppString[1] : (uchar*)"message from rsyslog"); if(iRet != RS_RET_OK) { - /* error! */ - dbgprintf("error sending mail, suspending\n"); + DBGPRINTF("error sending mail, suspending\n"); iRet = RS_RET_SUSPENDED; } ENDdoAction @@ -689,6 +701,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES ENDqueryEtryPt diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index af1f5a37..09f19768 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -4,7 +4,7 @@ * mongodb C interface is crap. Obtain the library here: * https://github.com/algernon/libmongo-client * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -71,6 +71,10 @@ typedef struct _instanceData { int bErrMsgPermitted; /* only one errmsg permitted per connection */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + /* tables for interfacing with the v6 config system */ /* action (instance) parameters */ @@ -89,10 +93,16 @@ static struct cnfparamblk actpblk = actpdescr }; +static pthread_mutex_t mutDoAct = PTHREAD_MUTEX_INITIALIZER; + BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* use this to specify if select features are supported by this @@ -126,6 +136,10 @@ CODESTARTfreeInstance free(pData->tplName); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -422,14 +436,17 @@ error: BEGINtryResume CODESTARTtryResume - if(pData->conn == NULL) { - iRet = initMongoDB(pData, 1); + if(pWrkrData->pData->conn == NULL) { + iRet = initMongoDB(pWrkrData->pData, 1); } ENDtryResume BEGINdoAction bson *doc = NULL; + instanceData *pData; CODESTARTdoAction + pthread_mutex_lock(&mutDoAct); + pData = pWrkrData->pData; /* see if we are ready to proceed */ if(pData->conn == NULL) { CHKiRet(initMongoDB(pData, 0)); @@ -454,6 +471,7 @@ CODESTARTdoAction } finalize_it: + pthread_mutex_unlock(&mutDoAct); if(doc != NULL) bson_free(doc); ENDdoAction @@ -560,6 +578,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index 49079ab1..c004d1c6 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -6,7 +6,7 @@ * * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,18 +55,22 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) typedef struct _instanceData { - MYSQL *f_hmysql; /* handle to MySQL */ char dbsrv[MAXHOSTNAMELEN+1]; /* IP or hostname of DB server*/ unsigned int dbsrvPort; /* port of MySQL server */ char dbname[_DB_MAXDBLEN+1]; /* DB name */ char dbuid[_DB_MAXUNAMELEN+1]; /* DB user */ char dbpwd[_DB_MAXPWDLEN+1]; /* DB user's password */ - unsigned uLastMySQLErrno; /* last errno returned by MySQL or 0 if all is well */ - uchar * f_configfile; /* MySQL Client Configuration File */ - uchar * f_configsection; /* MySQL Client Configuration Section */ - uchar *tplName; /* format template to use */ + uchar *configfile; /* MySQL Client Configuration File */ + uchar *configsection; /* MySQL Client Configuration Section */ + uchar *tplName; /* format template to use */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; + MYSQL *hmysql; /* handle to MySQL */ + unsigned uLastMySQLErrno; /* last errno returned by MySQL or 0 if all is well */ +} wrkrInstanceData_t; + typedef struct configSettings_s { int iSrvPort; /* database server port */ uchar *pszMySQLConfigFile; /* MySQL Client Configuration File */ @@ -104,6 +108,12 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->hmysql = NULL; +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -115,25 +125,28 @@ ENDisCompatibleWithFeature * MySQL connection. * Initially added 2004-10-28 */ -static void closeMySQL(instanceData *pData) +static void closeMySQL(wrkrInstanceData_t *pWrkrData) { - ASSERT(pData != NULL); - - if(pData->f_hmysql != NULL) { /* just to be on the safe side... */ - mysql_close(pData->f_hmysql); - pData->f_hmysql = NULL; + if(pWrkrData->hmysql != NULL) { /* just to be on the safe side... */ + mysql_close(pWrkrData->hmysql); + pWrkrData->hmysql = NULL; } } BEGINfreeInstance CODESTARTfreeInstance - free(pData->f_configfile); - free(pData->f_configsection); + free(pData->configfile); + free(pData->configsection); free(pData->tplName); - closeMySQL(pData); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + closeMySQL(pWrkrData); +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo /* nothing special here */ @@ -144,25 +157,23 @@ ENDdbgPrintInstInfo * We check if we have a valid MySQL handle. If not, we simply * report an error, but can not be specific. RGerhards, 2007-01-30 */ -static void reportDBError(instanceData *pData, int bSilent) +static void reportDBError(wrkrInstanceData_t *pWrkrData, int bSilent) { char errMsg[512]; unsigned uMySQLErrno; - ASSERT(pData != NULL); - /* output log message */ errno = 0; - if(pData->f_hmysql == NULL) { + if(pWrkrData->hmysql == NULL) { errmsg.LogError(0, NO_ERRCODE, "unknown DB error occured - could not obtain MySQL handle"); } else { /* we can ask mysql for the error description... */ - uMySQLErrno = mysql_errno(pData->f_hmysql); + uMySQLErrno = mysql_errno(pWrkrData->hmysql); snprintf(errMsg, sizeof(errMsg)/sizeof(char), "db error (%d): %s\n", uMySQLErrno, - mysql_error(pData->f_hmysql)); - if(bSilent || uMySQLErrno == pData->uLastMySQLErrno) + mysql_error(pWrkrData->hmysql)); + if(bSilent || uMySQLErrno == pWrkrData->uLastMySQLErrno) dbgprintf("mysql, DBError(silent): %s\n", errMsg); else { - pData->uLastMySQLErrno = uMySQLErrno; + pWrkrData->uLastMySQLErrno = uMySQLErrno; errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); } } @@ -175,25 +186,26 @@ static void reportDBError(instanceData *pData, int bSilent) * MySQL connection. * Initially added 2004-10-28 mmeckelein */ -static rsRetVal initMySQL(instanceData *pData, int bSilent) +static rsRetVal initMySQL(wrkrInstanceData_t *pWrkrData, int bSilent) { + instanceData *pData; DEFiRet; - ASSERT(pData != NULL); - ASSERT(pData->f_hmysql == NULL); - pData->f_hmysql = mysql_init(NULL); - if(pData->f_hmysql == NULL) { + ASSERT(pWrkrData->hmysql == NULL); + pData = pWrkrData->pData; + pWrkrData->hmysql = mysql_init(NULL); + if(pWrkrData->hmysql == NULL) { errmsg.LogError(0, RS_RET_SUSPENDED, "can not initialize MySQL handle"); iRet = RS_RET_SUSPENDED; } else { /* we could get the handle, now on with work... */ - mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_GROUP,((pData->f_configsection!=NULL)?(char*)pData->f_configsection:"client")); - if(pData->f_configfile!=NULL){ + mysql_options(pWrkrData->hmysql,MYSQL_READ_DEFAULT_GROUP,((pData->configsection!=NULL)?(char*)pData->configsection:"client")); + if(pData->configfile!=NULL){ FILE * fp; - fp=fopen((char*)pData->f_configfile,"r"); + fp=fopen((char*)pData->configfile,"r"); int err=errno; if(fp==NULL){ char msg[512]; - snprintf(msg,sizeof(msg)/sizeof(char),"Could not open '%s' for reading",pData->f_configfile); + snprintf(msg,sizeof(msg)/sizeof(char),"Could not open '%s' for reading",pData->configfile); if(bSilent) { char errStr[512]; rs_strerror_r(err, errStr, sizeof(errStr)); @@ -202,17 +214,17 @@ static rsRetVal initMySQL(instanceData *pData, int bSilent) errmsg.LogError(err,NO_ERRCODE,"mysql configuration error: %s\n",msg); } else { fclose(fp); - mysql_options(pData->f_hmysql,MYSQL_READ_DEFAULT_FILE,pData->f_configfile); + mysql_options(pWrkrData->hmysql,MYSQL_READ_DEFAULT_FILE,pData->configfile); } } /* Connect to database */ - if(mysql_real_connect(pData->f_hmysql, pData->dbsrv, pData->dbuid, + if(mysql_real_connect(pWrkrData->hmysql, pData->dbsrv, pData->dbuid, pData->dbpwd, pData->dbname, pData->dbsrvPort, NULL, 0) == NULL) { - reportDBError(pData, bSilent); - closeMySQL(pData); /* ignore any error we may get */ + reportDBError(pWrkrData, bSilent); + closeMySQL(pWrkrData); /* ignore any error we may get */ ABORT_FINALIZE(RS_RET_SUSPENDED); } - mysql_autocommit(pData->f_hmysql, 0); + mysql_autocommit(pWrkrData->hmysql, 0); } finalize_it: @@ -224,35 +236,32 @@ finalize_it: * to an established MySQL session. * Initially added 2004-10-28 mmeckelein */ -rsRetVal writeMySQL(uchar *psz, instanceData *pData) +rsRetVal writeMySQL(wrkrInstanceData_t *pWrkrData, uchar *psz) { DEFiRet; - ASSERT(psz != NULL); - ASSERT(pData != NULL); - /* see if we are ready to proceed */ - if(pData->f_hmysql == NULL) { - CHKiRet(initMySQL(pData, 0)); + if(pWrkrData->hmysql == NULL) { + CHKiRet(initMySQL(pWrkrData, 0)); } /* try insert */ - if(mysql_query(pData->f_hmysql, (char*)psz)) { + if(mysql_query(pWrkrData->hmysql, (char*)psz)) { /* error occured, try to re-init connection and retry */ - closeMySQL(pData); /* close the current handle */ - CHKiRet(initMySQL(pData, 0)); /* try to re-open */ - if(mysql_query(pData->f_hmysql, (char*)psz)) { /* re-try insert */ + closeMySQL(pWrkrData); /* close the current handle */ + CHKiRet(initMySQL(pWrkrData, 0)); /* try to re-open */ + if(mysql_query(pWrkrData->hmysql, (char*)psz)) { /* re-try insert */ /* we failed, giving up for now */ - reportDBError(pData, 0); - closeMySQL(pData); /* free ressources */ + reportDBError(pWrkrData, 0); + closeMySQL(pWrkrData); /* free ressources */ ABORT_FINALIZE(RS_RET_SUSPENDED); } } finalize_it: if(iRet == RS_RET_OK) { - pData->uLastMySQLErrno = 0; /* reset error for error supression */ + pWrkrData->uLastMySQLErrno = 0; /* reset error for error supression */ } RETiRet; @@ -261,28 +270,28 @@ finalize_it: BEGINtryResume CODESTARTtryResume - if(pData->f_hmysql == NULL) { - iRet = initMySQL(pData, 1); + if(pWrkrData->hmysql == NULL) { + iRet = initMySQL(pWrkrData, 1); } ENDtryResume BEGINbeginTransaction CODESTARTbeginTransaction - CHKiRet(writeMySQL((uchar*)"START TRANSACTION", pData)); + CHKiRet(writeMySQL(pWrkrData, (uchar*)"START TRANSACTION")); finalize_it: ENDbeginTransaction BEGINdoAction CODESTARTdoAction dbgprintf("\n"); - CHKiRet(writeMySQL(ppString[0], pData)); + CHKiRet(writeMySQL(pWrkrData, ppString[0])); iRet = RS_RET_DEFER_COMMIT; finalize_it: ENDdoAction BEGINendTransaction CODESTARTendTransaction - if (mysql_commit(pData->f_hmysql) != 0) { + if(mysql_commit(pWrkrData->hmysql) != 0) { dbgprintf("mysql server error: transaction not committed\n"); iRet = RS_RET_SUSPENDED; } @@ -293,10 +302,9 @@ static inline void setInstParamDefaults(instanceData *pData) { pData->dbsrvPort = 0; - pData->f_configfile = NULL; - pData->f_configsection = NULL; + pData->configfile = NULL; + pData->configsection = NULL; pData->tplName = NULL; - pData->f_hmysql = NULL; /* initialize, but connect only on first message (important for queued mode!) */ } @@ -338,9 +346,9 @@ CODESTARTnewActInst strncpy(pData->dbpwd, cstr, sizeof(pData->dbpwd)); free(cstr); } else if(!strcmp(actpblk.descr[i].name, "mysqlconfig.file")) { - pData->f_configfile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + pData->configfile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "mysqlconfig.section")) { - pData->f_configsection = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + pData->configsection = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(actpblk.descr[i].name, "template")) { pData->tplName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); } else { @@ -424,9 +432,8 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } else { pData->dbsrvPort = (unsigned) cs.iSrvPort; /* set configured port */ - pData->f_configfile = cs.pszMySQLConfigFile; - pData->f_configsection = cs.pszMySQLConfigSection; - pData->f_hmysql = NULL; /* initialize, but connect only on first message (important for queued mode!) */ + pData->configfile = cs.pszMySQLConfigFile; + pData->configsection = cs.pszMySQLConfigSection; } CODE_STD_FINALIZERparseSelectorAct @@ -446,6 +453,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt @@ -484,7 +492,7 @@ CODEmodInit_QueryRegCFSLineHdlr mysql_server_init(0, NULL, NULL) # endif ) { - errmsg.LogError(0, NO_ERRCODE, "ommysql: mysql_server_init() failed, plugin " + errmsg.LogError(0, NO_ERRCODE, "ommysql: intializing mysql client failed, plugin " "can not run"); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 11f346f6..87599484 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -6,7 +6,7 @@ * * File begun on 2007-10-18 by sur5r (converted from ommysql.c) * - * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2013 Rainer Gerhards and Adiscon GmbH. * * The following link my be useful for the not-so-postgres literate * when setting up a test environment (on Fedora): @@ -66,11 +66,17 @@ typedef struct _instanceData { ConnStatusType eLastPgSQLStatus; /* last status from postgres */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { EMPTY_STRUCT } configSettings_t; static configSettings_t __attribute__((unused)) cs; +static pthread_mutex_t mutDoAct = PTHREAD_MUTEX_INITIALIZER; + BEGINinitConfVars /* (re)set config variables to default values */ CODESTARTinitConfVars ENDinitConfVars @@ -82,6 +88,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -108,6 +118,9 @@ CODESTARTfreeInstance closePgSQL(pData); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -244,8 +257,8 @@ finalize_it: BEGINtryResume CODESTARTtryResume - if(pData->f_hpgsql == NULL) { - iRet = initPgSQL(pData, 1); + if(pWrkrData->pData->f_hpgsql == NULL) { + iRet = initPgSQL(pWrkrData->pData, 1); if(iRet == RS_RET_OK) { /* the code above seems not to actually connect to the database. As such, we do a * dummy statement (a pointless select...) to verify the connection and return @@ -253,7 +266,7 @@ CODESTARTtryResume * PostgreSQL expert, so any patch that does the desired result in a more * intelligent way is highly welcome. -- rgerhards, 2009-12-16 */ - iRet = writePgSQL((uchar*)"select 'a' as a", pData); + iRet = writePgSQL((uchar*)"select 'a' as a", pWrkrData->pData); } } @@ -263,23 +276,25 @@ ENDtryResume BEGINbeginTransaction CODESTARTbeginTransaction dbgprintf("ompgsql: beginTransaction\n"); - iRet = writePgSQL((uchar*) "begin", pData); /* TODO: make user-configurable */ + iRet = writePgSQL((uchar*) "begin", pWrkrData->pData); /* TODO: make user-configurable */ ENDbeginTransaction BEGINdoAction CODESTARTdoAction + pthread_mutex_lock(&mutDoAct); dbgprintf("\n"); - CHKiRet(writePgSQL(ppString[0], pData)); + CHKiRet(writePgSQL(ppString[0], pWrkrData->pData)); if(bCoreSupportsBatching) iRet = RS_RET_DEFER_COMMIT; finalize_it: + pthread_mutex_unlock(&mutDoAct); ENDdoAction BEGINendTransaction CODESTARTendTransaction - iRet = writePgSQL((uchar*) "commit;", pData); /* TODO: make user-configurable */ + iRet = writePgSQL((uchar*) "commit;", pWrkrData->pData); /* TODO: make user-configurable */ dbgprintf("ompgsql: endTransaction\n"); ENDendTransaction @@ -361,6 +376,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt @@ -372,6 +388,11 @@ INITLegCnfVars CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + +# warning: transaction support missing for v8 + bCoreSupportsBatching= 0; + DBGPRINTF("ompgsql: transactions are not yet supported on v8\n"); + DBGPRINTF("ompgsql: module compiled with rsyslog version %s.\n", VERSION); DBGPRINTF("ompgsql: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); ENDmodInit diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index cd07dcfb..1060b5c3 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -6,7 +6,7 @@ * * File begun on 2009-04-01 by RGerhards * - * Copyright 2009-2012 Adiscon GmbH. + * Copyright 2009-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -35,6 +35,7 @@ #include <errno.h> #include <unistd.h> #include <wait.h> +#include <pthread.h> #include "conf.h" #include "syslogd-types.h" #include "srUtils.h" @@ -60,8 +61,13 @@ typedef struct _instanceData { int fdPipe; /* file descriptor to write to */ int bIsRunning; /* is binary currently running? 0-no, 1-yes */ int iParams; /* Holds the count of parameters if set*/ + pthread_mutex_t mut; /* make sure only one instance is active */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { uchar *szBinary; /* name of binary to call */ } configSettings_t; @@ -89,8 +95,13 @@ ENDinitConfVars BEGINcreateInstance CODESTARTcreateInstance + pthread_mutex_init(&pData->mut, NULL); ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -102,6 +113,7 @@ ENDisCompatibleWithFeature BEGINfreeInstance int i; CODESTARTfreeInstance + pthread_mutex_destroy(&pData->mut); if(pData->szBinary != NULL) free(pData->szBinary); if(pData->aParams != NULL) { @@ -112,6 +124,10 @@ CODESTARTfreeInstance } ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -287,22 +303,21 @@ writePipe(instanceData *pData, uchar *szMsg) lenWrite = strlen((char*)szMsg); writeOffset = 0; - do - { + do { lenWritten = write(pData->fdPipe, ((char*)szMsg)+writeOffset, lenWrite); if(lenWritten == -1) { switch(errno) { - case EPIPE: - DBGPRINTF("omprog: Program '%s' terminated, trying to restart\n", - pData->szBinary); - CHKiRet(cleanup(pData)); - CHKiRet(tryRestart(pData)); - break; - default: - DBGPRINTF("omprog: error %d writing to pipe: %s\n", errno, - rs_strerror_r(errno, errStr, sizeof(errStr))); - ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); - break; + case EPIPE: + DBGPRINTF("omprog: Program '%s' terminated, trying to restart\n", + pData->szBinary); + CHKiRet(cleanup(pData)); + CHKiRet(tryRestart(pData)); + break; + default: + DBGPRINTF("omprog: error %d writing to pipe: %s\n", errno, + rs_strerror_r(errno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); + break; } } else { writeOffset += lenWritten; @@ -316,7 +331,10 @@ finalize_it: BEGINdoAction + instanceData *pData; CODESTARTdoAction + pData = pWrkrData->pData; + pthread_mutex_lock(&pData->mut); if(pData->bIsRunning == 0) { openPipe(pData); } @@ -325,6 +343,7 @@ CODESTARTdoAction if(iRet != RS_RET_OK) iRet = RS_RET_SUSPENDED; + pthread_mutex_unlock(&pData->mut); ENDdoAction @@ -496,6 +515,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 34511e46..3b1584e2 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -2,8 +2,17 @@ * * This is the implementation of the RELP output module. * - * NOTE: read comments in module-template.h to understand how this file - * works! + * Note that when multiple action workers are activated, we currently + * also create multiple actions. This may be the source of some mild + * message loss (!) if the worker instance is shut down while the + * connection to the remote system is in retry state. + * TODO: think if we should implement a mode where we do NOT + * support multiple action worker instances. This would be + * slower, but not have this loss opportunity. But it should + * definitely be optional and by default off due to the + * performance implications (and given the fact that message + * loss is pretty unlikely in usual cases). + * * * File begun on 2008-03-13 by RGerhards * @@ -63,13 +72,9 @@ static relpEngine_t *pRelpEngine; /* our relp engine */ typedef struct _instanceData { uchar *target; uchar *port; - int bInitialConnect; /* is this the initial connection request of our module? (0-no, 1-yes) */ - int bIsConnected; /* currently connected to server? 0 - no, 1 - yes */ int sizeWindow; /**< the RELP window size - 0=use default */ unsigned timeout; unsigned rebindInterval; - unsigned nSent; - relpClt_t *pRelpClt; /* relp client for this instance */ sbool bEnableTLS; sbool bEnableTLSZip; sbool bHadAuthFail; /**< set on auth failure, will cause retry to disable action */ @@ -85,11 +90,20 @@ typedef struct _instanceData { } permittedPeers; } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; + int bInitialConnect; /* is this the initial connection request of our module? (0-no, 1-yes) */ + int bIsConnected; /* currently connected to server? 0 - no, 1 - yes */ + relpClt_t *pRelpClt; /* relp client for this instance */ + unsigned nSent; /* number msgs sent - for rebind support */ +} wrkrInstanceData_t; + typedef struct configSettings_s { EMPTY_STRUCT } configSettings_t; static configSettings_t __attribute__((unused)) cs; +static rsRetVal doCreateRelpClient(wrkrInstanceData_t *pWrkrData); /* tables for interfacing with the v6 config system */ /* action (instance) parameters */ @@ -135,10 +149,10 @@ static uchar *getRelpPt(instanceData *pData) static void onErr(void *pUsr, char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) { - instanceData *pData = (instanceData*) pUsr; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t*) pUsr; errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "omrelp[%s:%s]: error '%s', object " " '%s' - action may not work as intended", - pData->target, pData->port, errmesg, objinfo); + pWrkrData->pData->target, pWrkrData->pData->port, errmesg, objinfo); } static void @@ -152,55 +166,58 @@ onGenericErr(char *objinfo, char* errmesg, __attribute__((unused)) relpRetVal er static void onAuthErr(void *pUsr, char *authinfo, char* errmesg, __attribute__((unused)) relpRetVal errcode) { - instanceData *pData = (instanceData*) pUsr; + instanceData *pData = ((wrkrInstanceData_t*) pUsr)->pData; errmsg.LogError(0, RS_RET_RELP_AUTH_FAIL, "omrelp[%s:%s]: authentication error '%s', peer " "is '%s' - DISABLING action", pData->target, pData->port, errmesg, authinfo); pData->bHadAuthFail = 1; } -static inline rsRetVal -doCreateRelpClient(instanceData *pData) +static rsRetVal +doCreateRelpClient(wrkrInstanceData_t *pWrkrData) { int i; + instanceData *pData; DEFiRet; - if(relpEngineCltConstruct(pRelpEngine, &pData->pRelpClt) != RELP_RET_OK) + + pData = pWrkrData->pData; + if(relpEngineCltConstruct(pRelpEngine, &pWrkrData->pRelpClt) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetTimeout(pData->pRelpClt, pData->timeout) != RELP_RET_OK) + if(relpCltSetTimeout(pWrkrData->pRelpClt, pData->timeout) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetWindowSize(pData->pRelpClt, pData->sizeWindow) != RELP_RET_OK) + if(relpCltSetWindowSize(pWrkrData->pRelpClt, pData->sizeWindow) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetUsrPtr(pData->pRelpClt, pData) != RELP_RET_OK) + if(relpCltSetUsrPtr(pWrkrData->pRelpClt, pWrkrData) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); if(pData->bEnableTLS) { - if(relpCltEnableTLS(pData->pRelpClt) != RELP_RET_OK) + if(relpCltEnableTLS(pWrkrData->pRelpClt) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); if(pData->bEnableTLSZip) { - if(relpCltEnableTLSZip(pData->pRelpClt) != RELP_RET_OK) + if(relpCltEnableTLSZip(pWrkrData->pRelpClt) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); } - if(relpCltSetGnuTLSPriString(pData->pRelpClt, (char*) pData->pristring) != RELP_RET_OK) + if(relpCltSetGnuTLSPriString(pWrkrData->pRelpClt, (char*) pData->pristring) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetAuthMode(pData->pRelpClt, (char*) pData->authmode) != RELP_RET_OK) { + if(relpCltSetAuthMode(pWrkrData->pRelpClt, (char*) pData->authmode) != RELP_RET_OK) { errmsg.LogError(0, RS_RET_RELP_ERR, "omrelp: invalid auth mode '%s'\n", pData->authmode); ABORT_FINALIZE(RS_RET_RELP_ERR); } - if(relpCltSetCACert(pData->pRelpClt, (char*) pData->caCertFile) != RELP_RET_OK) + if(relpCltSetCACert(pWrkrData->pRelpClt, (char*) pData->caCertFile) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetOwnCert(pData->pRelpClt, (char*) pData->myCertFile) != RELP_RET_OK) + if(relpCltSetOwnCert(pWrkrData->pRelpClt, (char*) pData->myCertFile) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); - if(relpCltSetPrivKey(pData->pRelpClt, (char*) pData->myPrivKeyFile) != RELP_RET_OK) + if(relpCltSetPrivKey(pWrkrData->pRelpClt, (char*) pData->myPrivKeyFile) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); for(i = 0 ; i < pData->permittedPeers.nmemb ; ++i) { - relpCltAddPermittedPeer(pData->pRelpClt, (char*)pData->permittedPeers.name[i]); + relpCltAddPermittedPeer(pWrkrData->pRelpClt, (char*)pData->permittedPeers.name[i]); } } if(glbl.GetSourceIPofLocalClient() == NULL) { /* ar Do we have a client IP set? */ - if(relpCltSetClientIP(pData->pRelpClt, glbl.GetSourceIPofLocalClient()) != RELP_RET_OK) + if(relpCltSetClientIP(pWrkrData->pRelpClt, glbl.GetSourceIPofLocalClient()) != RELP_RET_OK) ABORT_FINALIZE(RS_RET_RELP_ERR); } - pData->bInitialConnect = 1; - pData->nSent = 0; + pWrkrData->bInitialConnect = 1; + pWrkrData->nSent = 0; finalize_it: RETiRet; } @@ -221,11 +238,15 @@ CODESTARTcreateInstance pData->permittedPeers.nmemb = 0; ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->pRelpClt = NULL; + iRet = doCreateRelpClient(pWrkrData); +ENDcreateWrkrInstance + BEGINfreeInstance int i; CODESTARTfreeInstance - if(pData->pRelpClt != NULL) - relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt); free(pData->target); free(pData->port); free(pData->tplName); @@ -239,6 +260,12 @@ CODESTARTfreeInstance } ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + if(pWrkrData->pRelpClt != NULL) + relpEngineCltDestruct(pRelpEngine, &pWrkrData->pRelpClt); +ENDfreeWrkrInstance + static inline void setInstParamDefaults(instanceData *pData) { @@ -318,8 +345,6 @@ CODESTARTnewActInst "RSYSLOG_ForwardFormat" : (char*)pData->tplName), OMSR_NO_RQD_TPL_OPTS)); - CHKiRet(doCreateRelpClient(pData)); - CODE_STD_FINALIZERnewActInst if(pvals != NULL) cnfparamvalsDestruct(pvals, &actpblk); @@ -347,22 +372,23 @@ ENDdbgPrintInstInfo /* try to connect to server * rgerhards, 2008-03-21 */ -static rsRetVal doConnect(instanceData *pData) +static rsRetVal doConnect(wrkrInstanceData_t *pWrkrData) { DEFiRet; - if(pData->bInitialConnect) { - iRet = relpCltConnect(pData->pRelpClt, glbl.GetDefPFFamily(), pData->port, pData->target); + if(pWrkrData->bInitialConnect) { + iRet = relpCltConnect(pWrkrData->pRelpClt, glbl.GetDefPFFamily(), + pWrkrData->pData->port, pWrkrData->pData->target); if(iRet == RELP_RET_OK) - pData->bInitialConnect = 0; + pWrkrData->bInitialConnect = 0; } else { - iRet = relpCltReconnect(pData->pRelpClt); + iRet = relpCltReconnect(pWrkrData->pRelpClt); } if(iRet == RELP_RET_OK) { - pData->bIsConnected = 1; + pWrkrData->bIsConnected = 1; } else { - pData->bIsConnected = 0; + pWrkrData->bIsConnected = 0; iRet = RS_RET_SUSPENDED; } @@ -372,21 +398,21 @@ static rsRetVal doConnect(instanceData *pData) BEGINtryResume CODESTARTtryResume - if(pData->bHadAuthFail) { + if(pWrkrData->pData->bHadAuthFail) { ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } - iRet = doConnect(pData); + iRet = doConnect(pWrkrData); finalize_it: ENDtryResume static inline rsRetVal -doRebind(instanceData *pData) +doRebind(wrkrInstanceData_t *pWrkrData) { DEFiRet; DBGPRINTF("omrelp: destructing relp client due to rebindInterval\n"); - CHKiRet(relpEngineCltDestruct(pRelpEngine, &pData->pRelpClt)); - pData->bIsConnected = 0; - CHKiRet(doCreateRelpClient(pData)); + CHKiRet(relpEngineCltDestruct(pRelpEngine, &pWrkrData->pRelpClt)); + pWrkrData->bIsConnected = 0; + CHKiRet(doCreateRelpClient(pWrkrData)); finalize_it: RETiRet; } @@ -394,10 +420,10 @@ finalize_it: BEGINbeginTransaction CODESTARTbeginTransaction dbgprintf("omrelp: beginTransaction\n"); - if(!pData->bIsConnected) { - CHKiRet(doConnect(pData)); + if(!pWrkrData->bIsConnected) { + CHKiRet(doConnect(pWrkrData)); } - relpCltHintBurstBegin(pData->pRelpClt); + relpCltHintBurstBegin(pWrkrData->pRelpClt); finalize_it: ENDbeginTransaction @@ -405,11 +431,13 @@ BEGINdoAction uchar *pMsg; /* temporary buffering */ size_t lenMsg; relpRetVal ret; + instanceData *pData; CODESTARTdoAction + pData = pWrkrData->pData; dbgprintf(" %s:%s/RELP\n", pData->target, getRelpPt(pData)); - if(!pData->bIsConnected) { - CHKiRet(doConnect(pData)); + if(!pWrkrData->bIsConnected) { + CHKiRet(doConnect(pWrkrData)); } pMsg = ppString[0]; @@ -420,7 +448,7 @@ CODESTARTdoAction lenMsg = glbl.GetMaxLine(); /* forward */ - ret = relpCltSendSyslog(pData->pRelpClt, (uchar*) pMsg, lenMsg); + ret = relpCltSendSyslog(pWrkrData->pRelpClt, (uchar*) pMsg, lenMsg); if(ret != RELP_RET_OK) { /* error! */ dbgprintf("error forwarding via relp, suspending\n"); @@ -428,8 +456,8 @@ CODESTARTdoAction } if(pData->rebindInterval != 0 && - (++pData->nSent >= pData->rebindInterval)) { - doRebind(pData); + (++pWrkrData->nSent >= pData->rebindInterval)) { + doRebind(pWrkrData); } finalize_it: if(pData->bHadAuthFail) @@ -448,7 +476,7 @@ ENDdoAction BEGINendTransaction CODESTARTendTransaction dbgprintf("omrelp: endTransaction\n"); - relpCltHintBurstEnd(pData->pRelpClt); + relpCltHintBurstEnd(pWrkrData->pRelpClt); ENDendTransaction BEGINparseSelectorAct @@ -527,8 +555,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (uchar*) "RSYSLOG_ForwardFormat")); - CHKiRet(doCreateRelpClient(pData)); - CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -546,6 +572,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_TXIF_OMOD_QUERIES diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c index 11765507..73419915 100644 --- a/plugins/omruleset/omruleset.c +++ b/plugins/omruleset/omruleset.c @@ -10,7 +10,7 @@ * * File begun on 2009-11-02 by RGerhards * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -70,6 +70,10 @@ typedef struct _instanceData { uchar *pszRulesetName; /* primarily for debugging/display purposes */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { ruleset_t *pRuleset; /* ruleset to enqueue message to (NULL = Default, not recommended) */ uchar *pszRulesetName; @@ -87,11 +91,21 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINfreeInstance CODESTARTfreeInstance free(pData->pszRulesetName); @@ -117,9 +131,9 @@ BEGINdoAction CODESTARTdoAction CHKmalloc(pMsg = MsgDup((msg_t*) ppString[0])); DBGPRINTF(":omruleset: forwarding message %p to ruleset %s[%p]\n", pMsg, - (char*) pData->pszRulesetName, pData->pRuleset); + (char*) pWrkrData->pData->pszRulesetName, pWrkrData->pData->pRuleset); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - MsgSetRuleset(pMsg, pData->pRuleset); + MsgSetRuleset(pMsg, pWrkrData->pData->pRuleset); /* Note: we intentionally use submitMsg2() here, as we process messages * that were already run through the rate-limiter. So it is (at least) * questionable if they were rate-limited again. @@ -199,6 +213,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES ENDqueryEtryPt diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index 42d1de6b..02862c94 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -2,7 +2,7 @@ * * This module sends an snmp trap. * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -74,15 +74,19 @@ typedef struct _instanceData { * http://www.adiscon.org/download/ADISCON-MONITORWARE-MIB.txt * http://www.adiscon.org/download/ADISCON-MIB.txt */ - int iPort; /* Target Port */ - int iSNMPVersion; /* SNMP Version to use */ - int iTrapType; /* Snmp TrapType or GenericType */ - int iSpecificType; /* Snmp Specific Type */ + int iPort; /* Target Port */ + int iSNMPVersion; /* SNMP Version to use */ + int iTrapType; /* Snmp TrapType or GenericType */ + int iSpecificType; /* Snmp Specific Type */ - netsnmp_session *snmpsession; /* Holds to SNMP Session, NULL if not initialized */ - uchar *tplName; /* format template to use */ + uchar *tplName; /* format template to use */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; + netsnmp_session *snmpsession; /* Holds to SNMP Session, NULL if not initialized */ +} wrkrInstanceData_t; + typedef struct configSettings_s { uchar* pszTransport; /* default transport */ uchar* pszTarget; @@ -147,6 +151,10 @@ BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->snmpsession = NULL; +ENDcreateWrkrInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -171,14 +179,16 @@ ENDisCompatibleWithFeature /* Exit SNMP Session * alorbach, 2008-02-12 */ -static rsRetVal omsnmp_exitSession(instanceData *pData) +static rsRetVal +omsnmp_exitSession(wrkrInstanceData_t *pWrkrData) { DEFiRet; - if(pData->snmpsession != NULL) { - dbgprintf( "omsnmp_exitSession: Clearing Session to '%s' on Port = '%d'\n", pData->szTarget, pData->iPort); - snmp_close(pData->snmpsession); - pData->snmpsession = NULL; + if(pWrkrData->snmpsession != NULL) { + DBGPRINTF("omsnmp_exitSession: Clearing Session to '%s' on Port = '%d'\n", + pWrkrData->pData->szTarget, pWrkrData->pData->iPort); + snmp_close(pWrkrData->snmpsession); + pWrkrData->snmpsession = NULL; } RETiRet; @@ -187,15 +197,19 @@ static rsRetVal omsnmp_exitSession(instanceData *pData) /* Init SNMP Session * alorbach, 2008-02-12 */ -static rsRetVal omsnmp_initSession(instanceData *pData) +static rsRetVal +omsnmp_initSession(wrkrInstanceData_t *pWrkrData) { netsnmp_session session; + instanceData *pData; char szTargetAndPort[MAXHOSTNAMELEN+128]; /* work buffer for specifying a full target and port string */ DEFiRet; /* should not happen, but if session is not cleared yet - we do it now! */ - if (pData->snmpsession != NULL) - omsnmp_exitSession(pData); + if (pWrkrData->snmpsession != NULL) + omsnmp_exitSession(pWrkrData); + + pData = pWrkrData->pData; snprintf((char*)szTargetAndPort, sizeof(szTargetAndPort), "%s:%s:%d", (pData->szTransport == NULL) ? "udp" : (char*)pData->szTransport, @@ -217,8 +231,8 @@ static rsRetVal omsnmp_initSession(instanceData *pData) session.community_len = strlen((char*) session.community); } - pData->snmpsession = snmp_open(&session); - if (pData->snmpsession == NULL) { + pWrkrData->snmpsession = snmp_open(&session); + if (pWrkrData->snmpsession == NULL) { errmsg.LogError(0, RS_RET_SUSPENDED, "omsnmp_initSession: snmp_open to host '%s' on Port '%d' failed\n", pData->szTarget, pData->iPort); /* Stay suspended */ iRet = RS_RET_SUSPENDED; @@ -227,7 +241,7 @@ static rsRetVal omsnmp_initSession(instanceData *pData) RETiRet; } -static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) +static rsRetVal omsnmp_sendsnmp(wrkrInstanceData_t *pWrkrData, uchar *psz) { DEFiRet; @@ -239,10 +253,12 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) int status; char *trap = NULL; const char *strErr = NULL; + instanceData *pData; + pData = pWrkrData->pData; /* Init SNMP Session if necessary */ - if (pData->snmpsession == NULL) { - CHKiRet(omsnmp_initSession(pData)); + if (pWrkrData->snmpsession == NULL) { + CHKiRet(omsnmp_initSession(pWrkrData)); } /* String should not be NULL */ @@ -250,7 +266,7 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) dbgprintf( "omsnmp_sendsnmp: ENTER - Syslogmessage = '%s'\n", (char*)psz); /* If SNMP Version1 is configured !*/ - if(pData->snmpsession->version == SNMP_VERSION_1) { + if(pWrkrData->snmpsession->version == SNMP_VERSION_1) { pdu = snmp_pdu_create(SNMP_MSG_TRAP); /* Set enterprise */ @@ -275,7 +291,7 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) pdu->time = get_uptime(); } /* If SNMP Version2c is configured !*/ - else if (pData->snmpsession->version == SNMP_VERSION_2c) + else if (pWrkrData->snmpsession->version == SNMP_VERSION_2c) { long sysuptime; char csysuptime[20]; @@ -320,15 +336,15 @@ static rsRetVal omsnmp_sendsnmp(instanceData *pData, uchar *psz) } /* Send the TRAP */ - status = snmp_send(pData->snmpsession, pdu) == 0; + status = snmp_send(pWrkrData->snmpsession, pdu) == 0; if (status) { /* Debug Output! */ - int iErrorCode = pData->snmpsession->s_snmp_errno; + int iErrorCode = pWrkrData->snmpsession->s_snmp_errno; errmsg.LogError(0, RS_RET_SUSPENDED, "omsnmp_sendsnmp: snmp_send failed error '%d', Description='%s'\n", iErrorCode*(-1), api_errors[iErrorCode*(-1)]); /* Clear Session */ - omsnmp_exitSession(pData); + omsnmp_exitSession(pWrkrData); ABORT_FINALIZE(RS_RET_SUSPENDED); } @@ -347,7 +363,7 @@ finalize_it: BEGINtryResume CODESTARTtryResume - iRet = omsnmp_initSession(pData); + iRet = omsnmp_initSession(pWrkrData); ENDtryResume BEGINdoAction @@ -358,19 +374,20 @@ CODESTARTdoAction } /* This will generate and send the SNMP Trap */ - iRet = omsnmp_sendsnmp(pData, ppString[0]); + iRet = omsnmp_sendsnmp(pWrkrData, ppString[0]); finalize_it: ENDdoAction BEGINfreeInstance CODESTARTfreeInstance - /* free snmp Session here */ - omsnmp_exitSession(pData); - free(pData->tplName); free(pData->szTarget); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + omsnmp_exitSession(pWrkrData); +ENDfreeWrkrInstance static inline void setInstParamDefaults(instanceData *pData) @@ -499,9 +516,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* Set some defaults in the NetSNMP library */ netsnmp_ds_set_int(NETSNMP_DS_LIBRARY_ID, NETSNMP_DS_LIB_DEFAULT_PORT, pData->iPort ); - - /* Init Session Pointer */ - pData->snmpsession = NULL; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -545,6 +559,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES ENDqueryEtryPt diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index a84a7593..210b0165 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -6,7 +6,7 @@ * * File begun on 2009-03-19 by RGerhards * - * Copyright 2009-2012 Adiscon GmbH. + * Copyright 2009-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -60,6 +60,10 @@ typedef struct _instanceData { int bEnsureLFEnding; /* ensure that a linefeed is written at the end of EACH record (test aid for nettester) */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { int bUseArrayInterface; /* shall action use array instead of string template interface? */ int bEnsureLFEnding; /* shall action use array instead of string template interface? */ @@ -76,6 +80,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -88,6 +97,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo ENDdbgPrintInstInfo @@ -107,7 +121,7 @@ BEGINdoAction size_t len; int r; CODESTARTdoAction - if(pData->bUseArrayInterface) { + if(pWrkrData->pData->bUseArrayInterface) { /* if we use array passing, we need to put together a string * ourselves. At this point, please keep in mind that omstdout is * primarily a testing aid. Other modules may do different processing @@ -145,7 +159,7 @@ CODESTARTdoAction DBGPRINTF("omstdout: error %d writing to stdout[%d]: %s\n", r, len, toWrite); } - if(pData->bEnsureLFEnding && toWrite[len-1] != '\n') { + if(pWrkrData->pData->bEnsureLFEnding && toWrite[len-1] != '\n') { if((r = write(1, "\n", 1)) != 1) { /* write missing LF */ DBGPRINTF("omstdout: error %d writing \\n to stdout\n", r); @@ -186,6 +200,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES ENDqueryEtryPt diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index c9f1e06b..2cc1159e 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -22,7 +22,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -63,7 +63,6 @@ MODULE_CNFNAME("omtesting") */ DEF_OMOD_STATIC_DATA - typedef struct _instanceData { enum { MD_SLEEP, MD_FAIL, MD_RANDFAIL, MD_ALWAYS_SUSPEND } mode; @@ -76,6 +75,10 @@ typedef struct _instanceData { int iCurrRetries; } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { int bEchoStdout; /* echo non-failed messages to stdout */ } configSettings_t; @@ -93,6 +96,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo dbgprintf("Action delays rule by %d second(s) and %d millisecond(s)\n", @@ -170,11 +178,11 @@ static rsRetVal doRandFail(void) BEGINtryResume CODESTARTtryResume dbgprintf("omtesting tryResume() called\n"); - switch(pData->mode) { + switch(pWrkrData->pData->mode) { case MD_SLEEP: break; case MD_FAIL: - iRet = doFailOnResume(pData); + iRet = doFailOnResume(pWrkrData->pData); break; case MD_RANDFAIL: iRet = doRandFail(); @@ -187,8 +195,10 @@ ENDtryResume BEGINdoAction + instanceData *pData; CODESTARTdoAction dbgprintf("omtesting received msg '%s'\n", ppString[0]); + pData = pWrkrData->pData; switch(pData->mode) { case MD_SLEEP: iRet = doSleep(pData); @@ -220,6 +230,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINparseSelectorAct int i; uchar szBuf[1024]; @@ -313,6 +328,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES ENDqueryEtryPt diff --git a/plugins/omudpspoof/omudpspoof.c b/plugins/omudpspoof/omudpspoof.c index cb907bba..ad3508c8 100644 --- a/plugins/omudpspoof/omudpspoof.c +++ b/plugins/omudpspoof/omudpspoof.c @@ -98,15 +98,19 @@ typedef struct _instanceData { uchar *port; uchar *sourceTpl; int mtu; - int *pSockArray; /* sockets to use for UDP */ - struct addrinfo *f_addr; - u_short sourcePort; u_short sourcePortStart; /* for sorce port iteration */ u_short sourcePortEnd; int bReportLibnetInitErr; /* help prevent multiple error messages on init err */ +} instanceData; + +typedef struct wrkrInstanceData { + instanceData *pData; libnet_t *libnet_handle; + u_short sourcePort; + int *pSockArray; /* sockets to use for UDP */ + struct addrinfo *f_addr; char errbuf[LIBNET_ERRBUF_SIZE]; -} instanceData; +} wrkrInstanceData_t; #define DFLT_SOURCE_PORT_START 32000 #define DFLT_SOURCE_PORT_END 42000 @@ -172,7 +176,7 @@ ENDinitConfVars pthread_mutex_t mutLibnet; /* forward definitions */ -static rsRetVal doTryResume(instanceData *pData); +static rsRetVal doTryResume(wrkrInstanceData_t *pWrkrData); /* this function gets the default template. It coordinates action between @@ -217,15 +221,14 @@ finalize_it: * rgerhards, 2009-05-29 */ static rsRetVal -closeUDPSockets(instanceData *pData) +closeUDPSockets(wrkrInstanceData_t *pWrkrData) { DEFiRet; - assert(pData != NULL); - if(pData->pSockArray != NULL) { - net.closeUDPListenSockets(pData->pSockArray); - pData->pSockArray = NULL; - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; + if(pWrkrData->pSockArray != NULL) { + net.closeUDPListenSockets(pWrkrData->pSockArray); + pWrkrData->pSockArray = NULL; + freeaddrinfo(pWrkrData->f_addr); + pWrkrData->f_addr = NULL; } RETiRet; } @@ -310,12 +313,17 @@ ENDfreeCnf BEGINcreateInstance CODESTARTcreateInstance - pData->libnet_handle = NULL; pData->mtu = 1500; pData->bReportLibnetInitErr = 1; ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + pWrkrData->libnet_handle = NULL; + pWrkrData->sourcePort = pData->sourcePortStart; +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -326,15 +334,19 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance /* final cleanup */ - closeUDPSockets(pData); free(pData->tplName); free(pData->port); free(pData->host); free(pData->sourceTpl); - if(pData->libnet_handle != NULL) - libnet_destroy(pData->libnet_handle); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + closeUDPSockets(pWrkrData); + if(pWrkrData->libnet_handle != NULL) + libnet_destroy(pWrkrData->libnet_handle); +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -348,11 +360,12 @@ ENDdbgPrintInstInfo * rgehards, 2007-12-20 */ static inline rsRetVal -UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) +UDPSend(wrkrInstanceData_t *pWrkrData, uchar *pszSourcename, char *msg, size_t len) { struct addrinfo *r; int lsent = 0; int bSendSuccess; + instanceData *pData; struct sockaddr_in *tempaddr,source_ip; libnet_ptag_t ip, ipo; libnet_ptag_t udp; @@ -363,9 +376,10 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) unsigned maxPktLen, pktLen; DEFiRet; - if(pData->pSockArray == NULL) { - CHKiRet(doTryResume(pData)); + if(pWrkrData->pSockArray == NULL) { + CHKiRet(doTryResume(pWrkrData)); } + pData = pWrkrData->pData; if(len > 65528) { DBGPRINTF("omudpspoof: msg with length %d truncated to 64k: '%.768s'\n", @@ -374,8 +388,8 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) } ip = ipo = udp = 0; - if(pData->sourcePort++ >= pData->sourcePortEnd){ - pData->sourcePort = pData->sourcePortStart; + if(pWrkrData->sourcePort++ >= pData->sourcePortEnd){ + pWrkrData->sourcePort = pData->sourcePortStart; } inet_pton(AF_INET, (char*)pszSourcename, &(source_ip.sin_addr)); @@ -383,7 +397,7 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) bSendSuccess = RSFALSE; d_pthread_mutex_lock(&mutLibnet); bNeedUnlock = 1; - for (r = pData->f_addr; r && bSendSuccess == RSFALSE ; r = r->ai_next) { + for (r = pWrkrData->f_addr; r && bSendSuccess == RSFALSE ; r = r->ai_next) { tempaddr = (struct sockaddr_in *)r->ai_addr; /* Getting max payload size (must be multiple of 8) */ maxPktLen = (pData->mtu - LIBNET_IPV4_H) & ~0x07; @@ -400,19 +414,19 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) } DBGPRINTF("omudpspoof: stage 1: MF:%d, hdrOffs %d, pktLen %d\n", (hdrOffs & IP_MF) >> 13, (hdrOffs & 0x1FFF) << 3, pktLen); - libnet_clear_packet(pData->libnet_handle); + libnet_clear_packet(pWrkrData->libnet_handle); /* note: libnet does need ports in host order NOT in network byte order! -- rgerhards, 2009-11-12 */ udp = libnet_build_udp( - ntohs(pData->sourcePort),/* source port */ + ntohs(pWrkrData->sourcePort),/* source port */ ntohs(tempaddr->sin_port),/* destination port */ pktLen+LIBNET_UDP_H, /* packet length */ 0, /* checksum */ (u_char*)msg, /* payload */ pktLen, /* payload size */ - pData->libnet_handle, /* libnet handle */ + pWrkrData->libnet_handle, /* libnet handle */ udp); /* libnet id */ if (udp == -1) { - DBGPRINTF("omudpspoof: can't build UDP header: %s\n", libnet_geterror(pData->libnet_handle)); + DBGPRINTF("omudpspoof: can't build UDP header: %s\n", libnet_geterror(pWrkrData->libnet_handle)); } ip = libnet_build_ipv4( @@ -427,22 +441,22 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) tempaddr->sin_addr.s_addr, NULL, /* payload */ 0, /* payload size */ - pData->libnet_handle, /* libnet handle */ + pWrkrData->libnet_handle, /* libnet handle */ ip); /* libnet id */ if (ip == -1) { - DBGPRINTF("omudpspoof: can't build IP header: %s\n", libnet_geterror(pData->libnet_handle)); + DBGPRINTF("omudpspoof: can't build IP header: %s\n", libnet_geterror(pWrkrData->libnet_handle)); } /* Write it to the wire. */ - lsent = libnet_write(pData->libnet_handle); + lsent = libnet_write(pWrkrData->libnet_handle); if(lsent != (int) (LIBNET_IPV4_H+LIBNET_UDP_H+pktLen)) { /* note: access to fd is a libnet internal. If a newer version of libnet does * not expose that member, we should simply remove it. However, while it is there * it is useful for consolidating with strace output. */ DBGPRINTF("omudpspoof: write error (total len %d): pktLen %d, sent %d, fd %d: %s\n", - len, LIBNET_IPV4_H+LIBNET_UDP_H+pktLen, lsent, pData->libnet_handle->fd, - libnet_geterror(pData->libnet_handle)); + len, LIBNET_IPV4_H+LIBNET_UDP_H+pktLen, lsent, pWrkrData->libnet_handle->fd, + libnet_geterror(pWrkrData->libnet_handle)); if(lsent != -1) { bSendSuccess = RSTRUE; } @@ -452,7 +466,7 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) msgOffs += pktLen; /* We need to get rid of the UDP header to build the other fragments */ - libnet_clear_packet(pData->libnet_handle); + libnet_clear_packet(pWrkrData->libnet_handle); ip = LIBNET_PTAG_INITIALIZER; while(len > msgOffs ) { /* loop until all payload is sent */ /* check if there will be more fragments */ @@ -481,16 +495,16 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) tempaddr->sin_addr.s_addr, (uint8_t*)(msg+msgOffs), /* payload */ pktLen, /* payload size */ - pData->libnet_handle, /* libnet handle */ + pWrkrData->libnet_handle, /* libnet handle */ ip); /* libnet id */ if (ip == -1) { - DBGPRINTF("omudpspoof: can't build IP fragment header: %s\n", libnet_geterror(pData->libnet_handle)); + DBGPRINTF("omudpspoof: can't build IP fragment header: %s\n", libnet_geterror(pWrkrData->libnet_handle)); } /* Write it to the wire. */ - lsent = libnet_write(pData->libnet_handle); + lsent = libnet_write(pWrkrData->libnet_handle); if(lsent != (int) (LIBNET_IPV4_H+pktLen)) { DBGPRINTF("omudpspoof: fragment write error len %d, sent %d: %s\n", - LIBNET_IPV4_H+LIBNET_UDP_H+len, lsent, libnet_geterror(pData->libnet_handle)); + LIBNET_IPV4_H+LIBNET_UDP_H+len, lsent, libnet_geterror(pWrkrData->libnet_handle)); bSendSuccess = RSFALSE; continue; } @@ -500,9 +514,9 @@ UDPSend(instanceData *pData, uchar *pszSourcename, char *msg, size_t len) finalize_it: if(iRet != RS_RET_OK) { - if(pData->libnet_handle != NULL) { - libnet_destroy(pData->libnet_handle); - pData->libnet_handle = NULL; + if(pWrkrData->libnet_handle != NULL) { + libnet_destroy(pWrkrData->libnet_handle); + pWrkrData->libnet_handle = NULL; } } if(bNeedUnlock) { @@ -515,26 +529,28 @@ finalize_it: /* try to resume connection if it is not ready * rgerhards, 2007-08-02 */ -static rsRetVal doTryResume(instanceData *pData) +static rsRetVal doTryResume(wrkrInstanceData_t *pWrkrData) { int iErr; struct addrinfo *res; struct addrinfo hints; + instanceData *pData; DEFiRet; - if(pData->pSockArray != NULL) + if(pWrkrData->pSockArray != NULL) FINALIZE; + pData = pWrkrData->pData; - if(pData->libnet_handle == NULL) { + if(pWrkrData->libnet_handle == NULL) { /* Initialize the libnet library. Root priviledges are required. * this initializes a IPv4 socket to use for forging UDP packets. */ - pData->libnet_handle = libnet_init( + pWrkrData->libnet_handle = libnet_init( LIBNET_RAW4, /* injection type */ NULL, /* network interface */ - pData->errbuf); /* errbuf */ + pWrkrData->errbuf); /* errbuf */ - if(pData->libnet_handle == NULL) { + if(pWrkrData->libnet_handle == NULL) { if(pData->bReportLibnetInitErr) { errmsg.LogError(0, RS_RET_ERR_LIBNET_INIT, "omudpsoof: error " "initializing libnet - are you running as root?"); @@ -559,14 +575,14 @@ static rsRetVal doTryResume(instanceData *pData) ABORT_FINALIZE(RS_RET_SUSPENDED); } DBGPRINTF("%s found, resuming.\n", pData->host); - pData->f_addr = res; - pData->pSockArray = net.create_udp_socket((uchar*)pData->host, NULL, 0, 0); + pWrkrData->f_addr = res; + pWrkrData->pSockArray = net.create_udp_socket((uchar*)pData->host, NULL, 0, 0); finalize_it: if(iRet != RS_RET_OK) { - if(pData->f_addr != NULL) { - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; + if(pWrkrData->f_addr != NULL) { + freeaddrinfo(pWrkrData->f_addr); + pWrkrData->f_addr = NULL; } iRet = RS_RET_SUSPENDED; } @@ -577,7 +593,7 @@ finalize_it: BEGINtryResume CODESTARTtryResume - iRet = doTryResume(pData); + iRet = doTryResume(pWrkrData); ENDtryResume BEGINdoAction @@ -585,10 +601,10 @@ BEGINdoAction unsigned l; int iMaxLine; CODESTARTdoAction - CHKiRet(doTryResume(pData)); + CHKiRet(doTryResume(pWrkrData)); - DBGPRINTF(" %s:%s/omudpspoof, src '%s', msg strt '%.256s'\n", pData->host, - getFwdPt(pData), ppString[1], ppString[0]); + DBGPRINTF(" %s:%s/omudpspoof, src '%s', msg strt '%.256s'\n", pWrkrData->pData->host, + getFwdPt(pWrkrData->pData), ppString[1], ppString[0]); iMaxLine = glbl.GetMaxLine(); psz = (char*) ppString[0]; @@ -596,7 +612,7 @@ CODESTARTdoAction if((int) l > iMaxLine) l = iMaxLine; - CHKiRet(UDPSend(pData, ppString[1], psz, l)); + CHKiRet(UDPSend(pWrkrData, ppString[1], psz, l)); finalize_it: ENDdoAction @@ -660,7 +676,6 @@ CODESTARTnewActInst } } CODE_STD_STRING_REQUESTnewActInst(2) - pData->sourcePort = pData->sourcePortStart; tplToUse = ustrdup((pData->tplName == NULL) ? getDfltTpl() : pData->tplName); CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS)); @@ -699,7 +714,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(2) else CHKmalloc(pData->port = ustrdup(cs.pszTargetPort)); CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(sourceTpl), OMSR_NO_RQD_TPL_OPTS)); - pData->sourcePort = pData->sourcePortStart = cs.iSourcePortStart; + pData->sourcePortStart = cs.iSourcePortStart; pData->sourcePortEnd = cs.iSourcePortEnd; /* process template */ @@ -740,6 +755,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES diff --git a/plugins/omuxsock/omuxsock.c b/plugins/omuxsock/omuxsock.c index 583b9f94..da4e8e94 100644 --- a/plugins/omuxsock/omuxsock.c +++ b/plugins/omuxsock/omuxsock.c @@ -4,7 +4,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2010-2012 Adiscon GmbH. + * Copyright 2010-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -60,10 +60,14 @@ typedef struct _instanceData { permittedPeers_t *pPermPeers; uchar *sockName; int sock; - int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ struct sockaddr_un addr; } instanceData; + +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + /* config data */ typedef struct configSettings_s { uchar *tplName; /* name of the default template to use */ @@ -90,6 +94,7 @@ static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current l static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ +static pthread_mutex_t mutDoAct = PTHREAD_MUTEX_INITIALIZER; BEGINinitConfVars /* (re)set config variables to default values */ CODESTARTinitConfVars @@ -147,7 +152,6 @@ closeSocket(instanceData *pData) close(pData->sock); pData->sock = INVLD_SOCK; } -pData->bIsConnected = 0; // TODO: remove this variable altogether RETiRet; } @@ -224,6 +228,10 @@ CODESTARTcreateInstance pData->sock = INVLD_SOCK; ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature @@ -239,6 +247,10 @@ CODESTARTfreeInstance free(pData->sockName); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo @@ -332,7 +344,7 @@ static rsRetVal doTryResume(instanceData *pData) BEGINtryResume CODESTARTtryResume - iRet = doTryResume(pData); + iRet = doTryResume(pWrkrData->pData); ENDtryResume BEGINdoAction @@ -340,20 +352,22 @@ BEGINdoAction register unsigned l; int iMaxLine; CODESTARTdoAction - CHKiRet(doTryResume(pData)); + pthread_mutex_lock(&mutDoAct); + CHKiRet(doTryResume(pWrkrData->pData)); iMaxLine = glbl.GetMaxLine(); - DBGPRINTF(" omuxsock:%s\n", pData->sockName); + DBGPRINTF(" omuxsock:%s\n", pWrkrData->pData->sockName); psz = (char*) ppString[0]; l = strlen((char*) psz); if((int) l > iMaxLine) l = iMaxLine; - CHKiRet(sendMsg(pData, psz, l)); + CHKiRet(sendMsg(pWrkrData->pData, psz, l)); finalize_it: + pthread_mutex_unlock(&mutDoAct); ENDdoAction @@ -413,6 +427,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES ENDqueryEtryPt diff --git a/runtime/batch.h b/runtime/batch.h index 2ec07670..b0aa8574 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -46,17 +46,6 @@ typedef unsigned char batch_state_t; */ struct batch_obj_s { msg_t *pMsg; - /* work variables for action processing; these are reused for each action (or block of - * actions) - */ - sbool bPrevWasSuspended; - /* following are caches to save allocs if not absolutely necessary */ - uchar *staticActStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for strings */ - /* a cache to save malloc(), if not absolutely necessary */ - void *staticActParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for anything else */ - size_t staticLenStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; - /* and the same for the message length (if used) */ - /* end action work variables */ }; /* the batch @@ -77,11 +66,7 @@ struct batch_s { int maxElem; /* maximum number of elements that this batch supports */ int nElem; /* actual number of element in this entry */ int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ - int iDoneUpTo; /* all messages below this index have state other than RDY */ qDeqID deqID; /* ID of dequeue operation that generated this batch */ - int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */ - sbool *active; /* which messages are active for processing, NULL=all */ - sbool bSingleRuleset; /* do all msgs of this batch use a single ruleset? */ batch_obj_t *pElem; /* batch elements */ batch_state_t *eltState;/* state (array!) for individual objects. NOTE: we have moved this out of batch_obj_t because we @@ -93,24 +78,6 @@ struct batch_s { }; -/* some inline functions (we may move this off to an object .. or not) */ -static inline void -batchSetSingleRuleset(batch_t *pBatch, sbool val) { - pBatch->bSingleRuleset = val; -} - -/* get the batches ruleset (if we have a single ruleset) */ -static inline ruleset_t* -batchGetRuleset(batch_t *pBatch) { - return (pBatch->nElem > 0) ? pBatch->pElem[0].pMsg->pRuleset : NULL; -} - -/* get the ruleset of a specifc element of the batch (index not verified!) */ -static inline ruleset_t* -batchElemGetRuleset(batch_t *pBatch, int i) { - return pBatch->pElem[i].pMsg->pRuleset; -} - /* get number of msgs for this batch */ static inline int batchNumMsgs(batch_t *pBatch) { @@ -134,8 +101,7 @@ batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) { */ static inline int batchIsValidElem(batch_t *pBatch, int i) { - return( (pBatch->eltState[i] != BATCH_STATE_DISC) - && (pBatch->active == NULL || pBatch->active[i])); + return(pBatch->eltState[i] != BATCH_STATE_DISC); } @@ -152,7 +118,7 @@ batchFree(batch_t *pBatch) { /* staticActParams MUST be freed immediately (if required), * so we do not need to do that! */ - free(pBatch->pElem[i].staticActStrings[j]); + //TODO: do this in wti! free(pBatch->pElem[i].staticActStrings[j]); } } free(pBatch->pElem); @@ -167,11 +133,9 @@ batchFree(batch_t *pBatch) { static inline rsRetVal batchInit(batch_t *pBatch, int maxElem) { DEFiRet; - pBatch->iDoneUpTo = 0; pBatch->maxElem = maxElem; CHKmalloc(pBatch->pElem = calloc((size_t)maxElem, sizeof(batch_obj_t))); CHKmalloc(pBatch->eltState = calloc((size_t)maxElem, sizeof(batch_state_t))); - // TODO: replace calloc by inidividual writes? finalize_it: RETiRet; } diff --git a/runtime/conf.c b/runtime/conf.c index 2b000c60..83931bca 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -518,12 +518,10 @@ rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction) bHadWarning = 1; iRet = RS_RET_OK; } - if(iRet == RS_RET_OK || iRet == RS_RET_SUSPENDED) { - if((iRet = addAction(&pAction, pMod, pModData, pOMSR, NULL, NULL, - (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { + if(iRet == RS_RET_OK) { + if((iRet = addAction(&pAction, pMod, pModData, pOMSR, NULL, NULL)) == RS_RET_OK) { /* here check if the module is compatible with select features * (currently, we have no such features!) */ - pAction->eState = ACT_STATE_RDY; /* action is enabled */ conf->actions.nbrActions++; /* one more active action! */ } break; diff --git a/runtime/module-template.h b/runtime/module-template.h index 8a958f90..f65aaf0f 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -175,6 +175,44 @@ static rsRetVal freeInstance(void* pModData)\ RETiRet;\ } +/* createWrkrInstance() + */ +#define BEGINcreateWrkrInstance \ +static rsRetVal createWrkrInstance(wrkrInstanceData_t **ppWrkrData, instanceData *pData)\ + {\ + DEFiRet; /* store error code here */\ + wrkrInstanceData_t *pWrkrData; /* use this to point to data elements */ + +#define CODESTARTcreateWrkrInstance \ + if((pWrkrData = calloc(1, sizeof(wrkrInstanceData_t))) == NULL) {\ + *ppWrkrData = NULL;\ + ENDfunc \ + return RS_RET_OUT_OF_MEMORY;\ + } \ + pWrkrData->pData = pData; + +#define ENDcreateWrkrInstance \ + *ppWrkrData = pWrkrData;\ + RETiRet;\ +} + +/* freeWrkrInstance */ +#define BEGINfreeWrkrInstance \ +static rsRetVal freeWrkrInstance(void* pd)\ +{\ + DEFiRet;\ + wrkrInstanceData_t *pWrkrData; + +#define CODESTARTfreeWrkrInstance \ + pWrkrData = (wrkrInstanceData_t*) pd; + +#define ENDfreeWrkrInstance \ + if(pWrkrData != NULL)\ + free(pWrkrData); /* we need to free this in any case */\ + RETiRet;\ +} + + /* isCompatibleWithFeature() */ #define BEGINisCompatibleWithFeature \ @@ -194,7 +232,7 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF * introduced in v4.3.3 -- rgerhards, 2009-04-27 */ #define BEGINbeginTransaction \ -static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +static rsRetVal beginTransaction(wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ {\ DEFiRet; @@ -209,7 +247,7 @@ static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ * introduced in v4.3.3 -- rgerhards, 2009-04-27 */ #define BEGINendTransaction \ -static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +static rsRetVal endTransaction(wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ {\ DEFiRet; @@ -223,7 +261,7 @@ static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ /* doAction() */ #define BEGINdoAction \ -static rsRetVal doAction(uchar __attribute__((unused)) **ppString, unsigned __attribute__((unused)) iMsgOpts, instanceData __attribute__((unused)) *pData)\ +static rsRetVal doAction(uchar __attribute__((unused)) **ppString, unsigned __attribute__((unused)) iMsgOpts, wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ {\ DEFiRet; @@ -382,12 +420,12 @@ static rsRetVal newInpInst(struct nvlst *lst)\ * rgerhard, 2007-08-02 */ #define BEGINtryResume \ -static rsRetVal tryResume(instanceData __attribute__((unused)) *pData)\ +static rsRetVal tryResume(wrkrInstanceData_t __attribute__((unused)) *pWrkrData)\ {\ DEFiRet; #define CODESTARTtryResume \ - assert(pData != NULL); + assert(pWrkrData != NULL); #define ENDtryResume \ RETiRet;\ @@ -467,6 +505,13 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } +/* standard queries for output module interface in rsyslog v8+ */ +#define CODEqueryEtryPt_STD_OMOD8_QUERIES \ + else if(!strcmp((char*) name, "createWrkrInstance")) {\ + *pEtryPoint = createWrkrInstance;\ + } else if(!strcmp((char*) name, "freeWrkrInstance")) {\ + *pEtryPoint = freeWrkrInstance;\ + } /* the following definition is queryEtryPt block that must be added * if an output module supports the transactional interface. diff --git a/runtime/modules.c b/runtime/modules.c index 56606306..52096082 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -652,6 +652,9 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"doAction", &pNew->mod.om.doAction)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"createWrkrInstance", &pNew->mod.om.createWrkrInstance)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"freeWrkrInstance", &pNew->mod.om.freeWrkrInstance)); + /* try load optional interfaces */ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) diff --git a/runtime/modules.h b/runtime/modules.h index 23df22d6..a0637ac9 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -139,6 +139,8 @@ struct modInfo_s { rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); rsRetVal (*newActInst)(uchar *modName, struct nvlst *lst, void **, omodStringRequest_t **); rsRetVal (*SetShutdownImmdtPtr)(void *pData, void *pPtr); + rsRetVal (*createWrkrInstance)(void*ppWrkrData, void*pData); + rsRetVal (*freeWrkrInstance)(void*pWrkrData); } om; struct { /* data for library modules */ char dummy; diff --git a/runtime/queue.c b/runtime/queue.c index 8cbce011..ff1b30f4 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -86,7 +86,6 @@ static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiS static rsRetVal qAddDirect(qqueue_t *pThis, msg_t *pMsg); static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis); -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qDestructDisk(qqueue_t *pThis); rsRetVal qqueueSetSpoolDir(qqueue_t *pThis, uchar *pszSpoolDir, int lenSpoolDir); @@ -709,9 +708,13 @@ queueSwitchToEmergencyMode(qqueue_t *pThis, rsRetVal initiatingError) pThis->qType = QUEUETYPE_DIRECT; pThis->qConstruct = qConstructDirect; pThis->qDestruct = qDestructDirect; + /* these entry points shall not be used in direct mode + * To catch program errors, make us abort if that happens! + * rgerhards, 2013-11-05 + */ pThis->qAdd = qAddDirect; - pThis->qDel = qDelDirect; pThis->MultiEnq = qqueueMultiEnqObjDirect; + pThis->qDel = NULL; if(pThis->pqParent != NULL) { DBGOPRINT((obj_t*) pThis, "DA queue is in emergency mode, disabling DA in parent\n"); pThis->pqParent->bIsDA = 0; @@ -961,13 +964,11 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) return RS_RET_OK; } -static rsRetVal qAddDirect(qqueue_t *pThis, msg_t* pMsg) +static rsRetVal qAddDirectWithWti(qqueue_t *pThis, msg_t* pMsg, wti_t *pWti) { batch_t singleBatch; batch_obj_t batchObj; batch_state_t batchState = BATCH_STATE_RDY; - sbool active = 1; - int i; DEFiRet; //TODO: init batchObj (states _OK and new fields -- CHECK) @@ -987,46 +988,29 @@ static rsRetVal qAddDirect(qqueue_t *pThis, msg_t* pMsg) singleBatch.nElem = 1; /* there always is only one in direct mode */ singleBatch.pElem = &batchObj; singleBatch.eltState = &batchState; - singleBatch.active = &active; - iRet = pThis->pConsumer(pThis->pAction, &singleBatch, &pThis->bShutdownImmediate); - /* delete the batch string params: TODO: create its own "class" for this */ - for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) { - free(batchObj.staticActStrings[i]); - } + iRet = pThis->pConsumer(pThis->pAction, &singleBatch, pWti); msgDestruct(&pMsg); RETiRet; } -/* "enqueue" a batch in direct mode. This is a shortcut which saves all the overhead - * otherwise incured. -- rgerhards, ~2010-06-23 +/* this is called if we do not have a pWti. This currently only happens + * when we are called from a main queue in direct mode. If so, we need + * to obtain a dummy pWti. */ -rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch) +static rsRetVal +qAddDirect(qqueue_t *pThis, msg_t* pMsg) { + wti_t *pWti; DEFiRet; - ASSERT(pThis != NULL); - - /* calling the consumer is quite different here than it is from a worker thread */ - /* we need to provide the consumer's return value back to the caller because in direct - * mode the consumer probably has a lot to convey (which get's lost in the other modes - * because they are asynchronous. But direct mode is deliberately synchronous. - * rgerhards, 2008-02-12 - * We use our knowledge about the batch_t structure below, but without that, we - * pay a too-large performance toll... -- rgerhards, 2009-04-22 - */ - iRet = pThis->pConsumer(pThis->pAction, pBatch, NULL); - + pWti = wtiGetDummy(); + pWti->pbShutdownImmediate = &pThis->bShutdownImmediate; + iRet = qAddDirectWithWti(pThis, pMsg, pWti); RETiRet; } -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) -{ - return RS_RET_OK; -} - - /* --------------- end type-specific handlers -------------------- */ @@ -1321,7 +1305,7 @@ finalize_it: * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*, wti_t*)) { DEFiRet; qqueue_t *pThis; @@ -1700,6 +1684,9 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti) /* The rate limiter * + * IMPORTANT: the rate-limiter MUST unlock and re-lock the queue when + * it actually delays processing. Otherwise inputs are stalled. + * * Here we may wait if a dequeue time window is defined or if we are * rate-limited. TODO: If we do so, we should also look into the * way new worker threads are spawned. Obviously, it doesn't make much @@ -1785,8 +1772,10 @@ RateLimiter(qqueue_t *pThis) } if(iDelay > 0) { + pthread_mutex_unlock(pThis->mut); DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); srSleep(iDelay, 0); + pthread_mutex_lock(pThis->mut); } RETiRet; @@ -1880,7 +1869,8 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); - CHKiRet(pThis->pConsumer(pThis->pAction, &pWti->batch, &pThis->bShutdownImmediate)); + pWti->pbShutdownImmediate = &pThis->bShutdownImmediate; + CHKiRet(pThis->pConsumer(pThis->pAction, &pWti->batch, pWti)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 @@ -2103,9 +2093,13 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ case QUEUETYPE_DIRECT: pThis->qConstruct = qConstructDirect; pThis->qDestruct = qDestructDirect; + /* these entry points shall not be used in direct mode + * To catch program errors, make us abort if that happens! + * rgerhards, 2013-11-05 + */ pThis->qAdd = qAddDirect; - pThis->qDel = qDelDirect; pThis->MultiEnq = qqueueMultiEnqObjDirect; + pThis->qDel = NULL; break; } @@ -2169,9 +2163,12 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->iLowWtrMrk = 1; } } + if( pThis->iMinMsgsPerWrkr < 1 - || pThis->iMinMsgsPerWrkr > pThis->iMaxQueueSize ) + || pThis->iMinMsgsPerWrkr > pThis->iMaxQueueSize ) { pThis->iMinMsgsPerWrkr = pThis->iMaxQueueSize / pThis->iNumWorkerThreads; + } + if(pThis->iFullDlyMrk == -1 || pThis->iFullDlyMrk > pThis->iMaxQueueSize) { pThis->iFullDlyMrk = (pThis->iMaxQueueSize / 100) * 97; if(pThis->iFullDlyMrk == 0) { @@ -2777,13 +2774,14 @@ static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub) { int i; + wti_t *pWti; DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); - assert(pMultiSub != NULL); + pWti = wtiGetDummy(); + pWti->pbShutdownImmediate = &pThis->bShutdownImmediate; for(i = 0 ; i < pMultiSub->nElem ; ++i) { - CHKiRet(qAddDirect(pThis, (void*)pMultiSub->ppMsgs[i])); + CHKiRet(qAddDirectWithWti(pThis, (void*)pMultiSub->ppMsgs[i], pWti)); } finalize_it: @@ -2792,22 +2790,7 @@ finalize_it: /* ------------------------------ END multi-enqueue functions ------------------------------ */ -/* enqueue a new user data element in direct mode - * NOTE/TODO: This is a TESTER/EXPERIEMENTAL, to be changed to better - * code later on (like multi submit!) 2010-06-10 - * Enqueues the new element and awakes worker thread. - */ -rsRetVal -qqueueEnqMsgDirect(qqueue_t *pThis, msg_t *pMsg) -{ - DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); - iRet = qAddDirect(pThis, pMsg); - RETiRet; -} - - -/* enqueue a new user data element +/* enqueue a new user data element * Enqueues the new element and awakes worker thread. */ rsRetVal diff --git a/runtime/queue.h b/runtime/queue.h index 19ea735a..468e19bc 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -103,11 +103,10 @@ struct queue_s { * the user really wanted...). -- rgerhards, 2008-04-02 */ /* end dequeue time window */ - rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */ + rsRetVal (*pConsumer)(void *,batch_t*, wti_t*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 - * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero - * during normal operations and one if the consumer must urgently shut down. + * is pointer to an array of message message pointers) */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); @@ -195,14 +194,12 @@ struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); -rsRetVal qqueueEnqMsgDirect(qqueue_t *pThis, msg_t *pMsg); rsRetVal qqueueEnqMsg(qqueue_t *pThis, flowControl_t flwCtlType, msg_t *pMsg); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); -rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, wti_t *)); int queueCnfParamsSet(struct nvlst *lst); rsRetVal qqueueApplyCnfParam(qqueue_t *pThis, struct nvlst *lst); void qqueueSetDefaultsRulesetQueue(qqueue_t *pThis); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index b1016568..f2feb989 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -679,8 +679,12 @@ activateMainQueue() mainqCnfObj = glbl.GetmainqCnfObj(); DBGPRINTF("activateMainQueue: mainq cnf obj ptr is %p\n", mainqCnfObj); /* create message queue */ - CHKiRet_Hdlr(createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"), - (mainqCnfObj == NULL) ? NULL : mainqCnfObj->nvlst)) { + iRet = createMainQueue(&pMsgQueue, UCHAR_CONSTANT("main Q"), + (mainqCnfObj == NULL) ? NULL : mainqCnfObj->nvlst); + if(iRet == RS_RET_OK) { + iRet = startMainQueue(pMsgQueue); + } + if(iRet != RS_RET_OK) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); FINALIZE; @@ -743,6 +747,7 @@ activate(rsconf_t *cnf) tellModulesActivateConfig(); startInputModules(); CHKiRet(activateActions()); + CHKiRet(activateRulesetQueues()); CHKiRet(activateMainQueue()); /* finally let the inputs run... */ runInputModules(); @@ -1220,6 +1225,7 @@ ourConf = loadConf; // TODO: remove, once ourConf is gone! ABORT_FINALIZE(RS_RET_NO_ACTIONS); } tellLexEndParsing(); + DBGPRINTF("Number of actions in this configuration: %d\n", iActionNbr); rulesetOptimizeAll(loadConf); tellCoreConfigLoadDone(); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3c1e55d2..dfbfc448 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -49,6 +49,7 @@ #define CONF_PROGNAME_BUFSIZE 16 #define CONF_HOSTNAME_BUFSIZE 32 #define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ +#define CONF_IPARAMS_BUFSIZE 16 /* initial size of iparams array in wti (is automatically extended) */ #define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for * gain before we submit the message. But to do so we still need to diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 3c4d2eed..4bf8c057 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -48,6 +48,7 @@ #include "rainerscript.h" #include "srUtils.h" #include "modules.h" +#include "wti.h" #include "dirty.h" /* for main ruleset queue creation */ /* static data */ @@ -67,8 +68,8 @@ static struct cnfparamblk rspblk = }; /* forward definitions */ -static rsRetVal processBatch(batch_t *pBatch); -static rsRetVal scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active); +static rsRetVal processBatch(batch_t *pBatch, wti_t *pWti); +static rsRetVal scriptExec(struct cnfstmt *root, msg_t *pMsg, wti_t *pWti); /* ---------- linked-list key handling functions (ruleset) ---------- */ @@ -160,250 +161,130 @@ finalize_it: RETiRet; } - -/* This function is similar to processBatch(), but works on a batch that - * contains rules from multiple rulesets. In this case, we can not push - * the whole batch through the ruleset. Instead, we examine it and - * partition it into sub-rulesets which we then push through the system. - * rgerhards, 2010-06-15 - */ -static inline rsRetVal -processBatchMultiRuleset(batch_t *pBatch) +/* driver to iterate over all rulesets */ +DEFFUNC_llExecFunc(doActivateRulesetQueues) { - ruleset_t *currRuleset; - batch_t snglRuleBatch; - int i; - int iStart; /* start index of partial batch */ - int iNew; /* index for new (temporary) batch */ - int bHaveUnprocessed; /* do we (still) have unprocessed entries? (loop term predicate) */ DEFiRet; - - do { - bHaveUnprocessed = 0; - /* search for first unprocessed element */ - for(iStart = 0 ; iStart < pBatch->nElem && pBatch->eltState[iStart] == BATCH_STATE_DISC ; ++iStart) - /* just search, no action */; - if(iStart == pBatch->nElem) - break; /* everything processed */ - - /* prepare temporary batch */ - CHKiRet(batchInit(&snglRuleBatch, pBatch->nElem)); - snglRuleBatch.pbShutdownImmediate = pBatch->pbShutdownImmediate; - currRuleset = batchElemGetRuleset(pBatch, iStart); - iNew = 0; - for(i = iStart ; i < pBatch->nElem ; ++i) { - if(batchElemGetRuleset(pBatch, i) == currRuleset) { - /* for performance reasons, we copy only those members that we actually need */ - snglRuleBatch.pElem[iNew].pMsg = pBatch->pElem[i].pMsg; - snglRuleBatch.eltState[iNew] = pBatch->eltState[i]; - ++iNew; - /* We indicate the element also as done, so it will not be processed again */ - pBatch->eltState[i] = BATCH_STATE_DISC; - } else { - bHaveUnprocessed = 1; - } - } - snglRuleBatch.nElem = iNew; /* was left just right by the for loop */ - batchSetSingleRuleset(&snglRuleBatch, 1); - /* process temp batch */ - processBatch(&snglRuleBatch); - batchFree(&snglRuleBatch); - } while(bHaveUnprocessed == 1); - -finalize_it: + ruleset_t* pThis = (ruleset_t*) pData; + dbgprintf("Activating Ruleset Queue[%p] for Ruleset %s\n", + pThis->pQueue, pThis->pszName); + if(pThis->pQueue != NULL) + startMainQueue(pThis->pQueue); RETiRet; } - -/* return a new "active" structure for the batch. Free with freeActive(). */ -static inline sbool *newActive(batch_t *pBatch) +/* activate all ruleset queues */ +rsRetVal +activateRulesetQueues() { - return malloc(sizeof(sbool) * batchNumMsgs(pBatch)); - -} -static inline void freeActive(sbool *active) { free(active); } + DEFiRet; + llExecFunc(&(runConf->rulesets.llRulesets), doActivateRulesetQueues, NULL); -/* for details, see scriptExec() header comment! */ -/* call action for all messages with filter on */ -static rsRetVal -execAct(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) -{ - DEFiRet; -dbgprintf("RRRR: execAct [%s]: batch of %d elements, active %p\n", modGetName(stmt->d.act->pMod), batchNumMsgs(pBatch), active); - pBatch->active = active; - stmt->d.act->submitToActQ(stmt->d.act, pBatch); RETiRet; } + static rsRetVal -execSet(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +execAct(struct cnfstmt *stmt, msg_t *pMsg, wti_t *pWti) { - int i; - struct var result; DEFiRet; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if( pBatch->eltState[i] != BATCH_STATE_DISC - && (active == NULL || active[i])) { - cnfexprEval(stmt->d.s_set.expr, &result, pBatch->pElem[i].pMsg); - msgSetJSONFromVar(pBatch->pElem[i].pMsg, stmt->d.s_set.varname, - &result); - varDelete(&result); - } + if(stmt->d.act->bDisabled) { + DBGPRINTF("action %d died, do NOT execute\n", stmt->d.act->iActionNbr); + FINALIZE; + } + + DBGPRINTF("executing action %d\n", stmt->d.act->iActionNbr); + stmt->d.act->submitToActQ(stmt->d.act, pWti, pMsg); + if(iRet != RS_RET_DISCARDMSG) { + /* note: we ignore the error code here, as we do NEVER want to + * stop script execution due to action return code + */ + iRet = RS_RET_OK; } +finalize_it: RETiRet; } static rsRetVal -execUnset(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +execSet(struct cnfstmt *stmt, msg_t *pMsg) { - int i; + struct var result; DEFiRet; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if( pBatch->eltState[i] != BATCH_STATE_DISC - && (active == NULL || active[i])) { - msgDelJSON(pBatch->pElem[i].pMsg, stmt->d.s_unset.varname); - } - } + cnfexprEval(stmt->d.s_set.expr, &result, pMsg); + msgSetJSONFromVar(pMsg, stmt->d.s_set.varname, &result); + varDelete(&result); RETiRet; } -/* for details, see scriptExec() header comment! */ -/* "stop" simply discards the filtered items - it's just a (hopefully more intuitive - * shortcut for users. - */ static rsRetVal -execStop(batch_t *pBatch, sbool *active) +execUnset(struct cnfstmt *stmt, msg_t *pMsg) { - int i; DEFiRet; - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if( pBatch->eltState[i] != BATCH_STATE_DISC - && (active == NULL || active[i])) { - pBatch->eltState[i] = BATCH_STATE_DISC; - } - } + msgDelJSON(pMsg, stmt->d.s_unset.varname); RETiRet; } + static rsRetVal -execCall(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +execCall(struct cnfstmt *stmt, msg_t *pMsg, wti_t *pWti) { - msg_t *pMsg; - int i; DEFiRet; if(stmt->d.s_call.ruleset == NULL) { - scriptExec(stmt->d.s_call.stmt, pBatch, active); + CHKiRet(scriptExec(stmt->d.s_call.stmt, pMsg, pWti)); } else { - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - CHKmalloc(pMsg = MsgDup((msg_t*) pBatch->pElem[i].pMsg)); - DBGPRINTF("CALL: forwarding message %d to async ruleset %p\n", - i, stmt->d.s_call.ruleset->pQueue); - MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - MsgSetRuleset(pMsg, stmt->d.s_call.ruleset); - /* Note: we intentionally use submitMsg2() here, as we process messages - * that were already run through the rate-limiter. - */ - submitMsg2(pMsg); - } + CHKmalloc(pMsg = MsgDup((msg_t*) pMsg)); + DBGPRINTF("CALL: forwarding message to async ruleset %p\n", + stmt->d.s_call.ruleset->pQueue); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + MsgSetRuleset(pMsg, stmt->d.s_call.ruleset); + /* Note: we intentionally use submitMsg2() here, as we process messages + * that were already run through the rate-limiter. + */ + submitMsg2(pMsg); } finalize_it: RETiRet; } -/* for details, see scriptExec() header comment! */ -// save current filter, evaluate new one -// perform then (if any message) -// if ELSE given: -// set new filter, inverted -// perform else (if any messages) static rsRetVal -execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +execIf(struct cnfstmt *stmt, msg_t *pMsg, wti_t *pWti) { - sbool *newAct; - int i; sbool bRet; - sbool allInactive = 1; DEFiRet; - newAct = newActive(pBatch); - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(*(pBatch->pbShutdownImmediate)) - FINALIZE; - if(pBatch->eltState[i] == BATCH_STATE_DISC) - continue; /* will be ignored in any case */ - if(active == NULL || active[i]) { - bRet = cnfexprEvalBool(stmt->d.s_if.expr, pBatch->pElem[i].pMsg); - allInactive = 0; - } else - bRet = 0; - newAct[i] = bRet; - DBGPRINTF("batch: item %d: expr eval: %d\n", i, bRet); - } - - if(allInactive) { - DBGPRINTF("execIf: all batch elements are inactive, holding execution\n"); - freeActive(newAct); - FINALIZE; - } - - if(stmt->d.s_if.t_then != NULL) { - scriptExec(stmt->d.s_if.t_then, pBatch, newAct); - } - if(stmt->d.s_if.t_else != NULL) { - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(*(pBatch->pbShutdownImmediate)) - FINALIZE; - if(pBatch->eltState[i] != BATCH_STATE_DISC - && (active == NULL || active[i])) - newAct[i] = !newAct[i]; - } - scriptExec(stmt->d.s_if.t_else, pBatch, newAct); + bRet = cnfexprEvalBool(stmt->d.s_if.expr, pMsg); + DBGPRINTF("if condition result is %d\n", bRet); + if(bRet) { + if(stmt->d.s_if.t_then != NULL) + CHKiRet(scriptExec(stmt->d.s_if.t_then, pMsg, pWti)); + } else { + if(stmt->d.s_if.t_else != NULL) + CHKiRet(scriptExec(stmt->d.s_if.t_else, pMsg, pWti)); } - freeActive(newAct); finalize_it: RETiRet; } -/* for details, see scriptExec() header comment! */ -static void -execPRIFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +static rsRetVal +execPRIFILT(struct cnfstmt *stmt, msg_t *pMsg, wti_t *pWti) { - sbool *newAct; - msg_t *pMsg; int bRet; - int i; - newAct = newActive(pBatch); - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(*(pBatch->pbShutdownImmediate)) - return; - if(pBatch->eltState[i] == BATCH_STATE_DISC) - continue; /* will be ignored in any case */ - pMsg = pBatch->pElem[i].pMsg; - if(active == NULL || active[i]) { - if( (stmt->d.s_prifilt.pmask[pMsg->iFacility] == TABLE_NOPRI) || - ((stmt->d.s_prifilt.pmask[pMsg->iFacility] - & (1<<pMsg->iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else - bRet = 0; - newAct[i] = bRet; - DBGPRINTF("batch: item %d PRIFILT %d\n", i, newAct[i]); - } - - if(stmt->d.s_prifilt.t_then != NULL) { - scriptExec(stmt->d.s_prifilt.t_then, pBatch, newAct); - } - if(stmt->d.s_prifilt.t_else != NULL) { - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(*(pBatch->pbShutdownImmediate)) - return; - if(pBatch->eltState[i] != BATCH_STATE_DISC - && (active == NULL || active[i])) - newAct[i] = !newAct[i]; - } - scriptExec(stmt->d.s_prifilt.t_else, pBatch, newAct); + DEFiRet; + if( (stmt->d.s_prifilt.pmask[pMsg->iFacility] == TABLE_NOPRI) || + ((stmt->d.s_prifilt.pmask[pMsg->iFacility] + & (1<<pMsg->iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + + DBGPRINTF("PRIFILT condition result is %d\n", bRet); + if(bRet) { + if(stmt->d.s_prifilt.t_then != NULL) + CHKiRet(scriptExec(stmt->d.s_prifilt.t_then, pMsg, pWti)); + } else { + if(stmt->d.s_prifilt.t_else != NULL) + CHKiRet(scriptExec(stmt->d.s_prifilt.t_else, pMsg, pWti)); } - freeActive(newAct); +finalize_it: + RETiRet; } @@ -498,79 +379,67 @@ done: return bRet; } -/* for details, see scriptExec() header comment! */ -static void -execPROPFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +static rsRetVal +execPROPFILT(struct cnfstmt *stmt, msg_t *pMsg, wti_t *pWti) { - sbool *thenAct; sbool bRet; - int i; - thenAct = newActive(pBatch); - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(*(pBatch->pbShutdownImmediate)) - return; - if(pBatch->eltState[i] == BATCH_STATE_DISC) - continue; /* will be ignored in any case */ - if(active == NULL || active[i]) { - bRet = evalPROPFILT(stmt, pBatch->pElem[i].pMsg); - } else - bRet = 0; - thenAct[i] = bRet; - DBGPRINTF("batch: item %d PROPFILT %d\n", i, thenAct[i]); - } + DEFiRet; - scriptExec(stmt->d.s_propfilt.t_then, pBatch, thenAct); - freeActive(thenAct); + bRet = evalPROPFILT(stmt, pMsg); + DBGPRINTF("PROPFILT condition result is %d\n", bRet); + if(bRet) + CHKiRet(scriptExec(stmt->d.s_propfilt.t_then, pMsg, pWti)); +finalize_it: + RETiRet; } /* The rainerscript execution engine. It is debatable if that would be better * contained in grammer/rainerscript.c, HOWEVER, that file focusses primarily * on the parsing and object creation part. So as an actual executor, it is * better suited here. - * param active: if NULL, all messages are active (to be processed), if non-null - * this is an array of the same size as the batch. If 1, the message - * is to be processed, otherwise not. - * NOTE: this function must receive batches which contain a single ruleset ONLY! * rgerhards, 2012-09-04 */ static rsRetVal -scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active) +scriptExec(struct cnfstmt *root, msg_t *pMsg, wti_t *pWti) { - DEFiRet; struct cnfstmt *stmt; + DEFiRet; for(stmt = root ; stmt != NULL ; stmt = stmt->next) { + if(*pWti->pbShutdownImmediate) { + DBGPRINTF("scriptExec: ShutdownImmediate set, " + "force terminating\n"); + ABORT_FINALIZE(RS_RET_FORCE_TERM); + } if(Debug) { - dbgprintf("scriptExec: batch of %d elements, active %p, active[0]:%d\n", - batchNumMsgs(pBatch), active, (active == NULL ? 1 : active[0])); cnfstmtPrintOnly(stmt, 2, 0); } switch(stmt->nodetype) { case S_NOP: break; case S_STOP: - execStop(pBatch, active); + ABORT_FINALIZE(RS_RET_DISCARDMSG); break; case S_ACT: - execAct(stmt, pBatch, active); + CHKiRet(execAct(stmt, pMsg, pWti)); break; case S_SET: - execSet(stmt, pBatch, active); + CHKiRet(execSet(stmt, pMsg)); break; case S_UNSET: - execUnset(stmt, pBatch, active); + CHKiRet(execUnset(stmt, pMsg)); break; case S_CALL: - execCall(stmt, pBatch, active); + CHKiRet(execCall(stmt, pMsg, pWti)); break; case S_IF: - execIf(stmt, pBatch, active); + CHKiRet(execIf(stmt, pMsg, pWti)); break; case S_PRIFILT: - execPRIFILT(stmt, pBatch, active); + CHKiRet(execPRIFILT(stmt, pMsg, pWti)); break; case S_PROPFILT: - execPROPFILT(stmt, pBatch, active); + CHKiRet(execPROPFILT(stmt, pMsg, pWti)); break; default: dbgprintf("error: unknown stmt type %u during exec\n", @@ -578,36 +447,43 @@ scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active) break; } } +finalize_it: RETiRet; } /* Process (consume) a batch of messages. Calls the actions configured. - * If the whole batch uses a singel ruleset, we can process the batch as - * a whole. Otherwise, we need to process it slower, on a message-by-message - * basis (what can be optimized to a per-ruleset basis) - * rgerhards, 2005-10-13 + * This is called by MAIN queues. */ static rsRetVal -processBatch(batch_t *pBatch) +processBatch(batch_t *pBatch, wti_t *pWti) { - ruleset_t *pThis; + int i; + msg_t *pMsg; + ruleset_t *pRuleset; DEFiRet; - assert(pBatch != NULL); - - DBGPRINTF("processBatch: batch of %d elements must be processed\n", pBatch->nElem); - if(pBatch->bSingleRuleset) { - pThis = batchGetRuleset(pBatch); - if(pThis == NULL) - pThis = ourConf->rulesets.pDflt; - ISOBJ_TYPE_assert(pThis, ruleset); - CHKiRet(scriptExec(pThis->root, pBatch, NULL)); - } else { - CHKiRet(processBatchMultiRuleset(pBatch)); + + DBGPRINTF("processBATCH: batch of %d elements must be processed\n", pBatch->nElem); + + wtiResetExecState(pWti, pBatch); + + /* execution phase */ + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pWti->pbShutdownImmediate) ; ++i) { + pMsg = pBatch->pElem[i].pMsg; + DBGPRINTF("processBATCH: next msg %d: %.128s\n", i, pMsg->pszRawMsg); + pRuleset = (pMsg->pRuleset == NULL) ? ourConf->rulesets.pDflt : pMsg->pRuleset; + scriptExec(pRuleset->root, pMsg, pWti); + // TODO: think if we need a return state of scriptExec - most probably + // the answer is "no", as we need to process the batch in any case! + // TODO: we must refactor this! flag messages as committed + batchSetElemState(pBatch, i, BATCH_STATE_COMM); } -finalize_it: - DBGPRINTF("ruleset.ProcessMsg() returns %d\n", iRet); + /* commit phase */ + dbgprintf("END batch execution phase, entering to commit phase\n"); + actionCommitAllDirect(pWti); + + DBGPRINTF("processBATCH: batch of %d elements has been processed\n", pBatch->nElem); RETiRet; } diff --git a/runtime/ruleset.h b/runtime/ruleset.h index 9905b53c..d3dfd664 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -46,7 +46,7 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Destruct)(ruleset_t **ppThis); rsRetVal (*DestructAllActions)(rsconf_t *conf); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); - rsRetVal (*ProcessBatch)(batch_t*); + rsRetVal (*ProcessBatch)(batch_t*, wti_t *); rsRetVal (*GetRuleset)(rsconf_t *conf, ruleset_t **ppThis, uchar*); rsRetVal (*SetDefaultRuleset)(rsconf_t *conf, uchar*); rsRetVal (*SetCurrRuleset)(rsconf_t *conf, uchar*); @@ -64,8 +64,9 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ /* AddRule() removed */ /*TODO:REMOVE*/rsRetVal (*IterateAllActions)(rsconf_t *conf, rsRetVal (*pFunc)(void*, void*), void* pParam); void (*AddScript)(ruleset_t *pThis, struct cnfstmt *script); + /* v8: changed processBatch interface */ ENDinterface(ruleset) -#define rulesetCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ +#define rulesetCURR_IF_VERSION 8 /* increment whenever you change the interface structure! */ /* prototypes */ @@ -104,6 +105,7 @@ rulesetHasQueue(ruleset_t *pRuleset) rsRetVal rulesetGetRuleset(rsconf_t *conf, ruleset_t **ppRuleset, uchar *pszName); rsRetVal rulesetOptimizeAll(rsconf_t *conf); rsRetVal rulesetProcessCnf(struct cnfobj *o); +rsRetVal activateRulesetQueues(void); /* Set a current rule set to already-known pointer */ static inline void diff --git a/runtime/wti.c b/runtime/wti.c index 77197a95..c02d0573 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -44,12 +44,15 @@ #include "wti.h" #include "obj.h" #include "glbl.h" +#include "action.h" #include "atomic.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +pthread_key_t thrd_wti_key; + /* forward-definitions */ /* methods */ @@ -171,6 +174,7 @@ BEGINobjDestruct(wti) /* be sure to specify the object type also in END and CODE CODESTARTobjDestruct(wti) /* actual destruction */ batchFree(&pThis->batch); + free(pThis->actWrkrInfo); pthread_cond_destroy(&pThis->pcondBusy); DESTROY_ATOMIC_HELPER_MUT(pThis->mutIsRunning); @@ -197,11 +201,20 @@ wtiConstructFinalize(wti_t *pThis) ISOBJ_TYPE_assert(pThis, wti); - DBGPRINTF("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: finalizing construction of worker instance data (for %d actions)\n", + wtiGetDbgHdr(pThis), iActionNbr); /* initialize our thread instance descriptor (no concurrency here) */ pThis->bIsRunning = RSFALSE; + /* must use calloc as we need zero-init */ + CHKmalloc(pThis->actWrkrInfo = calloc(iActionNbr, sizeof(actWrkrInfo_t))); + + if(pThis->pWtp == NULL) { + dbgprintf("wtiConstructFinalize: pWtp not set, this may be intentional\n"); + FINALIZE; + } + /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); CHKiRet(batchInit(&pThis->batch, iDeqBatchSize)); @@ -279,6 +292,7 @@ wtiWorker(wti_t *pThis) rsRetVal localRet; rsRetVal terminateRet; int iCancelStateSave; + int i; DEFiRet; ISOBJ_TYPE_assert(pThis, wti); @@ -288,15 +302,27 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - +dbgprintf("DDDD: wti %p: worker starting\n", pThis); /* now we have our identity, on to real processing */ - while(1) { /* loop will be broken below - need to do mutex locks */ + + /* note: in this loop, the mutex is "never" unlocked. Of course, + * this is not true: it actually is unlocked when the actual processing + * is done, as part of pWtp->pfDoWork() processing. Note that this + * function is required to re-lock it when done. We cannot do the + * lock/unlock here ourselfs, as pfDoWork() needs to access queue + * structures itself. + * The same goes for pfRateLimiter(). While we could unlock/lock when + * we call it, in practice the function is often called without any + * ratelimiting actually done. Only the rate limiter itself knows + * that. As such, it needs to bear the burden of doing the locking + * when required. -- rgerhards, 2013-11-20 + */ + d_pthread_mutex_lock(pWtp->pmutUsr); + while(1) { /* loop will be broken below */ if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } - d_pthread_mutex_lock(pWtp->pmutUsr); - /* first check if we are in shutdown process (but evaluate a bit later) */ terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); if(terminateRet == RS_RET_TERMINATE_NOW) { @@ -304,39 +330,47 @@ wtiWorker(wti_t *pThis) localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); DBGOPRINT((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", localRet); - d_pthread_mutex_unlock(pWtp->pmutUsr); break; } /* try to execute and process whatever we have */ - /* Note that this function releases and re-aquires the mutex. The returned - * information on idle state must be processed before releasing the mutex again. - */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); if(localRet == RS_RET_ERR_QUEUE_EMERGENCY) { - d_pthread_mutex_unlock(pWtp->pmutUsr); break; /* end of loop */ } else if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { - d_pthread_mutex_unlock(pWtp->pmutUsr); DBGOPRINT((obj_t*) pThis, "terminating worker terminateRet=%d, bInactivityTOOccured=%d\n", terminateRet, bInactivityTOOccured); break; /* end of loop */ } doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); - d_pthread_mutex_unlock(pWtp->pmutUsr); continue; /* request next iteration */ } - d_pthread_mutex_unlock(pWtp->pmutUsr); - bInactivityTOOccured = 0; /* reset for next run */ } + d_pthread_mutex_unlock(pWtp->pmutUsr); + + DBGPRINTF("DDDD: wti %p: worker cleanup up action instances\n", pThis); + for(i = 0 ; i < iActionNbr ; ++i) { + dbgprintf("wti %p, action %d, ptr %p\n", pThis, i, pThis->actWrkrInfo[i].actWrkrData); + if(pThis->actWrkrInfo[i].actWrkrData != NULL) { + dbgprintf("DDDD: calling freeWrkrData!\n"); + pThis->actWrkrInfo[i].pAction->pMod->mod.om.freeWrkrInstance(pThis->actWrkrInfo[i].actWrkrData); + free(pThis->actWrkrInfo[i].iparams); + pThis->actWrkrInfo[i].actWrkrData = NULL; /* re-init for next activation */ + pThis->actWrkrInfo[i].iparams = NULL; + pThis->actWrkrInfo[i].currIParam = 0; + pThis->actWrkrInfo[i].maxIParams = 0; + } + } + /* indicate termination */ pthread_cleanup_pop(0); /* remove cleanup handler */ pthread_setcancelstate(iCancelStateSave, NULL); +dbgprintf("DDDD: wti %p: worker exiting\n", pThis); RETiRet; } @@ -376,6 +410,33 @@ finalize_it: } +/* This function returns (and creates if necessary) a dummy wti suitable + * for use by the rule engine. It is intended to be used for direct-mode + * main queues (folks, don't do that!). Once created, data is stored in + * thread-specific storage. + * Note: we do NOT do error checking -- if this functions fails, all the + * rest will fail as well... (also, it will only fail under OOM, so...). + * Memleak: we leak pWti's when run in direct mode. However, this is only + * a cosmetic leak, as we need them until all inputs are terminated, + * what means essentially until rsyslog itself is terminated. So we + * don't care -- it's just not nice in valgrind, but that's it. + */ +wti_t * +wtiGetDummy(void) +{ + wti_t *pWti; + + pWti = (wti_t*) pthread_getspecific(thrd_wti_key); + if(pWti == NULL) { + wtiConstruct(&pWti); + wtiConstructFinalize(pWti); + if(pthread_setspecific(thrd_wti_key, pWti) != 0) { + DBGPRINTF("wtiGetDummy: error setspecific thrd_wti_key\n"); + } + } + return pWti; +} + /* dummy */ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } @@ -385,6 +446,7 @@ BEGINObjClassExit(wti, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ CODESTARTObjClassExit(nsdsel_gtls) /* release objects we no longer need */ objRelease(glbl, CORE_COMPONENT); + pthread_key_delete(thrd_wti_key); ENDObjClassExit(wti) @@ -393,8 +455,14 @@ ENDObjClassExit(wti) * rgerhards, 2008-01-09 */ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most important for persisting) */ + int r; /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + r = pthread_key_create(&thrd_wti_key, NULL); + if(r != 0) { + dbgprintf("wti.c: pthread_key_create failed\n"); + iRet = RS_RET_ERR; + } ENDObjClassInit(wti) /* vi:set ai: diff --git a/runtime/wti.h b/runtime/wti.h index b0dc6c96..adc7897c 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -1,6 +1,6 @@ /* Definition of the worker thread instance (wti) class. * - * Copyright 2008-2012 Adiscon GmbH. + * Copyright 2008-2013 Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -26,19 +26,67 @@ #include "wtp.h" #include "obj.h" #include "batch.h" +#include "action.h" +#define ACT_STATE_RDY 0 /* action ready, waiting for new transaction */ +#define ACT_STATE_ITX 1 /* transaction active, waiting for new data or commit */ +#define ACT_STATE_COMM 2 /* transaction finished (a transient state) */ +#define ACT_STATE_RTRY 3 /* failure occured, trying to restablish ready state */ +#define ACT_STATE_SUSP 4 /* suspended due to failure (return fail until timeout expired) */ +/* note: 3 bit bit field --> highest value is 7! */ + +/* The following structure defines immutable parameters which need to + * be passed as action parameters. Note that the current implementation + * does NOT focus on performance, but on a simple PoC in order to get + * things going. TODO: Once it works, revisit this code and think about + * an array implementation. We also need to support other passing modes + * as well. -- gerhards, 2013-11-04 + */ +typedef struct actWrkrIParams { + int msgFlags; + /* following are caches to save allocs if not absolutely necessary */ + uchar *staticActStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for strings */ + /* a cache to save malloc(), if not absolutely necessary */ + unsigned staticLenStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; + /* and the same for the message length (if used) */ + void *staticActParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; +} actWrkrIParams_t; + +typedef struct actWrkrInfo { + action_t *pAction; + void *actWrkrData; + uint16_t uResumeOKinRow;/* number of times in a row that resume said OK with an immediate failure following */ + int iNbrResRtry; /* number of retries since last suspend */ + struct { + unsigned actState : 3; + } flags; + actWrkrIParams_t *iparams; + int currIParam; + int maxIParams; /* current max */ + void *staticActParams[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /* for non-strings */ +} actWrkrInfo_t; + /* the worker thread instance class */ struct wti_s { BEGINobjInstance; pthread_t thrdID; /* thread ID */ int bIsRunning; /* is this thread currently running? (must be int for atomic op!) */ sbool bAlwaysRunning; /* should this thread always run? */ + int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ + actWrkrInfo_t *actWrkrInfo; /* *array* of action wrkr infos for all actions (sized for max nbr of actions in config!) */ pthread_cond_t pcondBusy; /* condition to wake up the worker, protected by pmutUsr in wtp */ DEF_ATOMIC_HELPER_MUT(mutIsRunning); + struct { + uint8_t bPrevWasSuspended; + uint8_t bDoAutoCommit; /* do a commit after each message + * this is usually set for batches with 0 element, but may + * also be added as a user-selectable option (not implemented yet) + */ + } execState; /* state for the execution engine */ }; @@ -53,8 +101,96 @@ rsRetVal wtiSetAlwaysRunning(wti_t *pThis); rsRetVal wtiSetState(wti_t *pThis, sbool bNew); rsRetVal wtiWakeupThrd(wti_t *pThis); sbool wtiGetState(wti_t *pThis); +wti_t *wtiGetDummy(void); PROTOTYPEObjClassInit(wti); PROTOTYPEpropSetMeth(wti, pszDbgHdr, uchar*); PROTOTYPEpropSetMeth(wti, pWtp, wtp_t*); +static inline uint8_t +getActionStateByNbr(wti_t *pWti, int iActNbr) +{ + return((uint8_t) pWti->actWrkrInfo[iActNbr].flags.actState); +} + +static inline uint8_t +getActionState(wti_t *pWti, action_t *pAction) +{ + return((uint8_t) pWti->actWrkrInfo[pAction->iActionNbr].flags.actState); +} + +static inline void +setActionState(wti_t *pWti, action_t *pAction, uint8_t newState) +{ + pWti->actWrkrInfo[pAction->iActionNbr].flags.actState = newState; +} + +static inline uint16_t +getActionResumeInRow(wti_t *pWti, action_t *pAction) +{ + return(pWti->actWrkrInfo[pAction->iActionNbr].uResumeOKinRow); +} + +static inline void +setActionResumeInRow(wti_t *pWti, action_t *pAction, uint16_t val) +{ + pWti->actWrkrInfo[pAction->iActionNbr].uResumeOKinRow = val; +} + +static inline void +incActionResumeInRow(wti_t *pWti, action_t *pAction) +{ + pWti->actWrkrInfo[pAction->iActionNbr].uResumeOKinRow++; +} + +static inline int +getActionNbrResRtry(wti_t *pWti, action_t *pAction) +{ + return(pWti->actWrkrInfo[pAction->iActionNbr].iNbrResRtry); +} + +static inline void +setActionNbrResRtry(wti_t *pWti, action_t *pAction, uint16_t val) +{ + pWti->actWrkrInfo[pAction->iActionNbr].iNbrResRtry = val; +} + +static inline void +incActionNbrResRtry(wti_t *pWti, action_t *pAction) +{ + pWti->actWrkrInfo[pAction->iActionNbr].iNbrResRtry++; +} + +static inline rsRetVal +wtiNewIParam(wti_t *pWti, action_t *pAction, actWrkrIParams_t **piparams) +{ + actWrkrInfo_t *wrkrInfo; + actWrkrIParams_t *iparams; + int newMax; + DEFiRet; + + wrkrInfo = &(pWti->actWrkrInfo[pAction->iActionNbr]); + if(wrkrInfo->currIParam == wrkrInfo->maxIParams) { + /* we need to extend */ + dbgprintf("DDDD: extending iparams, max %d\n", wrkrInfo->maxIParams); + newMax = (wrkrInfo->maxIParams == 0) ? CONF_IPARAMS_BUFSIZE : 2 * wrkrInfo->maxIParams; + CHKmalloc(iparams = realloc(wrkrInfo->iparams, sizeof(actWrkrIParams_t) * newMax)); + wrkrInfo->iparams = iparams; + wrkrInfo->maxIParams = newMax; + } +dbgprintf("DDDD: adding param %d for action %d\n", wrkrInfo->currIParam, pAction->iActionNbr); + iparams = wrkrInfo->iparams + wrkrInfo->currIParam; + memset(iparams, 0, sizeof(actWrkrIParams_t)); + *piparams = iparams; + ++wrkrInfo->currIParam; + +finalize_it: + RETiRet; +} + +static inline void +wtiResetExecState(wti_t *pWti, batch_t *pBatch) +{ + pWti->execState.bPrevWasSuspended = 0; + pWti->execState.bDoAutoCommit = (batchNumMsgs(pBatch) == 1); +} #endif /* #ifndef WTI_H_INCLUDED */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 895c1ffe..66942e6b 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -8,7 +8,7 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -91,6 +91,7 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pthread_cond_init(&pThis->condThrdTrm, NULL); pthread_attr_init(&pThis->attrThrd); /* Set thread scheduling policy to default */ +#warning do we need this any longer? I think it was a cure for an already fixed bug.. #ifdef HAVE_PTHREAD_SETSCHEDPARAM pthread_attr_setschedpolicy(&pThis->attrThrd, default_thr_sched_policy); pthread_attr_setschedparam(&pThis->attrThrd, &default_sched_param); @@ -121,7 +122,8 @@ wtpConstructFinalize(wtp_t *pThis) ISOBJ_TYPE_assert(pThis, wtp); - DBGPRINTF("%s: finalizing construction of worker thread pool\n", wtpGetDbgHdr(pThis)); + DBGPRINTF("%s: finalizing construction of worker thread pool (numworkerThreads %d)\n", + wtpGetDbgHdr(pThis), pThis->iNumWorkerThreads); /* alloc and construct workers - this can only be done in finalizer as we previously do * not know the max number of workers */ diff --git a/tests/Makefile.am b/tests/Makefile.am index 4dff5648..5232e3ef 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,6 +10,7 @@ TESTS += \ stop-msgvar.sh \ rfc5424parser.sh \ arrayqueue.sh \ + global_vars.sh \ da-mainmsg-q.sh \ validation-run.sh \ imtcp-multiport.sh \ @@ -17,6 +18,7 @@ TESTS += \ diskqueue.sh \ diskqueue-fsync.sh \ rulesetmultiqueue.sh \ + rulesetmultiqueue-v6.sh \ manytcp.sh \ rsf_getenv.sh \ imtcp_conndrop.sh \ @@ -60,6 +62,7 @@ TESTS += \ discard-rptdmsg.sh \ discard-allmark.sh \ discard.sh \ + stop.sh \ failover-async.sh \ failover-double.sh \ failover-basic.sh \ @@ -298,10 +301,14 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rscript_stop.conf \ rscript_stop2.sh \ testsuites/rscript_stop2.conf \ + stop.sh \ + testsuites/stop.conf \ stop-localvar.sh \ testsuites/stop-localvar.conf \ stop-msgvar.sh \ testsuites/stop-msgvar.conf \ + global_vars.sh \ + testsuites/global_vars.conf \ rfc5424parser.sh \ testsuites/rfc5424parser.conf \ rs_optimizer_pri.sh \ @@ -477,6 +484,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/master.nolimittag \ rulesetmultiqueue.sh \ testsuites/rulesetmultiqueue.conf \ + rulesetmultiqueue-v6.sh \ + testsuites/rulesetmultiqueue-v6.conf \ omruleset.sh \ testsuites/omruleset.conf \ omruleset-queue.sh \ diff --git a/tests/diag.sh b/tests/diag.sh index 7883d93b..71d62111 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -142,7 +142,7 @@ case $1 in 'seq-check') # do the usual sequence check to see if everything was properly received. $2 is the instance. rm -f work cp rsyslog.out.log work-presort - sort < rsyslog.out.log > work + sort -g < rsyslog.out.log > work # $4... are just to have the abilit to pass in more options... # add -v to chkseq if you need more verbose output ./chkseq -fwork -s$2 -e$3 $4 $5 $6 $7 @@ -155,7 +155,7 @@ case $1 in # a duplicateof seq-check, but we could not change its calling conventions without # breaking a lot of exitings test cases, so we preferred to duplicate the code here. rm -f work2 - sort < rsyslog2.out.log > work2 + sort -g < rsyslog2.out.log > work2 # $4... are just to have the abilit to pass in more options... # add -v to chkseq if you need more verbose output ./chkseq -fwork2 -s$2 -e$3 $4 $5 $6 $7 @@ -168,7 +168,7 @@ case $1 in 'gzip-seq-check') # do the usual sequence check, but for gzip files rm -f work ls -l rsyslog.out.log - gunzip < rsyslog.out.log | sort > work + gunzip < rsyslog.out.log | sort -g > work ls -l work # $4... are just to have the abilit to pass in more options... ./chkseq -fwork -v -s$2 -e$3 $4 $5 $6 $7 diff --git a/tests/global_vars.sh b/tests/global_vars.sh new file mode 100755 index 00000000..5c6c579e --- /dev/null +++ b/tests/global_vars.sh @@ -0,0 +1,15 @@ +# Test for global variables +# added 2013-11-18 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[global_vars.sh\]: testing global variable support +source $srcdir/diag.sh init +source $srcdir/diag.sh startup global_vars.conf + +# 40000 messages should be enough +source $srcdir/diag.sh injectmsg 0 40000 + +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/rulesetmultiqueue-v6.sh b/tests/rulesetmultiqueue-v6.sh new file mode 100755 index 00000000..21166fe1 --- /dev/null +++ b/tests/rulesetmultiqueue-v6.sh @@ -0,0 +1,33 @@ +# Test for disk-only queue mode with v6+ config +# This tests defines three rulesets, each one with its own queue. Then, it +# sends data to them and checks the outcome. Note that we do need to +# use some custom code as the test driver framework does not (yet?) +# support multi-output-file operations. +# added 2013-11-14 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo =============================================================================== +echo \[rulesetmultiqueu.sh\]: testing multiple queues via rulesets +source $srcdir/diag.sh init +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh startup rulesetmultiqueue-v6.conf +source $srcdir/diag.sh wait-startup +# now fill the three files (a bit sequentially, but they should +# still get their share of concurrency - to increase the chance +# we use three connections per set). +source $srcdir/diag.sh tcpflood -c3 -p13514 -m20000 -i0 +source $srcdir/diag.sh tcpflood -c3 -p13515 -m20000 -i20000 +source $srcdir/diag.sh tcpflood -c3 -p13516 -m20000 -i40000 + +# in this version of the imdiag, we do not have the capability to poll +# all queues for emptyness. So we do a sleep in the hopes that this will +# sufficiently drain the queues. This is race, but the best we currently +# can do... - rgerhards, 2009-11-05 +sleep 2 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +# now consolidate all logs into a single one so that we can use the +# regular check logic +cat rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log > rsyslog.out.log +source $srcdir/diag.sh seq-check 0 59999 +rm -f rsyslog.out1.log rsyslog.out2.log rsyslog.out3.log +source $srcdir/diag.sh exit diff --git a/tests/stop.sh b/tests/stop.sh new file mode 100755 index 00000000..f3dcf993 --- /dev/null +++ b/tests/stop.sh @@ -0,0 +1,12 @@ +# Test for "stop" statement +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[stop.sh\]: testing stop statement +source $srcdir/diag.sh init +source $srcdir/diag.sh startup stop.conf +sleep 1 +source $srcdir/diag.sh tcpflood -m10 -i1 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 2 10 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/global_vars.conf b/tests/testsuites/global_vars.conf new file mode 100644 index 00000000..0d1a3cb8 --- /dev/null +++ b/tests/testsuites/global_vars.conf @@ -0,0 +1,17 @@ +$IncludeConfig diag-common.conf + +$MainMsgQueueTimeoutShutdown 10000 + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +template(name="outfmt" type="string" string="%$/msgnum%\n") +template(name="dynfile" type="string" string="rsyslog.out.log") /* trick to use relative path names! */ + +if $/msgnum == "" then + set $/msgnum = 0; + +if $msg contains "msgnum:" then { + action(type="omfile" dynaFile="dynfile" template="outfmt") + set $/msgnum = $/msgnum + 1; +} diff --git a/tests/testsuites/rulesetmultiqueue-v6.conf b/tests/testsuites/rulesetmultiqueue-v6.conf new file mode 100644 index 00000000..3aeaa337 --- /dev/null +++ b/tests/testsuites/rulesetmultiqueue-v6.conf @@ -0,0 +1,34 @@ +# Test for multiple ruleset queues (see .sh file for details) +# rgerhards, 2009-10-30 +$IncludeConfig diag-common.conf +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 + +# general definition +$template outfmt,"%msg:F,58:2%\n" + +# create the individual rulesets +$template dynfile1,"rsyslog.out1.log" # trick to use relative path names! +ruleset(name="file1" queue.type="linkedList") { + :msg, contains, "msgnum:" ?dynfile1;outfmt +} + +$template dynfile2,"rsyslog.out2.log" # trick to use relative path names! +ruleset(name="file2" queue.type="linkedList") { + :msg, contains, "msgnum:" ?dynfile2;outfmt +} + +$template dynfile3,"rsyslog.out3.log" # trick to use relative path names! +ruleset(name="file3" queue.type="linkedList") { + :msg, contains, "msgnum:" ?dynfile3;outfmt +} + +# start listeners and bind them to rulesets +$InputTCPServerBindRuleset file1 +$InputTCPServerRun 13514 + +$InputTCPServerBindRuleset file2 +$InputTCPServerRun 13515 + +$InputTCPServerBindRuleset file3 +$InputTCPServerRun 13516 diff --git a/tests/testsuites/stop.conf b/tests/testsuites/stop.conf new file mode 100644 index 00000000..84beab82 --- /dev/null +++ b/tests/testsuites/stop.conf @@ -0,0 +1,11 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") +input(type="imtcp" port="13514") + +if $msg contains "00000001" then + stop + +template(name="outfmt" type="string" string="%msg:F,58:2%\n") +if $msg contains "msgnum:" then + action(type="omfile" file="rsyslog.out.log" template="outfmt") diff --git a/tools/omdiscard.c b/tools/omdiscard.c index 15c6ea82..a76bcc33 100644 --- a/tools/omdiscard.c +++ b/tools/omdiscard.c @@ -6,7 +6,7 @@ * * File begun on 2007-07-24 by RGerhards * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -49,6 +49,10 @@ typedef struct _instanceData { EMPTY_STRUCT } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + /* we do not need a createInstance()! BEGINcreateInstance CODESTARTcreateInstance @@ -56,6 +60,11 @@ ENDcreateInstance */ +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo /* do nothing */ @@ -87,6 +96,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINparseSelectorAct CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(0) @@ -114,6 +128,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES ENDqueryEtryPt diff --git a/tools/omfile.c b/tools/omfile.c index fdcf355a..b1cbbfd9 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -133,6 +133,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; typedef struct _instanceData { + pthread_mutex_t mutWrite; /* guard against multiple instances writing to single file */ uchar *f_fname; /* file or template name (display only) */ uchar *tplName; /* name of assigned template */ strm_t *pStrm; /* our output stream */ @@ -181,6 +182,20 @@ typedef struct _instanceData { STATSCOUNTER_DEF(ctrMax, mutCtrMax); } instanceData; +/* to build a linked list for temporary storage of lines while we cannot commit */ +typedef struct linebuf { + uchar *filename; /* for dynafiles, make go away */ + uchar *ln; + unsigned iMsgOpts; + struct linebuf *pNext; +} linebuf_t; + +typedef struct wrkrInstanceData { + instanceData *pData; + linebuf_t *pRoot; + linebuf_t *pLast; +} wrkrInstanceData_t; + typedef struct configSettings_s { int iDynaFileCacheSize; /* max cache for dynamic files */ @@ -786,7 +801,7 @@ finalize_it: /* rgerhards 2004-11-11: write to a file output. */ static rsRetVal -writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +writeFile(instanceData *pData, linebuf_t *linebuf) { DEFiRet; @@ -796,7 +811,7 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) * check if it still is ok or a new file needs to be created */ if(pData->bDynamicName) { - CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts)); + CHKiRet(prepareDynFile(pData, linebuf->filename, linebuf->iMsgOpts)); } else { /* "regular", non-dynafile */ if(pData->pStrm == NULL) { CHKiRet(prepareFile(pData, pData->f_fname)); @@ -806,7 +821,7 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) } } - CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); + CHKiRet(doWrite(pData, linebuf->ln, ustrlen(linebuf->ln))); finalize_it: RETiRet; @@ -888,9 +903,15 @@ ENDfreeCnf BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; + pthread_mutex_init(&pData->mutWrite, NULL); ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINfreeInstance CODESTARTfreeInstance free(pData->tplName); @@ -913,9 +934,15 @@ CODESTARTfreeInstance free(pData->cryprovName); free(pData->cryprovNameFull); } + pthread_mutex_destroy(&pData->mutWrite); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINtryResume CODESTARTtryResume ENDtryResume @@ -926,8 +953,68 @@ CODESTARTbeginTransaction ENDbeginTransaction +static rsRetVal +bufferLine(wrkrInstanceData_t *pWrkrData, uchar *filename, uchar *line) +{ + linebuf_t *lb; + DEFiRet; + + CHKmalloc(lb = (linebuf_t*) malloc(sizeof(linebuf_t))); +dbgprintf("DDDD: filename '%s'\n", filename); + CHKmalloc(lb->filename = ustrdup(filename)); + CHKmalloc(lb->ln = ustrdup(line)); + lb->pNext = NULL; + if(pWrkrData->pRoot == NULL) { + pWrkrData->pRoot = pWrkrData->pLast = lb; + } else { + pWrkrData->pLast->pNext = lb; + pWrkrData->pLast = lb; + } +finalize_it: + RETiRet; +} + +static void +submitCachedLines(wrkrInstanceData_t *pWrkrData, instanceData *pData) +{ + linebuf_t *curr, *todel; + + for(curr = pWrkrData->pRoot ; curr != NULL ; ) { + DBGPRINTF("omfile: file to log to: %s\n", curr->filename); + DBGPRINTF("omfile: start of data: '%.128s'\n", curr->ln); + STATSCOUNTER_INC(pData->ctrRequests, pData->mutCtrRequests); + writeFile(pData, curr); + + todel = curr; + curr = curr->pNext; + free(todel->filename); + free(todel->ln); + free(todel); + } + pWrkrData->pRoot = NULL; +} + + +BEGINdoAction + instanceData *pData; +CODESTARTdoAction + pData = pWrkrData->pData; +dbgprintf("DDDD: bDynName %d, filename '%s'\n", pData->bDynamicName, ppString[1]); + iRet = bufferLine(pWrkrData, (pData->bDynamicName) ? ppString[1] : pData->f_fname, + ppString[0]); + if(iRet == RS_RET_OK) + iRet = RS_RET_DEFER_COMMIT; +ENDdoAction + BEGINendTransaction + instanceData *pData; CODESTARTendTransaction + pData = pWrkrData->pData; +dbgprintf("omfile: waiting on write lock (pWrkrData %p)\n", pWrkrData); + pthread_mutex_lock(&pData->mutWrite); +dbgprintf("omfile: aquired write lock (pWrkrData %p)\n", pWrkrData); + + submitCachedLines(pWrkrData, pData); /* Note: pStrm may be NULL if there was an error opening the stream */ if(pData->bFlushOnTXEnd && pData->pStrm != NULL) { /* if we have an async writer, it controls the flush via @@ -938,24 +1025,11 @@ CODESTARTendTransaction CHKiRet(strm.Flush(pData->pStrm)); } finalize_it: + pthread_mutex_unlock(&pData->mutWrite); +dbgprintf("omfile: free write lock (pWrkrData %p)\n", pWrkrData); ENDendTransaction -BEGINdoAction -CODESTARTdoAction - DBGPRINTF("file to log to: %s\n", - (pData->bDynamicName) ? ppString[1] : pData->f_fname); - DBGPRINTF("omfile: start of data: '%.128s'\n", ppString[0]); - STATSCOUNTER_INC(pData->ctrRequests, pData->mutCtrRequests); - CHKiRet(writeFile(ppString, iMsgOpts, pData)); - if(!bCoreSupportsBatching && pData->bFlushOnTXEnd) { - CHKiRet(strm.Flush(pData->pStrm)); - } -finalize_it: - if(iRet == RS_RET_OK) - iRet = RS_RET_DEFER_COMMIT; -ENDdoAction - static inline void setInstParamDefaults(instanceData *pData) @@ -1336,6 +1410,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINdoHUP CODESTARTdoHUP + pthread_mutex_lock(&pData->mutWrite); if(pData->bDynamicName) { dynaFileFreeCacheEntries(pData); } else { @@ -1343,6 +1418,7 @@ CODESTARTdoHUP closeFile(pData); } } + pthread_mutex_unlock(&pData->mutWrite); ENDdoHUP @@ -1358,6 +1434,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES diff --git a/tools/omfwd.c b/tools/omfwd.c index 6e5cf809..f0e92094 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -72,45 +72,51 @@ DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(tcpclt) + /* some local constants (just) for better readybility */ #define IS_FLUSH 1 #define NO_FLUSH 0 typedef struct _instanceData { uchar *tplName; /* name of assigned template */ - netstrms_t *pNS; /* netstream subsystem */ - netstrm_t *pNetstrm; /* our output netstream */ uchar *pszStrmDrvr; uchar *pszStrmDrvrAuthMode; permittedPeers_t *pPermPeers; int iStrmDrvrMode; char *target; - int *pSockArray; /* sockets to use for UDP */ - int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ - struct addrinfo *f_addr; int compressionLevel; /* 0 - no compression, else level for zlib */ char *port; int protocol; int iRebindInterval; /* rebind interval */ - int nXmit; /* number of transmissions since last (re-)bind */ # define FORW_UDP 0 # define FORW_TCP 1 /* following fields for TCP-based delivery */ TCPFRAMINGMODE tcp_framing; int bResendLastOnRecon; /* should the last message be re-sent on a successful reconnect? */ - tcpclt_t *pTCPClt; /* our tcpclt object */ # define COMPRESS_NEVER 0 # define COMPRESS_SINGLE_MSG 1 /* old, single-message compression */ /* all other settings are for stream-compression */ # define COMPRESS_STREAM_ALWAYS 2 uint8_t compressionMode; + int errsToReport; /* max number of errors to report (per instance) */ sbool strmCompFlushOnTxEnd; /* flush stream compression on transaction end? */ +} instanceData; + +typedef struct wrkrInstanceData { + instanceData *pData; + netstrms_t *pNS; /* netstream subsystem */ + netstrm_t *pNetstrm; /* our output netstream */ + struct addrinfo *f_addr; + int *pSockArray; /* sockets to use for UDP */ + int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ + int nXmit; /* number of transmissions since last (re-)bind */ + tcpclt_t *pTCPClt; /* our tcpclt object */ sbool bzInitDone; /* did we do an init of zstrm already? */ z_stream zstrm; /* zip stream to use for tcp compression */ uchar sndBuf[16*1024]; /* this is intensionally fixed -- see no good reason to make configurable */ unsigned offsSndBuf; /* next free spot in send buffer */ int errsToReport; /* (remaining) number of errors to report */ -} instanceData; +} wrkrInstanceData_t; /* config data */ typedef struct configSettings_s { @@ -169,6 +175,9 @@ static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current l static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current exec process */ +static rsRetVal initTCP(wrkrInstanceData_t *pWrkrData); + + BEGINinitConfVars /* (re)set config variables to default values */ CODESTARTinitConfVars cs.pszTplName = NULL; /* name of the default template to use */ @@ -182,8 +191,8 @@ CODESTARTinitConfVars ENDinitConfVars -static rsRetVal doTryResume(instanceData *pData); -static rsRetVal doZipFinish(instanceData *pData); +static rsRetVal doTryResume(wrkrInstanceData_t *); +static rsRetVal doZipFinish(wrkrInstanceData_t *); /* this function gets the default template. It coordinates action between * old-style and new-style configuration parts. @@ -227,17 +236,16 @@ finalize_it: * rgerhards, 2009-05-29 */ static rsRetVal -closeUDPSockets(instanceData *pData) +closeUDPSockets(wrkrInstanceData_t *pWrkrData) { DEFiRet; - assert(pData != NULL); - if(pData->pSockArray != NULL) { - net.closeUDPListenSockets(pData->pSockArray); - pData->pSockArray = NULL; - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; + if(pWrkrData->pSockArray != NULL) { + net.closeUDPListenSockets(pWrkrData->pSockArray); + pWrkrData->pSockArray = NULL; + freeaddrinfo(pWrkrData->f_addr); + pWrkrData->f_addr = NULL; } -pData->bIsConnected = 0; // TODO: remove this variable altogether +pWrkrData->bIsConnected = 0; // TODO: remove this variable altogether RETiRet; } @@ -248,18 +256,17 @@ pData->bIsConnected = 0; // TODO: remove this variable altogether * rgerhards, 2008-06-04 * Note that we DO NOT discard the current buffer contents * (if any). This permits us to save data between sessions. In - * the wort case, some duplication occurs, but we do not + * the worst case, some duplication occurs, but we do not * loose data. */ static inline void -DestructTCPInstanceData(instanceData *pData) +DestructTCPInstanceData(wrkrInstanceData_t *pWrkrData) { - assert(pData != NULL); - doZipFinish(pData); - if(pData->pNetstrm != NULL) - netstrm.Destruct(&pData->pNetstrm); - if(pData->pNS != NULL) - netstrms.Destruct(&pData->pNS); + doZipFinish(pWrkrData); + if(pWrkrData->pNetstrm != NULL) + netstrm.Destruct(&pWrkrData->pNetstrm); + if(pWrkrData->pNS != NULL) + netstrms.Destruct(&pWrkrData->pNS); } @@ -330,11 +337,25 @@ ENDfreeCnf BEGINcreateInstance CODESTARTcreateInstance - pData->offsSndBuf = 0; pData->errsToReport = 5; + if(cs.pszStrmDrvr != NULL) + CHKmalloc(pData->pszStrmDrvr = (uchar*)strdup((char*)cs.pszStrmDrvr)); + if(cs.pszStrmDrvrAuthMode != NULL) + CHKmalloc(pData->pszStrmDrvrAuthMode = + (uchar*)strdup((char*)cs.pszStrmDrvrAuthMode)); +finalize_it: ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance + dbgprintf("DDDD: createWrkrInstance: pWrkrData %p\n", pWrkrData); + pWrkrData->offsSndBuf = 0; + pWrkrData->errsToReport = pData->errsToReport; + iRet = initTCP(pWrkrData); +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -344,22 +365,25 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - /* final cleanup */ - DestructTCPInstanceData(pData); - closeUDPSockets(pData); - - if(pData->protocol == FORW_TCP) { - tcpclt.Destruct(&pData->pTCPClt); - } - - free(pData->port); - free(pData->target); free(pData->pszStrmDrvr); free(pData->pszStrmDrvrAuthMode); + free(pData->port); + free(pData->target); net.DestructPermittedPeers(&pData->pPermPeers); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance + DestructTCPInstanceData(pWrkrData); + closeUDPSockets(pWrkrData); + + if(pWrkrData->pData->protocol == FORW_TCP) { + tcpclt.Destruct(&pWrkrData->pTCPClt); + } +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo dbgprintf("%s", pData->target); @@ -369,7 +393,7 @@ ENDdbgPrintInstInfo /* Send a message via UDP * rgehards, 2007-12-20 */ -static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) +static rsRetVal UDPSend(wrkrInstanceData_t *pWrkrData, char *msg, size_t len) { DEFiRet; struct addrinfo *r; @@ -379,17 +403,17 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) int lasterrno; char errStr[1024]; - if(pData->iRebindInterval && (pData->nXmit++ % pData->iRebindInterval == 0)) { + if(pWrkrData->pData->iRebindInterval && (pWrkrData->nXmit++ % pWrkrData->pData->iRebindInterval == 0)) { dbgprintf("omfwd dropping UDP 'connection' (as configured)\n"); - pData->nXmit = 1; /* else we have an addtl wrap at 2^31-1 */ - CHKiRet(closeUDPSockets(pData)); + pWrkrData->nXmit = 1; /* else we have an addtl wrap at 2^31-1 */ + CHKiRet(closeUDPSockets(pWrkrData)); } - if(pData->pSockArray == NULL) { - CHKiRet(doTryResume(pData)); + if(pWrkrData->pSockArray == NULL) { + CHKiRet(doTryResume(pWrkrData)); } - if(pData->pSockArray != NULL) { + if(pWrkrData->pSockArray != NULL) { /* we need to track if we have success sending to the remote * peer. Success is indicated by at least one sendto() call * succeeding. We track this be bSendSuccess. We can not simply @@ -398,9 +422,9 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) * the sendto() succeeded. -- rgerhards, 2007-06-22 */ bSendSuccess = RSFALSE; - for (r = pData->f_addr; r; r = r->ai_next) { - for (i = 0; i < *pData->pSockArray; i++) { - lsent = sendto(pData->pSockArray[i+1], msg, len, 0, r->ai_addr, r->ai_addrlen); + for (r = pWrkrData->f_addr; r; r = r->ai_next) { + for (i = 0; i < *pWrkrData->pSockArray; i++) { + lsent = sendto(pWrkrData->pSockArray[i+1], msg, len, 0, r->ai_addr, r->ai_addrlen); if (lsent == len) { bSendSuccess = RSTRUE; break; @@ -417,17 +441,17 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) /* finished looping */ if(bSendSuccess == RSFALSE) { dbgprintf("error forwarding via udp, suspending\n"); - if(pData->errsToReport > 0) { + if(pWrkrData->errsToReport > 0) { rs_strerror_r(lasterrno, errStr, sizeof(errStr)); errmsg.LogError(0, RS_RET_ERR_UDPSEND, "omfwd: error sending " "via udp: %s", errStr); - if(pData->errsToReport == 1) { + if(pWrkrData->errsToReport == 1) { errmsg.LogError(0, RS_RET_LAST_ERRREPORT, "omfwd: " "max number of error message emitted " "- further messages will be " "suppressed"); } - --pData->errsToReport; + --pWrkrData->errsToReport; } iRet = RS_RET_SUSPENDED; } @@ -455,18 +479,18 @@ finalize_it: /* CODE FOR SENDING TCP MESSAGES */ static rsRetVal -TCPSendBufUncompressed(instanceData *pData, uchar *buf, unsigned len) +TCPSendBufUncompressed(wrkrInstanceData_t *pWrkrData, uchar *buf, unsigned len) { DEFiRet; unsigned alreadySent; ssize_t lenSend; alreadySent = 0; - CHKiRet(netstrm.CheckConnection(pData->pNetstrm)); /* hack for plain tcp syslog - see ptcp driver for details */ + CHKiRet(netstrm.CheckConnection(pWrkrData->pNetstrm)); /* hack for plain tcp syslog - see ptcp driver for details */ while(alreadySent != len) { lenSend = len - alreadySent; - CHKiRet(netstrm.Send(pData->pNetstrm, buf+alreadySent, &lenSend)); + CHKiRet(netstrm.Send(pWrkrData->pNetstrm, buf+alreadySent, &lenSend)); DBGPRINTF("omfwd: TCP sent %ld bytes, requested %u\n", (long) lenSend, len - alreadySent); alreadySent += lenSend; } @@ -475,14 +499,14 @@ finalize_it: if(iRet != RS_RET_OK) { /* error! */ dbgprintf("TCPSendBuf error %d, destruct TCP Connection!\n", iRet); - DestructTCPInstanceData(pData); + DestructTCPInstanceData(pWrkrData); iRet = RS_RET_SUSPENDED; } RETiRet; } static rsRetVal -TCPSendBufCompressed(instanceData *pData, uchar *buf, unsigned len, sbool bIsFlush) +TCPSendBufCompressed(wrkrInstanceData_t *pWrkrData, uchar *buf, unsigned len, sbool bIsFlush) { int zRet; /* zlib return state */ unsigned outavail; @@ -490,52 +514,52 @@ TCPSendBufCompressed(instanceData *pData, uchar *buf, unsigned len, sbool bIsFlu int op; DEFiRet; - if(!pData->bzInitDone) { + if(!pWrkrData->bzInitDone) { /* allocate deflate state */ - pData->zstrm.zalloc = Z_NULL; - pData->zstrm.zfree = Z_NULL; - pData->zstrm.opaque = Z_NULL; + pWrkrData->zstrm.zalloc = Z_NULL; + pWrkrData->zstrm.zfree = Z_NULL; + pWrkrData->zstrm.opaque = Z_NULL; /* see note in file header for the params we use with deflateInit2() */ - zRet = deflateInit(&pData->zstrm, 9); + zRet = deflateInit(&pWrkrData->zstrm, 9); if(zRet != Z_OK) { DBGPRINTF("error %d returned from zlib/deflateInit()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } - pData->bzInitDone = RSTRUE; + pWrkrData->bzInitDone = RSTRUE; } /* now doing the compression */ - pData->zstrm.next_in = (Bytef*) buf; - pData->zstrm.avail_in = len; - if(pData->strmCompFlushOnTxEnd && bIsFlush) + pWrkrData->zstrm.next_in = (Bytef*) buf; + pWrkrData->zstrm.avail_in = len; + if(pWrkrData->pData->strmCompFlushOnTxEnd && bIsFlush) op = Z_SYNC_FLUSH; else op = Z_NO_FLUSH; /* run deflate() on buffer until everything has been compressed */ do { - DBGPRINTF("omfwd: in deflate() loop, avail_in %d, total_in %ld, isFlush %d\n", pData->zstrm.avail_in, pData->zstrm.total_in, bIsFlush); - pData->zstrm.avail_out = sizeof(zipBuf); - pData->zstrm.next_out = zipBuf; - zRet = deflate(&pData->zstrm, op); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pData->zstrm.avail_out); - outavail = sizeof(zipBuf) - pData->zstrm.avail_out; + DBGPRINTF("omfwd: in deflate() loop, avail_in %d, total_in %ld, isFlush %d\n", pWrkrData->zstrm.avail_in, pWrkrData->zstrm.total_in, bIsFlush); + pWrkrData->zstrm.avail_out = sizeof(zipBuf); + pWrkrData->zstrm.next_out = zipBuf; + zRet = deflate(&pWrkrData->zstrm, op); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pWrkrData->zstrm.avail_out); + outavail = sizeof(zipBuf) - pWrkrData->zstrm.avail_out; if(outavail != 0) { - CHKiRet(TCPSendBufUncompressed(pData, zipBuf, outavail)); + CHKiRet(TCPSendBufUncompressed(pWrkrData, zipBuf, outavail)); } - } while (pData->zstrm.avail_out == 0); + } while (pWrkrData->zstrm.avail_out == 0); finalize_it: RETiRet; } static rsRetVal -TCPSendBuf(instanceData *pData, uchar *buf, unsigned len, sbool bIsFlush) +TCPSendBuf(wrkrInstanceData_t *pWrkrData, uchar *buf, unsigned len, sbool bIsFlush) { DEFiRet; - if(pData->compressionMode >= COMPRESS_STREAM_ALWAYS) - iRet = TCPSendBufCompressed(pData, buf, len, bIsFlush); + if(pWrkrData->pData->compressionMode >= COMPRESS_STREAM_ALWAYS) + iRet = TCPSendBufCompressed(pWrkrData, buf, len, bIsFlush); else - iRet = TCPSendBufUncompressed(pData, buf, len); + iRet = TCPSendBufUncompressed(pWrkrData, buf, len); RETiRet; } @@ -543,39 +567,39 @@ TCPSendBuf(instanceData *pData, uchar *buf, unsigned len, sbool bIsFlush) * running in stream mode). */ static rsRetVal -doZipFinish(instanceData *pData) +doZipFinish(wrkrInstanceData_t *pWrkrData) { int zRet; /* zlib return state */ DEFiRet; unsigned outavail; uchar zipBuf[32*1024]; - if(!pData->bzInitDone) + if(!pWrkrData->bzInitDone) goto done; // TODO: can we get this into a single common function? dbgprintf("DDDD: in doZipFinish()\n"); - pData->zstrm.avail_in = 0; + pWrkrData->zstrm.avail_in = 0; /* run deflate() on buffer until everything has been compressed */ do { - DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pData->zstrm.avail_in, pData->zstrm.total_in); - pData->zstrm.avail_out = sizeof(zipBuf); - pData->zstrm.next_out = zipBuf; - zRet = deflate(&pData->zstrm, Z_FINISH); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pData->zstrm.avail_out); - outavail = sizeof(zipBuf) - pData->zstrm.avail_out; + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pWrkrData->zstrm.avail_in, pWrkrData->zstrm.total_in); + pWrkrData->zstrm.avail_out = sizeof(zipBuf); + pWrkrData->zstrm.next_out = zipBuf; + zRet = deflate(&pWrkrData->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pWrkrData->zstrm.avail_out); + outavail = sizeof(zipBuf) - pWrkrData->zstrm.avail_out; if(outavail != 0) { - CHKiRet(TCPSendBufUncompressed(pData, zipBuf, outavail)); + CHKiRet(TCPSendBufUncompressed(pWrkrData, zipBuf, outavail)); } - } while (pData->zstrm.avail_out == 0); + } while (pWrkrData->zstrm.avail_out == 0); finalize_it: - zRet = deflateEnd(&pData->zstrm); + zRet = deflateEnd(&pWrkrData->zstrm); if(zRet != Z_OK) { DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); } - pData->bzInitDone = 0; + pWrkrData->bzInitDone = 0; done: RETiRet; } @@ -585,26 +609,26 @@ done: RETiRet; static rsRetVal TCPSendFrame(void *pvData, char *msg, size_t len) { DEFiRet; - instanceData *pData = (instanceData *) pvData; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t *) pvData; DBGPRINTF("omfwd: add %u bytes to send buffer (curr offs %u)\n", - (unsigned) len, pData->offsSndBuf); - if(pData->offsSndBuf != 0 && pData->offsSndBuf + len >= sizeof(pData->sndBuf)) { + (unsigned) len, pWrkrData->offsSndBuf); + if(pWrkrData->offsSndBuf != 0 && pWrkrData->offsSndBuf + len >= sizeof(pWrkrData->sndBuf)) { /* no buffer space left, need to commit previous records */ - CHKiRet(TCPSendBuf(pData, pData->sndBuf, pData->offsSndBuf, NO_FLUSH)); - pData->offsSndBuf = 0; + CHKiRet(TCPSendBuf(pWrkrData, pWrkrData->sndBuf, pWrkrData->offsSndBuf, NO_FLUSH)); + pWrkrData->offsSndBuf = 0; iRet = RS_RET_PREVIOUS_COMMITTED; } /* check if the message is too large to fit into buffer */ - if(len > sizeof(pData->sndBuf)) { - CHKiRet(TCPSendBuf(pData, (uchar*)msg, len, NO_FLUSH)); + if(len > sizeof(pWrkrData->sndBuf)) { + CHKiRet(TCPSendBuf(pWrkrData, (uchar*)msg, len, NO_FLUSH)); ABORT_FINALIZE(RS_RET_OK); /* committed everything so far */ } /* we now know the buffer has enough free space */ - memcpy(pData->sndBuf + pData->offsSndBuf, msg, len); - pData->offsSndBuf += len; + memcpy(pWrkrData->sndBuf + pWrkrData->offsSndBuf, msg, len); + pWrkrData->offsSndBuf += len; iRet = RS_RET_DEFER_COMMIT; finalize_it: @@ -619,10 +643,10 @@ finalize_it: static rsRetVal TCPSendPrepRetry(void *pvData) { DEFiRet; - instanceData *pData = (instanceData *) pvData; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t *) pvData; - assert(pData != NULL); - DestructTCPInstanceData(pData); + assert(pWrkrData != NULL); + DestructTCPInstanceData(pWrkrData); RETiRet; } @@ -633,36 +657,39 @@ static rsRetVal TCPSendPrepRetry(void *pvData) static rsRetVal TCPSendInit(void *pvData) { DEFiRet; - instanceData *pData = (instanceData *) pvData; + wrkrInstanceData_t *pWrkrData = (wrkrInstanceData_t *) pvData; + instanceData *pData; - assert(pData != NULL); - if(pData->pNetstrm == NULL) { + assert(pWrkrData != NULL); + pData = pWrkrData->pData; + + if(pWrkrData->pNetstrm == NULL) { dbgprintf("TCPSendInit CREATE\n"); - CHKiRet(netstrms.Construct(&pData->pNS)); + CHKiRet(netstrms.Construct(&pWrkrData->pNS)); /* the stream driver must be set before the object is finalized! */ - CHKiRet(netstrms.SetDrvrName(pData->pNS, pData->pszStrmDrvr)); - CHKiRet(netstrms.ConstructFinalize(pData->pNS)); + CHKiRet(netstrms.SetDrvrName(pWrkrData->pNS, pData->pszStrmDrvr)); + CHKiRet(netstrms.ConstructFinalize(pWrkrData->pNS)); /* now create the actual stream and connect to the server */ - CHKiRet(netstrms.CreateStrm(pData->pNS, &pData->pNetstrm)); - CHKiRet(netstrm.ConstructFinalize(pData->pNetstrm)); - CHKiRet(netstrm.SetDrvrMode(pData->pNetstrm, pData->iStrmDrvrMode)); + CHKiRet(netstrms.CreateStrm(pWrkrData->pNS, &pWrkrData->pNetstrm)); + CHKiRet(netstrm.ConstructFinalize(pWrkrData->pNetstrm)); + CHKiRet(netstrm.SetDrvrMode(pWrkrData->pNetstrm, pData->iStrmDrvrMode)); /* now set optional params, but only if they were actually configured */ if(pData->pszStrmDrvrAuthMode != NULL) { - CHKiRet(netstrm.SetDrvrAuthMode(pData->pNetstrm, pData->pszStrmDrvrAuthMode)); + CHKiRet(netstrm.SetDrvrAuthMode(pWrkrData->pNetstrm, pData->pszStrmDrvrAuthMode)); } if(pData->pPermPeers != NULL) { - CHKiRet(netstrm.SetDrvrPermPeers(pData->pNetstrm, pData->pPermPeers)); + CHKiRet(netstrm.SetDrvrPermPeers(pWrkrData->pNetstrm, pData->pPermPeers)); } /* params set, now connect */ - CHKiRet(netstrm.Connect(pData->pNetstrm, glbl.GetDefPFFamily(), + CHKiRet(netstrm.Connect(pWrkrData->pNetstrm, glbl.GetDefPFFamily(), (uchar*)pData->port, (uchar*)pData->target)); } finalize_it: if(iRet != RS_RET_OK) { dbgprintf("TCPSendInit FAILED with %d.\n", iRet); - DestructTCPInstanceData(pData); + DestructTCPInstanceData(pWrkrData); } RETiRet; @@ -672,15 +699,17 @@ finalize_it: /* try to resume connection if it is not ready * rgerhards, 2007-08-02 */ -static rsRetVal doTryResume(instanceData *pData) +static rsRetVal doTryResume(wrkrInstanceData_t *pWrkrData) { int iErr; struct addrinfo *res; struct addrinfo hints; + instanceData *pData; DEFiRet; - if(pData->bIsConnected) + if(pWrkrData->bIsConnected) FINALIZE; + pData = pWrkrData->pData; /* The remote address is not yet known and needs to be obtained */ dbgprintf(" %s\n", pData->target); @@ -696,20 +725,20 @@ static rsRetVal doTryResume(instanceData *pData) ABORT_FINALIZE(RS_RET_SUSPENDED); } dbgprintf("%s found, resuming.\n", pData->target); - pData->f_addr = res; - pData->bIsConnected = 1; - if(pData->pSockArray == NULL) { - pData->pSockArray = net.create_udp_socket((uchar*)pData->target, NULL, 0, 0); + pWrkrData->f_addr = res; + pWrkrData->bIsConnected = 1; + if(pWrkrData->pSockArray == NULL) { + pWrkrData->pSockArray = net.create_udp_socket((uchar*)pData->target, NULL, 0, 0); } } else { - CHKiRet(TCPSendInit((void*)pData)); + CHKiRet(TCPSendInit((void*)pWrkrData)); } finalize_it: if(iRet != RS_RET_OK) { - if(pData->f_addr != NULL) { - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; + if(pWrkrData->f_addr != NULL) { + freeaddrinfo(pWrkrData->f_addr); + pWrkrData->f_addr = NULL; } iRet = RS_RET_SUSPENDED; } @@ -720,13 +749,15 @@ finalize_it: BEGINtryResume CODESTARTtryResume - iRet = doTryResume(pData); + dbgprintf("DDDD: tryResume: pWrkrData %p\n", pWrkrData); + iRet = doTryResume(pWrkrData); ENDtryResume BEGINbeginTransaction CODESTARTbeginTransaction dbgprintf("omfwd: beginTransaction\n"); + iRet = doTryResume(pWrkrData); ENDbeginTransaction @@ -737,8 +768,11 @@ BEGINdoAction # ifdef USE_NETZIP Bytef *out = NULL; /* for compression */ # endif + instanceData *pData; CODESTARTdoAction - CHKiRet(doTryResume(pData)); + dbgprintf("DDDD: doAction: pWrkrData %p\n", pWrkrData); + pData = pWrkrData->pData; + CHKiRet(doTryResume(pWrkrData)); iMaxLine = glbl.GetMaxLine(); @@ -791,14 +825,14 @@ CODESTARTdoAction if(pData->protocol == FORW_UDP) { /* forward via UDP */ - CHKiRet(UDPSend(pData, psz, l)); + CHKiRet(UDPSend(pWrkrData, psz, l)); } else { /* forward via TCP */ - iRet = tcpclt.Send(pData->pTCPClt, pData, psz, l); + iRet = tcpclt.Send(pWrkrData->pTCPClt, pWrkrData, psz, l); if(iRet != RS_RET_OK && iRet != RS_RET_DEFER_COMMIT && iRet != RS_RET_PREVIOUS_COMMITTED) { /* error! */ dbgprintf("error forwarding via tcp, suspending\n"); - DestructTCPInstanceData(pData); + DestructTCPInstanceData(pWrkrData); iRet = RS_RET_SUSPENDED; } if(pData->compressionMode >= COMPRESS_STREAM_ALWAYS && pData->strmCompFlushOnTxEnd) @@ -814,10 +848,10 @@ ENDdoAction BEGINendTransaction CODESTARTendTransaction -dbgprintf("omfwd: endTransaction, offsSndBuf %u\n", pData->offsSndBuf); - if(pData->offsSndBuf != 0) { - iRet = TCPSendBuf(pData, pData->sndBuf, pData->offsSndBuf, IS_FLUSH); - pData->offsSndBuf = 0; +dbgprintf("omfwd: endTransaction, offsSndBuf %u\n", pWrkrData->offsSndBuf); + if(pWrkrData->offsSndBuf != 0) { + iRet = TCPSendBuf(pWrkrData, pWrkrData->sndBuf, pWrkrData->offsSndBuf, IS_FLUSH); + pWrkrData->offsSndBuf = 0; } ENDendTransaction @@ -844,24 +878,22 @@ finalize_it: * created. */ static rsRetVal -initTCP(instanceData *pData) +initTCP(wrkrInstanceData_t *pWrkrData) { + instanceData *pData; DEFiRet; + + pData = pWrkrData->pData; if(pData->protocol == FORW_TCP) { /* create our tcpclt */ - CHKiRet(tcpclt.Construct(&pData->pTCPClt)); - CHKiRet(tcpclt.SetResendLastOnRecon(pData->pTCPClt, pData->bResendLastOnRecon)); + CHKiRet(tcpclt.Construct(&pWrkrData->pTCPClt)); + CHKiRet(tcpclt.SetResendLastOnRecon(pWrkrData->pTCPClt, pData->bResendLastOnRecon)); /* and set callbacks */ - CHKiRet(tcpclt.SetSendInit(pData->pTCPClt, TCPSendInit)); - CHKiRet(tcpclt.SetSendFrame(pData->pTCPClt, TCPSendFrame)); - CHKiRet(tcpclt.SetSendPrepRetry(pData->pTCPClt, TCPSendPrepRetry)); - CHKiRet(tcpclt.SetFraming(pData->pTCPClt, pData->tcp_framing)); - CHKiRet(tcpclt.SetRebindInterval(pData->pTCPClt, pData->iRebindInterval)); - if(cs.pszStrmDrvr != NULL) - CHKmalloc(pData->pszStrmDrvr = (uchar*)strdup((char*)cs.pszStrmDrvr)); - if(cs.pszStrmDrvrAuthMode != NULL) - CHKmalloc(pData->pszStrmDrvrAuthMode = - (uchar*)strdup((char*)cs.pszStrmDrvrAuthMode)); + CHKiRet(tcpclt.SetSendInit(pWrkrData->pTCPClt, TCPSendInit)); + CHKiRet(tcpclt.SetSendFrame(pWrkrData->pTCPClt, TCPSendFrame)); + CHKiRet(tcpclt.SetSendPrepRetry(pWrkrData->pTCPClt, TCPSendPrepRetry)); + CHKiRet(tcpclt.SetFraming(pWrkrData->pTCPClt, pData->tcp_framing)); + CHKiRet(tcpclt.SetRebindInterval(pWrkrData->pTCPClt, pData->iRebindInterval)); } finalize_it: RETiRet; @@ -1046,7 +1078,6 @@ CODESTARTnewActInst tplToUse = ustrdup((pData->tplName == NULL) ? getDfltTpl() : pData->tplName); CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS)); - CHKiRet(initTCP(pData)); CODE_STD_FINALIZERnewActInst cnfparamvalsDestruct(pvals, &actpblk); ENDnewActInst @@ -1215,7 +1246,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) cs.pPermPeers = NULL; } } - CHKiRet(initTCP(pData)); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -1251,6 +1281,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_setModCnf_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES diff --git a/tools/ompipe.c b/tools/ompipe.c index df8066b1..79f3ae84 100644 --- a/tools/ompipe.c +++ b/tools/ompipe.c @@ -12,7 +12,7 @@ * NOTE: read comments in module-template.h to understand how this pipe * works! * - * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -69,9 +69,14 @@ typedef struct _instanceData { uchar *pipe; /* pipe or template name (display only) */ uchar *tplName; /* format template to use */ short fd; /* pipe descriptor for (current) pipe */ + pthread_mutex_t mutWrite; /* guard against multiple instances writing to same pipe */ sbool bHadError; /* did we already have/report an error on this pipe? */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { EMPTY_STRUCT } configSettings_t; @@ -276,25 +281,42 @@ CODESTARTcreateInstance pData->pipe = NULL; pData->fd = -1; pData->bHadError = 0; + pthread_mutex_init(&pData->mutWrite, NULL); ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINfreeInstance CODESTARTfreeInstance + pthread_mutex_destroy(&pData->mutWrite); free(pData->pipe); if(pData->fd != -1) close(pData->fd); ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINtryResume CODESTARTtryResume ENDtryResume BEGINdoAction + instanceData *pData; CODESTARTdoAction + pData = pWrkrData->pData; DBGPRINTF(" (%s)\n", pData->pipe); + /* this module is single-threaded by nature */ + pthread_mutex_lock(&pData->mutWrite); iRet = writePipe(ppString, pData); + pthread_mutex_unlock(&pData->mutWrite); ENDdoAction @@ -390,6 +412,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_doHUP CODEqueryEtryPt_STD_CONF2_QUERIES CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES diff --git a/tools/omshell.c b/tools/omshell.c index ac62fa62..ad6e979f 100644 --- a/tools/omshell.c +++ b/tools/omshell.c @@ -19,7 +19,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -63,11 +63,20 @@ typedef struct _instanceData { uchar progName[MAXFNAME]; /* program to execute */ } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + BEGINcreateInstance CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -80,6 +89,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo printf("%s", pData->progName); @@ -92,13 +106,9 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - /* TODO: using pData->progName is not clean from the point of - * modularization. We'll change that as we go ahead with modularization. - * rgerhards, 2007-07-20 - */ dbgprintf("\n"); - if(execProg((uchar*) pData->progName, 1, ppString[0]) == 0) - errmsg.LogError(0, NO_ERRCODE, "Executing program '%s' failed", (char*)pData->progName); + if(execProg((uchar*) pWrkrData->pData->progName, 1, ppString[0]) == 0) + errmsg.LogError(0, NO_ERRCODE, "Executing program '%s' failed", (char*)pWrkrData->pData->progName); ENDdoAction @@ -139,6 +149,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES ENDqueryEtryPt diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index f4cc4094..5d0b088f 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -8,7 +8,7 @@ * File begun on 2007-07-20 by RGerhards (extracted from syslogd.c, which at the * time of the fork from sysklogd was under BSD license) * - * Copyright 2007-2012 Adiscon GmbH. + * Copyright 2007-2013 Adiscon GmbH. * * This file is part of rsyslog. * @@ -87,6 +87,10 @@ typedef struct _instanceData { uchar *tplName; } instanceData; +typedef struct wrkrInstanceData { + instanceData *pData; +} wrkrInstanceData_t; + typedef struct configSettings_s { EMPTY_STRUCT } configSettings_t; @@ -115,6 +119,11 @@ CODESTARTcreateInstance ENDcreateInstance +BEGINcreateWrkrInstance +CODESTARTcreateWrkrInstance +ENDcreateWrkrInstance + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature if(eFeat == sFEATURERepeatedMsgReduction) @@ -128,6 +137,11 @@ CODESTARTfreeInstance ENDfreeInstance +BEGINfreeWrkrInstance +CODESTARTfreeWrkrInstance +ENDfreeWrkrInstance + + BEGINdbgPrintInstInfo register int i; CODESTARTdbgPrintInstInfo @@ -276,7 +290,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction dbgprintf("\n"); - iRet = wallmsg(ppString[0], pData); + iRet = wallmsg(ppString[0], pWrkrData->pData); ENDdoAction @@ -435,6 +449,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_STD_OMOD8_QUERIES CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES ENDqueryEtryPt diff --git a/tools/syslogd.c b/tools/syslogd.c index d8d09117..f9cc4182 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -494,24 +494,19 @@ finalize_it: * rgerhards, 2010-06-09 */ static inline rsRetVal -preprocessBatch(batch_t *pBatch) { +preprocessBatch(batch_t *pBatch, int *pbShutdownImmediate) { prop_t *ip; prop_t *fqdn; prop_t *localName; prop_t *propFromHost = NULL; prop_t *propFromHostIP = NULL; - int bSingleRuleset; - ruleset_t *batchRuleset; /* the ruleset used for all message inside the batch, if there is a single one */ int bIsPermitted; msg_t *pMsg; int i; rsRetVal localRet; DEFiRet; - bSingleRuleset = 1; - batchRuleset = (pBatch->nElem > 0) ? pBatch->pElem[0].pMsg->pRuleset : NULL; - - for(i = 0 ; i < pBatch->nElem && !*(pBatch->pbShutdownImmediate) ; i++) { + for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { pMsg = pBatch->pElem[i].pMsg; if((pMsg->msgFlags & NEEDS_ACLCHK_U) != 0) { DBGPRINTF("msgConsumer: UDP ACL must be checked for message (hostname-based)\n"); @@ -536,12 +531,8 @@ preprocessBatch(batch_t *pBatch) { pBatch->eltState[i] = BATCH_STATE_DISC; } } - if(pMsg->pRuleset != batchRuleset) - bSingleRuleset = 0; } - batchSetSingleRuleset(pBatch, bSingleRuleset); - finalize_it: if(propFromHost != NULL) prop.Destruct(&propFromHost); @@ -557,17 +548,16 @@ finalize_it: * for the main queue. */ static rsRetVal -msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShutdownImmediate) +msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, wti_t *pWti) { DEFiRet; assert(pBatch != NULL); - pBatch->pbShutdownImmediate = pbShutdownImmediate; /* TODO: move this to batch creation! */ - preprocessBatch(pBatch); - ruleset.ProcessBatch(pBatch); + preprocessBatch(pBatch, pWti->pbShutdownImmediate); + ruleset.ProcessBatch(pBatch, pWti); //TODO: the BATCH_STATE_COMM must be set somewhere down the road, but we //do not have this yet and so we emulate -- 2010-06-10 int i; - for(i = 0 ; i < pBatch->nElem && !*pbShutdownImmediate ; i++) { + for(i = 0 ; i < pBatch->nElem && !*pWti->pbShutdownImmediate ; i++) { pBatch->eltState[i] = BATCH_STATE_COMM; } RETiRet; @@ -1128,9 +1118,14 @@ rsRetVal createMainQueue(qqueue_t **ppQueue, uchar *pszQueueName, struct nvlst * qqueueSetDefaultsRulesetQueue(*ppQueue); qqueueApplyCnfParam(*ppQueue, lst); } + RETiRet; +} - /* ... and finally start the queue! */ - CHKiRet_Hdlr(qqueueStart(*ppQueue)) { +rsRetVal +startMainQueue(qqueue_t *pQueue) +{ + DEFiRet; + CHKiRet_Hdlr(qqueueStart(pQueue)) { /* no queue is fatal, we need to give up in that case... */ errmsg.LogError(0, iRet, "could not start (ruleset) main message queue"); \ } @@ -1330,6 +1325,11 @@ static void printVersion(void) #else printf("\t64bit Atomic operations supported:\tNo\n"); #endif +#ifdef HAVE_JEMALLOC + printf("\tmemory allocator:\t\t\tjemalloc\n"); +#else + printf("\tmemory allocator:\t\t\tsystem default\n"); +#endif #ifdef RTINST printf("\tRuntime Instrumentation (slow code):\tYes\n"); #else |