detect: set mpm/prefilter during signature parsing

In preparation of flowbit prefilter work that needs this info earlier. Track potential prefilter sm's to avoid unnecessary looping during setup.
OISF · Jan 15, 2025 · bd3e65b · bd3e65b
1 parent 05853fb
commit bd3e65b
Show file tree

Hide file tree

Showing 9 changed files with 162 additions and 212 deletions.
diff --git a/src/detect-byte-extract.c b/src/detect-byte-extract.c
@@ -1945,9 +1945,8 @@ static int DetectByteExtractTest43(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_OFFSET_VAR |
-                      DETECT_CONTENT_OFFSET) ||
-        cd->offset != bed->local_id) {
+            cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
+            cd->offset != bed->local_id) {
         printf("three failed\n");
         result = 0;
         goto end;
@@ -2049,9 +2048,8 @@ static int DetectByteExtractTest44(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_OFFSET_VAR |
-                      DETECT_CONTENT_OFFSET) ||
-        cd->offset != bed1->local_id) {
+            cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
+            cd->offset != bed1->local_id) {
         printf("four failed\n");
         result = 0;
         goto end;
@@ -2158,10 +2156,8 @@ static int DetectByteExtractTest45(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_DEPTH_VAR |
-                      DETECT_CONTENT_DEPTH) ||
-        cd->depth != bed->local_id ||
-        cd->offset != 0) {
+            cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
+            cd->depth != bed->local_id || cd->offset != 0) {
         printf("three failed\n");
         result = 0;
         goto end;
@@ -2263,9 +2259,8 @@ static int DetectByteExtractTest46(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_DEPTH_VAR |
-                      DETECT_CONTENT_DEPTH) ||
-        cd->depth != bed1->local_id) {
+            cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
+            cd->depth != bed1->local_id) {
         printf("four failed\n");
         result = 0;
         goto end;
@@ -2372,11 +2367,9 @@ static int DetectByteExtractTest47(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
-                      DETECT_CONTENT_DISTANCE) ||
-        cd->distance != bed->local_id ||
-        cd->offset != 0 ||
-        cd->depth != 0) {
+            cd->flags !=
+                    (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM) ||
+            cd->distance != bed->local_id || cd->offset != 0 || cd->depth != 0) {
         printf("three failed\n");
         result = 0;
         goto end;
@@ -2478,12 +2471,9 @@ static int DetectByteExtractTest48(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
-                      DETECT_CONTENT_DISTANCE |
-                      DETECT_CONTENT_DISTANCE_NEXT) ||
-        cd->distance != bed1->local_id ||
-        cd->depth != 0 ||
-        cd->offset != 0) {
+            cd->flags != (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE |
+                                 DETECT_CONTENT_DISTANCE_NEXT | DETECT_CONTENT_MPM) ||
+            cd->distance != bed1->local_id || cd->depth != 0 || cd->offset != 0) {
         printf("four failed\n");
         result = 0;
         goto end;
@@ -2592,12 +2582,8 @@ static int DetectByteExtractTest49(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_WITHIN_VAR |
-                      DETECT_CONTENT_WITHIN) ||
-        cd->within != bed->local_id ||
-        cd->offset != 0 ||
-        cd->depth != 0 ||
-        cd->distance != 0) {
+            cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
+            cd->within != bed->local_id || cd->offset != 0 || cd->depth != 0 || cd->distance != 0) {
         printf("three failed\n");
         result = 0;
         goto end;
@@ -2699,13 +2685,10 @@ static int DetectByteExtractTest50(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_WITHIN_VAR |
-                      DETECT_CONTENT_WITHIN|
-                      DETECT_CONTENT_WITHIN_NEXT) ||
-        cd->within != bed1->local_id ||
-        cd->depth != 0 ||
-        cd->offset != 0 ||
-        cd->distance != 0) {
+            cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN |
+                                 DETECT_CONTENT_WITHIN_NEXT | DETECT_CONTENT_MPM) ||
+            cd->within != bed1->local_id || cd->depth != 0 || cd->offset != 0 ||
+            cd->distance != 0) {
         printf("four failed\n");
         result = 0;
         goto end;
@@ -2970,7 +2953,7 @@ static int DetectByteExtractTest53(void)
     SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH];
     FAIL_IF(sm->type != DETECT_CONTENT);
     DetectContentData *cd = (DetectContentData *)sm->ctx;
-    FAIL_IF(cd->flags != 0);
+    FAIL_IF(cd->flags != (DETECT_CONTENT_MPM | DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED));
 
     sm = sm->next;
     FAIL_IF_NULL(sm);
@@ -3092,11 +3075,8 @@ static int DetectByteExtractTest54(void)
         goto end;
     }
     bjd = (DetectBytejumpData *)sm->ctx;
-    if (bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR || bjd->offset != 1) {
-        printf("four failed\n");
-        result = 0;
-        goto end;
-    }
+    FAIL_IF(bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR);
+    FAIL_IF(bjd->offset != 1);
 
     if (sm->next != NULL)
         goto end;
@@ -3197,12 +3177,10 @@ static int DetectByteExtractTest55(void)
     }
     cd = (DetectContentData *)sm->ctx;
     if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
-        cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
-                      DETECT_CONTENT_WITHIN_VAR |
-                      DETECT_CONTENT_DISTANCE |
-                      DETECT_CONTENT_WITHIN) ||
-        cd->within != bed1->local_id ||
-        cd->distance != bed2->local_id) {
+            cd->flags !=
+                    (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_WITHIN_VAR |
+                            DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
+            cd->within != bed1->local_id || cd->distance != bed2->local_id) {
         printf("four failed: ");
         goto end;
     }
@@ -3829,8 +3807,8 @@ static int DetectByteExtractTest60(void)
         goto end;
     }
     cd = (DetectContentData *)sm->ctx;
-    if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
-        strncmp((char *)cd->content, "three", cd->content_len) != 0) {
+    if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
+            strncmp((char *)cd->content, "three", cd->content_len) != 0) {
         printf("one failed\n");
         result = 0;
         goto end;
@@ -3950,8 +3928,8 @@ static int DetectByteExtractTest61(void)
         goto end;
     }
     cd = (DetectContentData *)sm->ctx;
-    if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
-        strncmp((char *)cd->content, "three", cd->content_len) != 0) {
+    if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
+            strncmp((char *)cd->content, "three", cd->content_len) != 0) {
         printf("one failed\n");
         result = 0;
         goto end;

diff --git a/src/detect-content.c b/src/detect-content.c
@@ -1504,7 +1504,7 @@ static int DetectContentParseTest19(void)
     FAIL_IF_NOT(s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL);
 
     DetectContentData *data = (DetectContentData *)sm->ctx;
-    FAIL_IF_NOT(data->flags == DETECT_CONTENT_DISTANCE);
+    FAIL_IF_NOT(data->flags == (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM));
 
     s = DetectEngineAppendSig(de_ctx,
             "alert tcp any any -> any any "

diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
@@ -1703,7 +1703,6 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
     }
 }
 
-extern int g_skip_prefilter;
 /**
  * \brief Preprocess signature, classify ip-only, etc, build sig array
  *
@@ -1787,39 +1786,6 @@ int SigPrepareStage1(DetectEngineCtx *de_ctx)
 
         RuleSetScore(s);
 
-        /* if keyword engines are enabled in the config, handle them here */
-        if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
-                !(s->flags & SIG_FLAG_PREFILTER)) {
-            int prefilter_list = DETECT_TBLSIZE;
-
-            // TODO buffers?
-
-            /* get the keyword supporting prefilter with the lowest type */
-            for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
-                for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
-                    if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
-                        if (sigmatch_table[sm->type].SupportsPrefilter(s)) {
-                            prefilter_list = MIN(prefilter_list, sm->type);
-                        }
-                    }
-                }
-            }
-
-            /* apply that keyword as prefilter */
-            if (prefilter_list != DETECT_TBLSIZE) {
-                for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
-                    for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
-                        if (sm->type == prefilter_list) {
-                            s->init_data->prefilter_sm = sm;
-                            s->flags |= SIG_FLAG_PREFILTER;
-                            SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-
         /* run buffer type callbacks if any */
         for (int x = 0; x < DETECT_SM_LIST_MAX; x++) {
             if (s->init_data->smlists[x])

diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c
@@ -167,7 +167,6 @@ static int DetectLoadSigFile(
         sig = DetectEngineAppendSig(de_ctx, line);
         if (sig != NULL) {
             if (rule_engine_analysis_set || fp_engine_analysis_set) {
-                RetrieveFPForSig(de_ctx, sig);
                 if (fp_engine_analysis_set) {
                     EngineAnalysisFP(de_ctx, sig, line);
                 }

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
@@ -2418,12 +2418,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
 {
     uint32_t cnt = 0;
     for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
-        if (s->flags & SIG_FLAG_PREFILTER)
-            continue;
-
-        RetrieveFPForSig(de_ctx, s);
         if (s->init_data->mpm_sm != NULL) {
-            s->flags |= SIG_FLAG_PREFILTER;
             cnt++;
         }
     }

diff --git a/src/detect-flow.c b/src/detect-flow.c
@@ -445,14 +445,18 @@ void DetectFlowFree(DetectEngineCtx *de_ctx, void *ptr)
 static void
 PrefilterPacketFlowMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
 {
+    SCEnter();
+
     const PrefilterPacketHeaderCtx *ctx = pectx;
 
     if (!PrefilterPacketHeaderExtraMatch(ctx, p))
         return;
 
     if (FlowMatch(p->flags, p->flowflags, ctx->v1.u16[0], ctx->v1.u16[1])) {
+        SCLogDebug("match: adding sids");
         PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt);
     }
+    SCReturn;
 }
 
 static void

diff --git a/src/detect-parse.c b/src/detect-parse.c
@@ -914,6 +914,9 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr,
     }
     s->init_data->negated = false;
 
+    const enum DetectKeywordId idx = SigTableGetIndex(st);
+    s->init_data->has_possible_prefilter |= de_ctx->sm_types_prefilter[idx];
+
     if (st->flags & SIGMATCH_INFO_DEPRECATED) {
 #define URL "https://suricata.io/our-story/deprecation-policy/"
         if (st->alternative == 0)
@@ -1021,7 +1024,6 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr,
 
         /* handle 'silent' error case */
         if (setup_ret == -2) {
-            enum DetectKeywordId idx = SigTableGetIndex(st);
             if (de_ctx->sm_types_silent_error[idx] == false) {
                 de_ctx->sm_types_silent_error[idx] = true;
                 return -1;
@@ -1877,6 +1879,61 @@ SigMatchData* SigMatchList2DataArray(SigMatch *head)
     return out;
 }
 
+extern int g_skip_prefilter;
+
+static void SigSetupPrefilter(DetectEngineCtx *de_ctx, Signature *s)
+{
+    SCEnter();
+    if (s->init_data->prefilter_sm != NULL || s->init_data->mpm_sm != NULL) {
+        SCReturn;
+    }
+
+    SCLogDebug("s %u: set up prefilter/mpm", s->id);
+    RetrieveFPForSig(de_ctx, s);
+    if (s->init_data->mpm_sm != NULL) {
+        s->flags |= SIG_FLAG_PREFILTER;
+        SCReturn;
+    }
+
+    SCLogDebug("s %u: no mpm; prefilter? de_ctx->prefilter_setting %u "
+               "s->init_data->has_possible_prefilter %s",
+            s->id, de_ctx->prefilter_setting, BOOL2STR(s->init_data->has_possible_prefilter));
+
+    if (!s->init_data->has_possible_prefilter)
+        SCReturn;
+
+    if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
+            !(s->flags & SIG_FLAG_PREFILTER)) {
+        int prefilter_list = DETECT_TBLSIZE;
+        /* get the keyword supporting prefilter with the lowest type */
+        for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
+            for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
+                if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
+                    if (sigmatch_table[sm->type].SupportsPrefilter(s)) {
+                        prefilter_list = MIN(prefilter_list, sm->type);
+                    }
+                }
+            }
+        }
+
+        /* apply that keyword as prefilter */
+        if (prefilter_list != DETECT_TBLSIZE) {
+            for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
+                for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
+                    if (sm->type == prefilter_list) {
+                        s->init_data->prefilter_sm = sm;
+                        s->flags |= SIG_FLAG_PREFILTER;
+                        SCLogConfig("sid %u: prefilter is on \"%s\"", s->id,
+                                sigmatch_table[sm->type].name);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    SCReturn;
+}
+
 /**
  *  \internal
  *  \brief validate a just parsed signature for internal inconsistencies
@@ -2219,6 +2276,8 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr,
         DetectEngineBufferRunSetupCallback(de_ctx, sig->init_data->buffers[x].id, sig);
     }
 
+    SigSetupPrefilter(de_ctx, sig);
+
     /* validate signature, SigValidate will report the error reason */
     if (SigValidate(de_ctx, sig) == 0) {
         goto error;

diff --git a/src/detect.h b/src/detect.h
@@ -551,6 +551,9 @@ typedef struct SignatureInitData_ {
     bool src_contains_negation;
     bool dst_contains_negation;
 
+    /** see if any of the sigmatches supports an enabled prefilter */
+    bool has_possible_prefilter;
+
     /* used to hold flags that are used during init */
     uint32_t init_flags;
     /* coccinelle: SignatureInitData:init_flags:SIG_FLAG_INIT_ */