hyperscan: add caching mechanism for hyperscan contexts v10

Makefile.am

@@ -37,6 +37,7 @@ install-conf:
 	install -d "$(DESTDIR)$(e_rundir)"
 	install -m 770 -d "$(DESTDIR)$(e_localstatedir)"
 	install -m 770 -d "$(DESTDIR)$(e_datadir)"
+	install -m 660 -d "$(DESTDIR)$(e_sghcachedir)"
 
 install-rules:
 if INSTALL_SURICATA_UPDATE

configure.ac

@@ -2455,6 +2455,7 @@ if test "$WINDOWS_PATH" = "yes"; then
 
     e_sysconfdir="${e_winbase}\\\\"
     e_defaultruledir="$e_winbase\\\\rules\\\\"
+    e_sghcachedir="$e_winbase\\\\cache\\\\sgh\\\\"
     e_magic_file="$e_winbase\\\\magic.mgc"
     e_logdir="$e_winbase\\\\log"
     e_logfilesdir="$e_logdir\\\\files"
@@ -2476,6 +2477,7 @@ else
     EXPAND_VARIABLE(sysconfdir, e_sysconfdir, "/suricata/")
     EXPAND_VARIABLE(localstatedir, e_localstatedir, "/run/suricata")
     EXPAND_VARIABLE(datadir, e_datarulesdir, "/suricata/rules")
+    EXPAND_VARIABLE(localstatedir, e_sghcachedir, "/lib/suricata/cache/sgh")
     EXPAND_VARIABLE(localstatedir, e_datadir, "/lib/suricata/data")
     EXPAND_VARIABLE(localstatedir, e_defaultruledir, "/lib/suricata/rules")
 
@@ -2489,6 +2491,8 @@ AC_SUBST(e_logcertsdir)
 AC_SUBST(e_sysconfdir)
 AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR])
 AC_SUBST(e_localstatedir)
+AC_SUBST(e_sghcachedir)
+AC_DEFINE_UNQUOTED([SGH_CACHE_DIR],["$e_sghcachedir"],[Directory path for signature group head cache])
 AC_SUBST(e_datadir)
 AC_DEFINE_UNQUOTED([DATA_DIR],["$e_datadir"],[Our DATA_DIR])
 AC_SUBST(e_magic_file)

doc/userguide/performance/hyperscan.rst

@@ -81,4 +81,28 @@ if it is present on the system in case of the "auto" setting.
 
 
 If the current suricata installation does not have hyperscan
-support, refer to :ref:`installation`
+support, refer to :ref:`installation`
+
+Hyperscan caching
+~~~~~~~~~~~~~~~~~
+
+Upon startup, Hyperscan compiles and optimizes the ruleset into its own
+internal structure. Suricata optimizes the startup process by saving
+the Hyperscan internal structures to disk and loading them on the next start.
+This prevents the recompilation of the ruleset and results in faster
+initialization. If the ruleset is changed, new necessary cache files are
+automatically created.
+
+To enable this function, in `suricata.yaml` configure:
+
+::
+
+  # Cache MPM contexts to the disk to avoid rule compilation at the startup.
+  # Cache files are created in the standard library directory.
+  sgh-mpm-caching: yes
+  sgh-mpm-caching-path: /var/lib/suricata/cache/hs
+
+
+**Note**:
+You might need to create and adjust permissions to the default caching folder
+path, especially if you are running Suricata as a non-root user.

src/Makefile.am

@@ -529,6 +529,8 @@ noinst_HEADERS = \
 	util-mpm-ac-ks.h \
 	util-mpm.h \
 	util-mpm-hs.h \
+	util-mpm-hs-cache.h \
+	util-mpm-hs-core.h \
 	util-optimize.h \
 	util-pages.h \
 	util-path.h \
@@ -1076,6 +1078,8 @@ libsuricata_c_a_SOURCES = \
 	util-mpm-ac-ks-small.c \
 	util-mpm.c \
 	util-mpm-hs.c \
+	util-mpm-hs-cache.c \
+	util-mpm-hs-core.c \
 	util-pages.c \
 	util-path.c \
 	util-pidfile.c \

src/app-layer-detect-proto.c

@@ -1297,7 +1297,7 @@ static int AppLayerProtoDetectPMPrepareMpm(AppLayerProtoDetectPMCtx *ctx)
     int ret = 0;
     MpmCtx *mpm_ctx = &ctx->mpm_ctx;
 
-    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx) < 0)
+    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx, false) < 0)
         goto error;
 
     goto end;

src/app-layer-ftp.c

@@ -1296,8 +1296,7 @@ static void FTPSetMpmState(void)
                        i /*  id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx);
-
+    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx, false);
 }
 
 static void FTPFreeMpmState(void)

src/app-layer-smtp.c

@@ -1636,7 +1636,7 @@ static void SMTPSetMpmState(void)
                         i /* pattern id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx);
+    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx, false);
 }
 
 static void SMTPFreeMpmState(void)

src/detect-engine-loader.c

@@ -33,6 +33,7 @@
 #include "tm-threads.h"
 #include "queue.h"
 
+#include "detect-engine.h"
 #include "detect-engine-loader.h"
 #include "detect-engine-build.h"
 #include "detect-engine-analyzer.h"
@@ -402,6 +403,10 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc
 
     ret = 0;
 
+    if (de_ctx->mpm_cache_to_disk && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
+        mpm_table[de_ctx->mpm_matcher].CacheRuleset();
+    }
+
  end:
     gettimeofday(&de_ctx->last_reload, NULL);
     if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) {

src/detect-engine-mpm.c

@@ -295,7 +295,7 @@ int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                 }
             }
         }
@@ -524,7 +524,7 @@ int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx)
             SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -689,7 +689,7 @@ int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -744,40 +744,40 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
     if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
@@ -1621,7 +1621,7 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
     } else {
         if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
             if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
-                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx);
+                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx, de_ctx->mpm_cache_to_disk);
             }
         }
     }

src/detect-engine.c

@@ -2481,6 +2481,35 @@ static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
     return -1;
 }
 
+static bool DetectEngineMpmCachingEnabled(void)
+{
+    const char *strval = NULL;
+    if (ConfGet("detect.sgh-mpm-caching", &strval) != 1)
+        return false;
+
+    int sgh_mpm_caching = 0;
+    (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching);
+    return (bool)sgh_mpm_caching;
+}
+
+const char *DetectEngineMpmCachingGetPath(void)
+{
+    char yamlpath[] = "detect.sgh-mpm-caching-path";
+    const char *strval = NULL;
+    ConfGet(yamlpath, &strval);
+
+    if (strval != NULL) {
+        return strval;
+    }
+
+    static bool notified = false;
+    if (!notified) {
+        SCLogInfo("%s has no path specified, using %s", yamlpath, SGH_CACHE_DIR);
+        notified = true;
+    }
+    return SGH_CACHE_DIR;
+}
+
 static DetectEngineCtx *DetectEngineCtxInitReal(
         enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
 {
@@ -2512,6 +2541,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
     de_ctx->failure_fatal = (failure_fatal == 1);
 
     de_ctx->mpm_matcher = PatternMatchDefaultMatcher();
+    de_ctx->mpm_cache_to_disk = DetectEngineMpmCachingEnabled();
     de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher();
     SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
         mpm_table[de_ctx->mpm_matcher].name,
@@ -3031,15 +3061,6 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     return 0;
 }
 
-/*
- * getting & (re)setting the internal sig i
- */
-
-//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx)
-//{
-//    return de_ctx->signum;
-//}
-
 void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
 {
     de_ctx->signum = 0;

src/detect-engine.h

@@ -100,7 +100,7 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i
 
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
-//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *);
+const char *DetectEngineMpmCachingGetPath(void);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
 void DetectEngineResetMaxSigId(DetectEngineCtx *);

src/detect.h

@@ -1050,6 +1050,9 @@ typedef struct DetectEngineCtx_ {
 
     /* number of signatures using filestore, limited as u16 */
     uint16_t filestore_cnt;
+
+    /** If enabled, MPM matchers can store compiled pattern databases to disk */
+    bool mpm_cache_to_disk;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */