diff --git a/Makefile.am b/Makefile.am index 20e50bdc4a03..aafaae0b862d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,6 +37,7 @@ install-conf: install -d "$(DESTDIR)$(e_rundir)" install -m 770 -d "$(DESTDIR)$(e_localstatedir)" install -m 770 -d "$(DESTDIR)$(e_datadir)" + install -m 660 -d "$(DESTDIR)$(e_sghcachedir)" install-rules: if INSTALL_SURICATA_UPDATE diff --git a/configure.ac b/configure.ac index ca964d9039a0..ccc4b85d7a6f 100644 --- a/configure.ac +++ b/configure.ac @@ -2455,6 +2455,7 @@ if test "$WINDOWS_PATH" = "yes"; then e_sysconfdir="${e_winbase}\\\\" e_defaultruledir="$e_winbase\\\\rules\\\\" + e_sghcachedir="$e_winbase\\\\cache\\\\sgh\\\\" e_magic_file="$e_winbase\\\\magic.mgc" e_logdir="$e_winbase\\\\log" e_logfilesdir="$e_logdir\\\\files" @@ -2476,6 +2477,7 @@ else EXPAND_VARIABLE(sysconfdir, e_sysconfdir, "/suricata/") EXPAND_VARIABLE(localstatedir, e_localstatedir, "/run/suricata") EXPAND_VARIABLE(datadir, e_datarulesdir, "/suricata/rules") + EXPAND_VARIABLE(localstatedir, e_sghcachedir, "/lib/suricata/cache/sgh") EXPAND_VARIABLE(localstatedir, e_datadir, "/lib/suricata/data") EXPAND_VARIABLE(localstatedir, e_defaultruledir, "/lib/suricata/rules") @@ -2489,6 +2491,8 @@ AC_SUBST(e_logcertsdir) AC_SUBST(e_sysconfdir) AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR]) AC_SUBST(e_localstatedir) +AC_SUBST(e_sghcachedir) +AC_DEFINE_UNQUOTED([SGH_CACHE_DIR],["$e_sghcachedir"],[Directory path for signature group head cache]) AC_SUBST(e_datadir) AC_DEFINE_UNQUOTED([DATA_DIR],["$e_datadir"],[Our DATA_DIR]) AC_SUBST(e_magic_file) diff --git a/doc/userguide/performance/hyperscan.rst b/doc/userguide/performance/hyperscan.rst index 055fa7f21b75..9b39bc22e95b 100644 --- a/doc/userguide/performance/hyperscan.rst +++ b/doc/userguide/performance/hyperscan.rst @@ -81,4 +81,28 @@ if it is present on the system in case of the "auto" setting. If the current suricata installation does not have hyperscan -support, refer to :ref:`installation` \ No newline at end of file +support, refer to :ref:`installation` + +Hyperscan caching +~~~~~~~~~~~~~~~~~ + +Upon startup, Hyperscan compiles and optimizes the ruleset into its own +internal structure. Suricata optimizes the startup process by saving +the Hyperscan internal structures to disk and loading them on the next start. +This prevents the recompilation of the ruleset and results in faster +initialization. If the ruleset is changed, new necessary cache files are +automatically created. + +To enable this function, in `suricata.yaml` configure: + +:: + + # Cache MPM contexts to the disk to avoid rule compilation at the startup. + # Cache files are created in the standard library directory. + sgh-mpm-caching: yes + sgh-mpm-caching-path: /var/lib/suricata/cache/hs + + +**Note**: +You might need to create and adjust permissions to the default caching folder +path, especially if you are running Suricata as a non-root user. diff --git a/src/Makefile.am b/src/Makefile.am index 82155d7f825d..56e761a5c2e9 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -529,6 +529,8 @@ noinst_HEADERS = \ util-mpm-ac-ks.h \ util-mpm.h \ util-mpm-hs.h \ + util-mpm-hs-cache.h \ + util-mpm-hs-core.h \ util-optimize.h \ util-pages.h \ util-path.h \ @@ -1076,6 +1078,8 @@ libsuricata_c_a_SOURCES = \ util-mpm-ac-ks-small.c \ util-mpm.c \ util-mpm-hs.c \ + util-mpm-hs-cache.c \ + util-mpm-hs-core.c \ util-pages.c \ util-path.c \ util-pidfile.c \ diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index 52e1f2922c02..c4d6cc7eb268 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -1297,7 +1297,7 @@ static int AppLayerProtoDetectPMPrepareMpm(AppLayerProtoDetectPMCtx *ctx) int ret = 0; MpmCtx *mpm_ctx = &ctx->mpm_ctx; - if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx) < 0) + if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx, false) < 0) goto error; goto end; diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index a6a1f632fcb4..fd9b0fc0919f 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -1296,8 +1296,7 @@ static void FTPSetMpmState(void) i /* id */, i /* rule id */ , 0 /* no flags */); } - mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx); - + mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx, false); } static void FTPFreeMpmState(void) diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index b2bf22a53145..dcb79d2bc657 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -1636,7 +1636,7 @@ static void SMTPSetMpmState(void) i /* pattern id */, i /* rule id */ , 0 /* no flags */); } - mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx); + mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx, false); } static void SMTPFreeMpmState(void) diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index 950812a187c9..82a3d3463cc8 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -33,6 +33,7 @@ #include "tm-threads.h" #include "queue.h" +#include "detect-engine.h" #include "detect-engine-loader.h" #include "detect-engine-build.h" #include "detect-engine-analyzer.h" @@ -402,6 +403,10 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc ret = 0; + if (de_ctx->mpm_cache_to_disk && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheRuleset(); + } + end: gettimeofday(&de_ctx->last_reload, NULL); if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) { diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 1c9984ea9541..8adf02ab2778 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -295,7 +295,7 @@ int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx) MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } } @@ -524,7 +524,7 @@ int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx) SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); SCLogDebug("%s: %d", am->name, r); } } @@ -689,7 +689,7 @@ int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx) MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); SCLogDebug("%s: %d", am->name, r); } } @@ -744,40 +744,40 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx) if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } @@ -1621,7 +1621,7 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms) } else { if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) { if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) { - mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx); + mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx, de_ctx->mpm_cache_to_disk); } } } diff --git a/src/detect-engine.c b/src/detect-engine.c index d77aa1a8db0a..7354c4afe63a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2481,6 +2481,35 @@ static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx) return -1; } +static bool DetectEngineMpmCachingEnabled(void) +{ + const char *strval = NULL; + if (ConfGet("detect.sgh-mpm-caching", &strval) != 1) + return false; + + int sgh_mpm_caching = 0; + (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching); + return (bool)sgh_mpm_caching; +} + +const char *DetectEngineMpmCachingGetPath(void) +{ + char yamlpath[] = "detect.sgh-mpm-caching-path"; + const char *strval = NULL; + ConfGet(yamlpath, &strval); + + if (strval != NULL) { + return strval; + } + + static bool notified = false; + if (!notified) { + SCLogInfo("%s has no path specified, using %s", yamlpath, SGH_CACHE_DIR); + notified = true; + } + return SGH_CACHE_DIR; +} + static DetectEngineCtx *DetectEngineCtxInitReal( enum DetectEngineType type, const char *prefix, uint32_t tenant_id) { @@ -2512,6 +2541,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal( de_ctx->failure_fatal = (failure_fatal == 1); de_ctx->mpm_matcher = PatternMatchDefaultMatcher(); + de_ctx->mpm_cache_to_disk = DetectEngineMpmCachingEnabled(); de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher(); SCLogConfig("pattern matchers: MPM: %s, SPM: %s", mpm_table[de_ctx->mpm_matcher].name, @@ -3031,15 +3061,6 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) return 0; } -/* - * getting & (re)setting the internal sig i - */ - -//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx) -//{ -// return de_ctx->signum; -//} - void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx) { de_ctx->signum = 0; diff --git a/src/detect-engine.h b/src/detect-engine.h index b75d124f9cd4..f52b6d3df05b 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -100,7 +100,7 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); -//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *); +const char *DetectEngineMpmCachingGetPath(void); /* faster as a macro than a inline function on my box -- VJ */ #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum) void DetectEngineResetMaxSigId(DetectEngineCtx *); diff --git a/src/detect.h b/src/detect.h index ea81092487b5..3f1714135afe 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1050,6 +1050,9 @@ typedef struct DetectEngineCtx_ { /* number of signatures using filestore, limited as u16 */ uint16_t filestore_cnt; + + /** If enabled, MPM matchers can store compiled pattern databases to disk */ + bool mpm_cache_to_disk; } DetectEngineCtx; /* Engine groups profiles (low, medium, high, custom) */ diff --git a/src/util-hash-lookup3.c b/src/util-hash-lookup3.c index 2354c183d219..c2b0fe275828 100644 --- a/src/util-hash-lookup3.c +++ b/src/util-hash-lookup3.c @@ -805,7 +805,211 @@ void hashlittle2( *pc=c; *pb=b; } +/* + * hashlittle2: return 2 32-bit hash values + * + * This is identical to hashlittle(), except it returns two 32-bit hash + * values instead of just one. This is good enough for hash table + * lookup with 2^^64 buckets, or if you want a second hash if you're not + * happy with the first, or if you want a probably-unique 64-bit ID for + * the key. *pc is better mixed than *pb, so use *pc first. If you want + * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)". + */ +void hashlittle2_safe(const void *key, /* the key to hash */ + size_t length, /* length of the key */ + uint32_t *pc, /* IN: primary initval, OUT: primary hash */ + uint32_t *pb) /* IN: secondary initval, OUT: secondary hash */ +{ + uint32_t a, b, c; /* internal state */ + union { + const void *ptr; + size_t i; + } u; /* needed for Mac Powerbook G4 */ + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t)length) + *pc; + c += *pb; + + u.ptr = key; + if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { + const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length > 12) { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a, b, c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * Note that unlike hashlittle() above, we use the "safe" version of this + * block that is #ifdef VALGRIND above, in order to avoid warnings from + * Valgrind or Address Sanitizer. + */ + const uint8_t *k8 = (const uint8_t *)k; + switch (length) { + case 12: + c += k[2]; + b += k[1]; + a += k[0]; + break; + case 11: + c += ((uint32_t)k8[10]) << 16; /* fall through */ + case 10: + c += ((uint32_t)k8[9]) << 8; /* fall through */ + case 9: + c += k8[8]; /* fall through */ + case 8: + b += k[1]; + a += k[0]; + break; + case 7: + b += ((uint32_t)k8[6]) << 16; /* fall through */ + case 6: + b += ((uint32_t)k8[5]) << 8; /* fall through */ + case 5: + b += k8[4]; /* fall through */ + case 4: + a += k[0]; + break; + case 3: + a += ((uint32_t)k8[2]) << 16; /* fall through */ + case 2: + a += ((uint32_t)k8[1]) << 8; /* fall through */ + case 1: + a += k8[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { + const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ + const uint8_t *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length > 12) { + a += k[0] + (((uint32_t)k[1]) << 16); + b += k[2] + (((uint32_t)k[3]) << 16); + c += k[4] + (((uint32_t)k[5]) << 16); + mix(a, b, c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const uint8_t *)k; + switch (length) { + case 12: + c += k[4] + (((uint32_t)k[5]) << 16); + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 11: + c += ((uint32_t)k8[10]) << 16; /* fall through */ + case 10: + c += k[4]; + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 9: + c += k8[8]; /* fall through */ + case 8: + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 7: + b += ((uint32_t)k8[6]) << 16; /* fall through */ + case 6: + b += k[2]; + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 5: + b += k8[4]; /* fall through */ + case 4: + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 3: + a += ((uint32_t)k8[2]) << 16; /* fall through */ + case 2: + a += k[0]; + break; + case 1: + a += k8[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + + } else { /* need to read the key one byte at a time */ + const uint8_t *k = (const uint8_t *)key; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) { + a += k[0]; + a += ((uint32_t)k[1]) << 8; + a += ((uint32_t)k[2]) << 16; + a += ((uint32_t)k[3]) << 24; + b += k[4]; + b += ((uint32_t)k[5]) << 8; + b += ((uint32_t)k[6]) << 16; + b += ((uint32_t)k[7]) << 24; + c += k[8]; + c += ((uint32_t)k[9]) << 8; + c += ((uint32_t)k[10]) << 16; + c += ((uint32_t)k[11]) << 24; + mix(a, b, c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch (length) /* all the case statements fall through */ + { + case 12: + c += ((uint32_t)k[11]) << 24; /* fall through */ + case 11: + c += ((uint32_t)k[10]) << 16; /* fall through */ + case 10: + c += ((uint32_t)k[9]) << 8; /* fall through */ + case 9: + c += k[8]; /* fall through */ + case 8: + b += ((uint32_t)k[7]) << 24; /* fall through */ + case 7: + b += ((uint32_t)k[6]) << 16; /* fall through */ + case 6: + b += ((uint32_t)k[5]) << 8; /* fall through */ + case 5: + b += k[4]; /* fall through */ + case 4: + a += ((uint32_t)k[3]) << 24; /* fall through */ + case 3: + a += ((uint32_t)k[2]) << 16; /* fall through */ + case 2: + a += ((uint32_t)k[1]) << 8; /* fall through */ + case 1: + a += k[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + } + final(a, b, c); + *pc = c; + *pb = b; +} /* * hashbig(): diff --git a/src/util-hash-lookup3.h b/src/util-hash-lookup3.h index 62d3d14270b1..ab7f6d3d88f3 100644 --- a/src/util-hash-lookup3.h +++ b/src/util-hash-lookup3.h @@ -62,6 +62,11 @@ void hashlittle2(const void *key, /* the key to hash */ uint32_t *pc, /* IN: primary initval, OUT: primary hash */ uint32_t *pb); /* IN: secondary initval, OUT: secondary hash */ +/* A variant of hashlittle2() that ensures avoids accesses beyond the last byte + * of the string, which will cause warnings from tools like Valgrind or Address + * Sanitizer. */ +void hashlittle2_safe(const void *key, size_t length, uint32_t *pc, uint32_t *pb); + uint32_t hashbig( const void *key, size_t length, uint32_t initval); #endif /* SURICATA_UTIL_HASH_LOOKUP3_H */ diff --git a/src/util-hash.c b/src/util-hash.c index ea04f2d7bb94..bdfc1c6f74ff 100644 --- a/src/util-hash.c +++ b/src/util-hash.c @@ -208,6 +208,21 @@ void *HashTableLookup(HashTable *ht, void *data, uint16_t datalen) return NULL; } +// CallbackFn is an iterator, first argument is the data, second is user auxilary data +void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux) +{ + if (ht == NULL || CallbackFn == NULL) + return; + + for (uint32_t i = 0; i < ht->array_size; i++) { + HashTableBucket *hashbucket = ht->array[i]; + while (hashbucket != NULL) { + CallbackFn(hashbucket->data, aux); + hashbucket = hashbucket->next; + } + } +} + uint32_t HashTableGenericHash(HashTable *ht, void *data, uint16_t datalen) { uint8_t *d = (uint8_t *)data; diff --git a/src/util-hash.h b/src/util-hash.h index ec96f5bcac71..40241b3db71a 100644 --- a/src/util-hash.h +++ b/src/util-hash.h @@ -51,6 +51,7 @@ void HashTableFree(HashTable *); int HashTableAdd(HashTable *, void *, uint16_t); int HashTableRemove(HashTable *, void *, uint16_t); void *HashTableLookup(HashTable *, void *, uint16_t); +void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux); uint32_t HashTableGenericHash(HashTable *, void *, uint16_t); char HashTableDefaultCompare(void *, uint16_t, void *, uint16_t); diff --git a/src/util-mpm-ac-ks.c b/src/util-mpm-ac-ks.c index 60bb1b7a9e42..6eeff3e3a29c 100644 --- a/src/util-mpm-ac-ks.c +++ b/src/util-mpm-ac-ks.c @@ -90,7 +90,7 @@ int SCACTileAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCACTileAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCACTilePreparePatterns(MpmCtx *mpm_ctx); +int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCACTileSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, uint32_t buflen); @@ -861,7 +861,7 @@ static void SCACTilePrepareSearch(MpmCtx *mpm_ctx) * * \param mpm_ctx Pointer to the mpm context. */ -int SCACTilePreparePatterns(MpmCtx *mpm_ctx) +int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) { SCACTileSearchCtx *search_ctx = (SCACTileSearchCtx *)mpm_ctx->ctx; @@ -1403,6 +1403,7 @@ void MpmACTileRegister(void) mpm_table[MPM_AC_KS].AddPattern = SCACTileAddPatternCS; mpm_table[MPM_AC_KS].AddPatternNocase = SCACTileAddPatternCI; mpm_table[MPM_AC_KS].Prepare = SCACTilePreparePatterns; + mpm_table[MPM_AC_KS].CacheRuleset = NULL; mpm_table[MPM_AC_KS].Search = SCACTileSearch; mpm_table[MPM_AC_KS].PrintCtx = SCACTilePrintInfo; #ifdef UNITTESTS @@ -1432,7 +1433,7 @@ static int SCACTileTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1464,7 +1465,7 @@ static int SCACTileTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1499,7 +1500,7 @@ static int SCACTileTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1531,7 +1532,7 @@ static int SCACTileTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1563,7 +1564,7 @@ static int SCACTileTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1593,7 +1594,7 @@ static int SCACTileTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcd"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1635,7 +1636,7 @@ static int SCACTileTest07(void) PmqSetup(&pmq); /* total matches: 135: 6 unique */ - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1662,7 +1663,7 @@ static int SCACTileTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"a", 1); @@ -1692,7 +1693,7 @@ static int SCACTileTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"ab", 2); @@ -1722,7 +1723,7 @@ static int SCACTileTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" @@ -1763,7 +1764,7 @@ static int SCACTileTest11(void) goto end; PmqSetup(&pmq); - if (SCACTilePreparePatterns(&mpm_ctx) == -1) + if (SCACTilePreparePatterns(&mpm_ctx, false) == -1) goto end; result = 1; @@ -1804,7 +1805,7 @@ static int SCACTileTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1836,7 +1837,7 @@ static int SCACTileTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1868,7 +1869,7 @@ static int SCACTileTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1900,7 +1901,7 @@ static int SCACTileTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1932,7 +1933,7 @@ static int SCACTileTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1964,7 +1965,7 @@ static int SCACTileTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2001,7 +2002,7 @@ static int SCACTileTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2033,7 +2034,7 @@ static int SCACTileTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2071,7 +2072,7 @@ static int SCACTileTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2102,7 +2103,7 @@ static int SCACTileTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"AA", 2); @@ -2134,7 +2135,7 @@ static int SCACTileTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2165,7 +2166,7 @@ static int SCACTileTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -2195,7 +2196,7 @@ static int SCACTileTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -2226,7 +2227,7 @@ static int SCACTileTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2257,7 +2258,7 @@ static int SCACTileTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "works"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2288,7 +2289,7 @@ static int SCACTileTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "tone"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2319,7 +2320,7 @@ static int SCACTileTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "tONE"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c index 6e115acaa305..c8c01442ea73 100644 --- a/src/util-mpm-ac.c +++ b/src/util-mpm-ac.c @@ -68,7 +68,7 @@ int SCACAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCACAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCACPreparePatterns(MpmCtx *mpm_ctx); +int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCACSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, uint32_t buflen); void SCACPrintInfo(MpmCtx *mpm_ctx); @@ -702,7 +702,7 @@ static void SCACPrepareStateTable(MpmCtx *mpm_ctx) * * \param mpm_ctx Pointer to the mpm context. */ -int SCACPreparePatterns(MpmCtx *mpm_ctx) +int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) { SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx; @@ -1102,6 +1102,7 @@ void MpmACRegister(void) mpm_table[MPM_AC].AddPattern = SCACAddPatternCS; mpm_table[MPM_AC].AddPatternNocase = SCACAddPatternCI; mpm_table[MPM_AC].Prepare = SCACPreparePatterns; + mpm_table[MPM_AC].CacheRuleset = NULL; mpm_table[MPM_AC].Search = SCACSearch; mpm_table[MPM_AC].PrintCtx = SCACPrintInfo; #ifdef UNITTESTS @@ -1130,7 +1131,7 @@ static int SCACTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1162,7 +1163,7 @@ static int SCACTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1197,7 +1198,7 @@ static int SCACTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1229,7 +1230,7 @@ static int SCACTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1261,7 +1262,7 @@ static int SCACTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1291,7 +1292,7 @@ static int SCACTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcd"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1333,7 +1334,7 @@ static int SCACTest07(void) PmqSetup(&pmq); /* total matches: 135: unique matches: 6 */ - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1360,7 +1361,7 @@ static int SCACTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"a", 1); @@ -1390,7 +1391,7 @@ static int SCACTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"ab", 2); @@ -1420,7 +1421,7 @@ static int SCACTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" @@ -1461,7 +1462,7 @@ static int SCACTest11(void) goto end; PmqSetup(&pmq); - if (SCACPreparePatterns(&mpm_ctx) == -1) + if (SCACPreparePatterns(&mpm_ctx, false) == -1) goto end; result = 1; @@ -1502,7 +1503,7 @@ static int SCACTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1534,7 +1535,7 @@ static int SCACTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1566,7 +1567,7 @@ static int SCACTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1598,7 +1599,7 @@ static int SCACTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1630,7 +1631,7 @@ static int SCACTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1662,7 +1663,7 @@ static int SCACTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1699,7 +1700,7 @@ static int SCACTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1731,7 +1732,7 @@ static int SCACTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1769,7 +1770,7 @@ static int SCACTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1800,7 +1801,7 @@ static int SCACTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"AA", 2); @@ -1832,7 +1833,7 @@ static int SCACTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1863,7 +1864,7 @@ static int SCACTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -1893,7 +1894,7 @@ static int SCACTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -1924,7 +1925,7 @@ static int SCACTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1955,7 +1956,7 @@ static int SCACTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "works"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1986,7 +1987,7 @@ static int SCACTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "tone"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2016,7 +2017,7 @@ static int SCACTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "tONE"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2083,7 +2084,7 @@ static int SCACTest30(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"xyz", 3, 0, 0, 0, 0, MPM_PATTERN_FLAG_ENDSWITH); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf1 = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf1, strlen(buf1)); diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c new file mode 100644 index 000000000000..83107385696b --- /dev/null +++ b/src/util-mpm-hs-cache.c @@ -0,0 +1,255 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Lukas Sismis + * + * MPM pattern matcher that calls the Hyperscan regex matcher. + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "detect-engine.h" +#include "util-debug.h" +#include "util-hash-lookup3.h" +#include "util-mpm-hs-core.h" +#include "util-mpm-hs-cache.h" +#include "util-path.h" + +#ifdef BUILD_HYPERSCAN + +#include + +static const char *HSCacheConstructFPath(uint64_t hs_db_hash) +{ + static char hash_file_path[PATH_MAX]; + + char hash_file_path_suffix[] = "_v1.hs"; + char filename[PATH_MAX]; + uint64_t r = + snprintf(filename, sizeof(filename), "%020lu%s", hs_db_hash, hash_file_path_suffix); + if (r != (uint64_t)(20 + strlen(hash_file_path_suffix))) + return NULL; + + r = PathMerge( + hash_file_path, sizeof(hash_file_path), DetectEngineMpmCachingGetPath(), filename); + if (r) + return NULL; + + return hash_file_path; +} + +static char *HSReadStream(const char *file_path, size_t *buffer_sz) +{ + FILE *file = fopen(file_path, "rb"); + if (!file) { + SCLogDebug("Failed to open file %s: %s", file_path, strerror(errno)); + return NULL; + } + + // Seek to the end of the file to determine its size + fseek(file, 0, SEEK_END); + long file_sz = ftell(file); + if (file_sz < 0) { + SCLogDebug("Failed to determine file size of %s: %s", file_path, strerror(errno)); + fclose(file); + return NULL; + } + + char *buffer = (char *)SCCalloc(file_sz, sizeof(char)); + if (!buffer) { + SCLogWarning("Failed to allocate memory"); + fclose(file); + return NULL; + } + + // Rewind file pointer and read the file into the buffer + rewind(file); + size_t bytes_read = fread(buffer, 1, file_sz, file); + if (bytes_read != (size_t)file_sz) { + SCLogDebug("Failed to read the entire file %s: %s", file_path, strerror(errno)); + SCFree(buffer); + fclose(file); + return NULL; + } + + *buffer_sz = file_sz; + fclose(file); + return buffer; +} + +/** + * Function to hash the searched pattern, only things relevant to Hyperscan + * compilation are hashed. + */ +static void SCHSCachePatternHash(const SCHSPattern *p, uint32_t *h1, uint32_t *h2) +{ + BUG_ON(p->original_pat == NULL); + hashlittle2_safe(&p->len, sizeof(p->len), h1, h2); + hashlittle2_safe(&p->flags, sizeof(p->flags), h1, h2); + hashlittle2_safe(p->original_pat, p->len, h1, h2); + hashlittle2_safe(&p->offset, sizeof(p->offset), h1, h2); + hashlittle2_safe(&p->depth, sizeof(p->depth), h1, h2); +} + +int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash) +{ + const char *hash_file_static = HSCacheConstructFPath(hs_db_hash); + if (hash_file_static == NULL) + return -1; + + SCLogDebug("Loading the cached HS DB from %s", hash_file_static); + if (!SCPathExists(hash_file_static)) + return -1; + + FILE *db_cache = fopen(hash_file_static, "r"); + char *buffer = NULL; + int ret = 0; + if (db_cache) { + size_t buffer_size; + buffer = HSReadStream(hash_file_static, &buffer_size); + if (!buffer) { + SCLogWarning("Hyperscan cached DB file %s cannot be read", hash_file_static); + ret = -1; + goto freeup; + } + + hs_error_t error = hs_deserialize_database(buffer, buffer_size, hs_db); + if (error != HS_SUCCESS) { + SCLogWarning("Failed to deserialize Hyperscan database of %s: %s", hash_file_static, + HSErrorToStr(error)); + ret = -1; + goto freeup; + } + + ret = 0; + goto freeup; + } + +freeup: + if (db_cache) + fclose(db_cache); + if (buffer) + SCFree(buffer); + return ret; +} + +static int HSSaveCache(hs_database_t *hs_db, uint64_t hs_db_hash) +{ + static bool notified = false; + char *db_stream = NULL; + size_t db_size; + int ret = -1; + + hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size); + if (err != HS_SUCCESS) { + SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err)); + goto cleanup; + } + + const char *hash_file_static = HSCacheConstructFPath(hs_db_hash); + SCLogDebug("Caching the compiled HS at %s", hash_file_static); + if (SCPathExists(hash_file_static)) { + // potentially signs that it might not work as expected as we got into + // hash collision. If this happens with older and not used caches it is + // fine. + // It is problematic when one ruleset yields two colliding MPM groups. + SCLogWarning("Overwriting cache file %s. If the problem persists consider switching off " + "the caching", + hash_file_static); + } + + if (SCCreateDirectoryTree(DetectEngineMpmCachingGetPath(), true) != 0) { + if (!notified) { + SCLogWarning("Failed to create Hyperscan cache folder, make sure " + "the parent folder is writeable " + "or adjust sgh-mpm-caching-path setting (%s)", + DetectEngineMpmCachingGetPath()); + notified = true; + } + return -1; + } + + FILE *db_cache_out = fopen(hash_file_static, "w"); + if (!db_cache_out) { + if (!notified) { + SCLogWarning("Failed to create Hyperscan cache file, make sure the folder exist and is " + "writable or adjust sgh-mpm-caching-path setting (%s)", + hash_file_static); + notified = true; + } + goto cleanup; + } + size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out); + if (r > 0 && (size_t)r != db_size) { + SCLogWarning("Failed to write to file: %s", hash_file_static); + if (r != db_size) { + // possibly a corrupted DB cache was created + r = remove(hash_file_static); + if (r != 0) { + SCLogWarning("Failed to remove corrupted cache file: %s", hash_file_static); + } + } + } + ret = fclose(db_cache_out); + if (ret != 0) { + SCLogWarning("Failed to close file: %s", hash_file_static); + goto cleanup; + } + + ret = 0; +cleanup: + if (db_stream) + SCFree(db_stream); + return ret; +} + +uint64_t HSHashDb(const PatternDatabase *pd) +{ + uint64_t cached_hash = 0; + uint32_t *hash = (uint32_t *)(&cached_hash); + hashword2(&pd->pattern_cnt, 1, &hash[0], &hash[1]); + for (uint32_t i = 0; i < pd->pattern_cnt; i++) { + SCHSCachePatternHash(pd->parray[i], &hash[0], &hash[1]); + } + + return cached_hash; +} + +void HSSaveCacheIterator(void *data, void *aux) +{ + PatternDatabase *pd = (PatternDatabase *)data; + PatternDatabaseCache *pd_stats = (PatternDatabaseCache *)aux; + if (pd->no_cache) + return; + + // count only cacheable DBs + pd_stats->hs_cacheable_dbs_cnt++; + if (pd->cached) { + pd_stats->hs_dbs_cache_loaded_cnt++; + return; + } + + if (HSSaveCache(pd->hs_db, HSHashDb(pd)) == 0) { + pd->cached = true; // for rule reloads + pd_stats->hs_dbs_cache_saved_cnt++; + } +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-cache.h b/src/util-mpm-hs-cache.h new file mode 100644 index 000000000000..cc17d9846cc7 --- /dev/null +++ b/src/util-mpm-hs-cache.h @@ -0,0 +1,37 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Lukas Sismis + * + * Hyperscan caching logic for faster database compilation. + */ + +#ifndef SURICATA_UTIL_MPM_HS_CACHE__H +#define SURICATA_UTIL_MPM_HS_CACHE__H + +#include "util-mpm-hs-core.h" + +#ifdef BUILD_HYPERSCAN +int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash); +uint64_t HSHashDb(const PatternDatabase *pd); +void HSSaveCacheIterator(void *data, void *aux); +#endif /* BUILD_HYPERSCAN */ + +#endif /* SURICATA_UTIL_MPM_HS_CACHE__H */ diff --git a/src/util-mpm-hs-core.c b/src/util-mpm-hs-core.c new file mode 100644 index 000000000000..b8663ae220d2 --- /dev/null +++ b/src/util-mpm-hs-core.c @@ -0,0 +1,83 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jim Xu + * \author Justin Viiret + * \author Lukas Sismis + * + * MPM pattern matcher core function for the Hyperscan regex matcher. + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "util-mpm-hs-core.h" + +#ifdef BUILD_HYPERSCAN + +#include + +/** + * Translates Hyperscan error codes to human-readable messages. + * + * \param error_code + * The error code returned by a Hyperscan function. + * \return + * A string describing the error. + */ +const char *HSErrorToStr(hs_error_t error_code) +{ + switch (error_code) { + case HS_SUCCESS: + return "HS_SUCCESS: The engine completed normally"; + case HS_INVALID: + return "HS_INVALID: A parameter passed to this function was invalid"; + case HS_NOMEM: + return "HS_NOMEM: A memory allocation failed"; + case HS_SCAN_TERMINATED: + return "HS_SCAN_TERMINATED: The engine was terminated by callback"; + case HS_COMPILER_ERROR: + return "HS_COMPILER_ERROR: The pattern compiler failed"; + case HS_DB_VERSION_ERROR: + return "HS_DB_VERSION_ERROR: The given database was built for a different version of " + "Hyperscan"; + case HS_DB_PLATFORM_ERROR: + return "HS_DB_PLATFORM_ERROR: The given database was built for a different platform " + "(i.e., CPU type)"; + case HS_DB_MODE_ERROR: + return "HS_DB_MODE_ERROR: The given database was built for a different mode of " + "operation"; + case HS_BAD_ALIGN: + return "HS_BAD_ALIGN: A parameter passed to this function was not correctly aligned"; + case HS_BAD_ALLOC: + return "HS_BAD_ALLOC: The memory allocator did not return correctly aligned memory"; + case HS_SCRATCH_IN_USE: + return "HS_SCRATCH_IN_USE: The scratch region was already in use"; + case HS_ARCH_ERROR: + return "HS_ARCH_ERROR: Unsupported CPU architecture"; + case HS_INSUFFICIENT_SPACE: + return "HS_INSUFFICIENT_SPACE: Provided buffer was too small"; + case HS_UNKNOWN_ERROR: + return "HS_UNKNOWN_ERROR: Unexpected internal error"; + default: + return "Unknown error code"; + } +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h new file mode 100644 index 000000000000..699dd69568a3 --- /dev/null +++ b/src/util-mpm-hs-core.h @@ -0,0 +1,101 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jim Xu + * \author Justin Viiret + * \author Lukas Sismis + * + * MPM pattern matcher core function for the Hyperscan regex matcher. + */ + +#ifndef SURICATA_UTIL_MPM_HS_CORE__H +#define SURICATA_UTIL_MPM_HS_CORE__H + +#include "suricata-common.h" +#include "suricata.h" + +#ifdef BUILD_HYPERSCAN +#include + +typedef struct SCHSPattern_ { + /** length of the pattern */ + uint16_t len; + /** flags describing the pattern */ + uint8_t flags; + /** holds the original pattern that was added */ + uint8_t *original_pat; + /** pattern id */ + uint32_t id; + + uint16_t offset; + uint16_t depth; + + /** sid(s) for this pattern */ + uint32_t sids_size; + SigIntId *sids; + + /** only used at ctx init time, when this structure is part of a hash + * table. */ + struct SCHSPattern_ *next; +} SCHSPattern; + +typedef struct SCHSCtx_ { + /** hash used during ctx initialization */ + SCHSPattern **init_hash; + + /** pattern database and pattern arrays. */ + void *pattern_db; + + /** size of database, for accounting. */ + size_t hs_db_size; +} SCHSCtx; + +typedef struct SCHSThreadCtx_ { + /** Hyperscan scratch space region for this thread, capable of handling any + * database that has been compiled. */ + void *scratch; + + /** size of scratch space, for accounting. */ + size_t scratch_size; +} SCHSThreadCtx; + +typedef struct PatternDatabase_ { + SCHSPattern **parray; + hs_database_t *hs_db; + uint32_t pattern_cnt; + + /** Reference count: number of MPM contexts using this pattern database. */ + uint32_t ref_cnt; + /** Signals if the matcher has loaded/saved the pattern database to disk */ + bool cached; + /** Matcher will not cache this pattern DB */ + bool no_cache; +} PatternDatabase; + +typedef struct PatternDatabaseCache_ { + uint32_t hs_cacheable_dbs_cnt; + uint32_t hs_dbs_cache_loaded_cnt; + uint32_t hs_dbs_cache_saved_cnt; +} PatternDatabaseCache; + +const char *HSErrorToStr(hs_error_t error_code); + +#endif /* BUILD_HYPERSCAN */ +#endif /* SURICATA_UTIL_MPM_HS_CORE__H */ diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index 82b91b4ec0e8..7d6dc44736f5 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -33,15 +33,19 @@ #include "detect-engine-build.h" #include "conf.h" +#include "util-conf.h" #include "util-debug.h" #include "util-unittest.h" #include "util-unittest-helper.h" #include "util-memcmp.h" #include "util-mpm-hs.h" +#include "util-mpm-hs-cache.h" +#include "util-mpm-hs-core.h" #include "util-memcpy.h" #include "util-hash.h" #include "util-hash-lookup3.h" #include "util-hyperscan.h" +#include "util-path.h" #ifdef BUILD_HYPERSCAN @@ -55,7 +59,7 @@ int SCHSAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCHSAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCHSPreparePatterns(MpmCtx *mpm_ctx); +int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCHSSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, const uint32_t buflen); void SCHSPrintInfo(MpmCtx *mpm_ctx); @@ -379,7 +383,7 @@ typedef struct SCHSCompileData_ { unsigned int pattern_cnt; } SCHSCompileData; -static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt) +static SCHSCompileData *CompileDataAlloc(unsigned int pattern_cnt) { SCHSCompileData *cd = SCCalloc(pattern_cnt, sizeof(SCHSCompileData)); if (cd == NULL) { @@ -422,7 +426,7 @@ static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt) return NULL; } -static void SCHSFreeCompileData(SCHSCompileData *cd) +static void CompileDataFree(SCHSCompileData *cd) { if (cd == NULL) { return; @@ -445,15 +449,6 @@ static void SCHSFreeCompileData(SCHSCompileData *cd) SCFree(cd); } -typedef struct PatternDatabase_ { - SCHSPattern **parray; - hs_database_t *hs_db; - uint32_t pattern_cnt; - - /* Reference count: number of MPM contexts using this pattern database. */ - uint32_t ref_cnt; -} PatternDatabase; - static uint32_t SCHSPatternHash(const SCHSPattern *p, uint32_t hash) { BUG_ON(p->original_pat == NULL); @@ -559,6 +554,7 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt) pd->pattern_cnt = pattern_cnt; pd->ref_cnt = 0; pd->hs_db = NULL; + pd->cached = false; /* alloc the pattern array */ pd->parray = (SCHSPattern **)SCCalloc(pd->pattern_cnt, sizeof(SCHSPattern *)); @@ -570,38 +566,20 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt) return pd; } -/** - * \brief Process the patterns added to the mpm, and create the internal tables. - * - * \param mpm_ctx Pointer to the mpm context. - */ -int SCHSPreparePatterns(MpmCtx *mpm_ctx) +static int HSCheckPatterns(MpmCtx *mpm_ctx, SCHSCtx *ctx) { - SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx; - if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) { SCLogDebug("no patterns supplied to this mpm_ctx"); return 0; } + return 1; +} - hs_error_t err; - hs_compile_error_t *compile_err = NULL; - SCHSCompileData *cd = NULL; - PatternDatabase *pd = NULL; - - cd = SCHSAllocCompileData(mpm_ctx->pattern_cnt); - if (cd == NULL) { - goto error; - } - - pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt); - if (pd == NULL) { - goto error; - } - - /* populate the pattern array with the patterns in the hash */ +static void HSPatternArrayPopulate(SCHSCtx *ctx, PatternDatabase *pd) +{ for (uint32_t i = 0, p = 0; i < INIT_HASH_SIZE; i++) { - SCHSPattern *node = ctx->init_hash[i], *nnode = NULL; + SCHSPattern *node = ctx->init_hash[i]; + SCHSPattern *nnode = NULL; while (node != NULL) { nnode = node->next; node->next = NULL; @@ -609,105 +587,228 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx) node = nnode; } } +} +static void HSPatternArrayInit(SCHSCtx *ctx, PatternDatabase *pd) +{ + HSPatternArrayPopulate(ctx, pd); /* we no longer need the hash, so free its memory */ SCFree(ctx->init_hash); ctx->init_hash = NULL; +} - /* Serialise whole database compilation as a relatively easy way to ensure - * dedupe is safe. */ - SCMutexLock(&g_db_table_mutex); - - /* Init global pattern database hash if necessary. */ +static int HSGlobalPatternDatabaseInit(void) +{ if (g_db_table == NULL) { g_db_table = HashTableInit(INIT_DB_HASH_SIZE, PatternDatabaseHash, PatternDatabaseCompare, PatternDatabaseTableFree); if (g_db_table == NULL) { - SCMutexUnlock(&g_db_table_mutex); - goto error; + return -1; } } + return 0; +} + +static void HSLogCompileError(hs_compile_error_t *compile_err) +{ + SCLogError("failed to compile hyperscan database"); + if (compile_err) { + SCLogError("compile error: %s", compile_err->message); + hs_free_compile_error(compile_err); + } +} +static int HSScratchAlloc(const hs_database_t *db) +{ + SCMutexLock(&g_scratch_proto_mutex); + hs_error_t err = hs_alloc_scratch(db, &g_scratch_proto); + SCMutexUnlock(&g_scratch_proto_mutex); + if (err != HS_SUCCESS) { + SCLogError("failed to allocate scratch"); + return -1; + } + return 0; +} + +static int PatternDatabaseGetSize(PatternDatabase *pd, size_t *db_size) +{ + hs_error_t err = hs_database_size(pd->hs_db, db_size); + if (err != HS_SUCCESS) { + SCLogError("failed to query database size: %s", HSErrorToStr(err)); + return -1; + } + return 0; +} + +static void SCHSCleanupOnError(PatternDatabase *pd, SCHSCompileData *cd) +{ + if (pd) { + PatternDatabaseFree(pd); + } + if (cd) { + CompileDataFree(cd); + } +} + +static int CompileDataExtensionsInit(hs_expr_ext_t **ext, const SCHSPattern *p) +{ + if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) { + *ext = SCCalloc(1, sizeof(hs_expr_ext_t)); + if ((*ext) == NULL) { + return -1; + } + if (p->flags & MPM_PATTERN_FLAG_OFFSET) { + (*ext)->flags |= HS_EXT_FLAG_MIN_OFFSET; + (*ext)->min_offset = p->offset + p->len; + } + if (p->flags & MPM_PATTERN_FLAG_DEPTH) { + (*ext)->flags |= HS_EXT_FLAG_MAX_OFFSET; + (*ext)->max_offset = p->offset + p->depth; + } + } + + return 0; +} + +/** + * \brief Initialize the pattern database - try to get existing pd + * from the global hash table, or load it from disk if caching is enabled. + * + * \param PatternDatabase* [in/out] Pointer to the pattern database to use. + * \param SCHSCompileData* [in] Pointer to the compile data. + * \retval 0 On success, negative value on failure. + */ +static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd, bool disk_cache) +{ /* Check global hash table to see if we've seen this pattern database * before, and reuse the Hyperscan database if so. */ - PatternDatabase *pd_cached = HashTableLookup(g_db_table, pd, 1); - + PatternDatabase *pd_cached = HashTableLookup(g_db_table, *pd, 1); if (pd_cached != NULL) { SCLogDebug("Reusing cached database %p with %" PRIu32 " patterns (ref_cnt=%" PRIu32 ")", pd_cached->hs_db, pd_cached->pattern_cnt, pd_cached->ref_cnt); pd_cached->ref_cnt++; - ctx->pattern_db = pd_cached; - SCMutexUnlock(&g_db_table_mutex); - PatternDatabaseFree(pd); - SCHSFreeCompileData(cd); + PatternDatabaseFree(*pd); + CompileDataFree(cd); + *pd = pd_cached; return 0; + } else if (disk_cache) { + pd_cached = *pd; + uint64_t db_lookup_hash = HSHashDb(pd_cached); + if (HSLoadCache(&pd_cached->hs_db, db_lookup_hash) == 0) { + pd_cached->ref_cnt = 1; + pd_cached->cached = true; + if (HSScratchAlloc(pd_cached->hs_db) != 0) { + goto recover; + } + if (HashTableAdd(g_db_table, pd_cached, 1) < 0) { + goto recover; + } + CompileDataFree(cd); + return 0; + + recover: + pd_cached->ref_cnt = 0; + pd_cached->cached = false; + return -1; + } } - BUG_ON(ctx->pattern_db != NULL); /* already built? */ + return -1; // not cached +} +static int PatternDatabaseCompile(PatternDatabase *pd, SCHSCompileData *cd) +{ for (uint32_t i = 0; i < pd->pattern_cnt; i++) { const SCHSPattern *p = pd->parray[i]; - cd->ids[i] = i; cd->flags[i] = HS_FLAG_SINGLEMATCH; if (p->flags & MPM_PATTERN_FLAG_NOCASE) { cd->flags[i] |= HS_FLAG_CASELESS; } - cd->expressions[i] = HSRenderPattern(p->original_pat, p->len); + if (CompileDataExtensionsInit(&cd->ext[i], p) != 0) { + return -1; + } + } - if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) { - cd->ext[i] = SCCalloc(1, sizeof(hs_expr_ext_t)); - if (cd->ext[i] == NULL) { - SCMutexUnlock(&g_db_table_mutex); - goto error; - } + hs_compile_error_t *compile_err = NULL; + hs_error_t err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags, cd->ids, + (const hs_expr_ext_t *const *)cd->ext, cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db, + &compile_err); + if (err != HS_SUCCESS) { + HSLogCompileError(compile_err); + return -1; + } - if (p->flags & MPM_PATTERN_FLAG_OFFSET) { - cd->ext[i]->flags |= HS_EXT_FLAG_MIN_OFFSET; - cd->ext[i]->min_offset = p->offset + p->len; - } - if (p->flags & MPM_PATTERN_FLAG_DEPTH) { - cd->ext[i]->flags |= HS_EXT_FLAG_MAX_OFFSET; - cd->ext[i]->max_offset = p->offset + p->depth; - } - } + if (HSScratchAlloc(pd->hs_db) != 0) { + return -1; } - BUG_ON(mpm_ctx->pattern_cnt == 0); + if (HashTableAdd(g_db_table, pd, 1) < 0) { + return -1; + } + pd->ref_cnt = 1; + return 0; +} + +/** + * \brief Process the patterns added to the mpm, and create the internal tables. + * + * \param mpm_ctx Pointer to the mpm context. + */ +int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) +{ + SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx; - err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags, - cd->ids, (const hs_expr_ext_t *const *)cd->ext, - cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db, - &compile_err); + if (HSCheckPatterns(mpm_ctx, ctx) == 0) { + return 0; + } - if (err != HS_SUCCESS) { - SCLogError("failed to compile hyperscan database"); - if (compile_err) { - SCLogError("compile error: %s", compile_err->message); - } - hs_free_compile_error(compile_err); + SCHSCompileData *cd = CompileDataAlloc(mpm_ctx->pattern_cnt); + PatternDatabase *pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt); + if (cd == NULL || pd == NULL) { + goto error; + } + + HSPatternArrayInit(ctx, pd); + pd->no_cache = cache_to_disk ? false : true; + /* Serialise whole database compilation as a relatively easy way to ensure + * dedupe is safe. */ + SCMutexLock(&g_db_table_mutex); + if (HSGlobalPatternDatabaseInit() == -1) { SCMutexUnlock(&g_db_table_mutex); goto error; } - ctx->pattern_db = pd; + if (PatternDatabaseGetCached(&pd, cd, cache_to_disk) == 0 && pd != NULL) { + ctx->pattern_db = pd; + if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) { + SCMutexUnlock(&g_db_table_mutex); + goto error; + } - SCMutexLock(&g_scratch_proto_mutex); - err = hs_alloc_scratch(pd->hs_db, &g_scratch_proto); - SCMutexUnlock(&g_scratch_proto_mutex); - if (err != HS_SUCCESS) { - SCLogError("failed to allocate scratch"); + if (pd->ref_cnt == 1) { + // freshly allocated + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += ctx->hs_db_size; + } + SCMutexUnlock(&g_db_table_mutex); + return 0; + } + + BUG_ON(ctx->pattern_db != NULL); /* already built? */ + BUG_ON(mpm_ctx->pattern_cnt == 0); + + if (PatternDatabaseCompile(pd, cd) != 0) { SCMutexUnlock(&g_db_table_mutex); goto error; } - err = hs_database_size(pd->hs_db, &ctx->hs_db_size); - if (err != HS_SUCCESS) { - SCLogError("failed to query database size"); + ctx->pattern_db = pd; + if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) { SCMutexUnlock(&g_db_table_mutex); goto error; } @@ -715,29 +816,31 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx) mpm_ctx->memory_cnt++; mpm_ctx->memory_size += ctx->hs_db_size; - SCLogDebug("Built %" PRIu32 " patterns into a database of size %" PRIuMAX - " bytes", mpm_ctx->pattern_cnt, (uintmax_t)ctx->hs_db_size); - - /* Cache this database globally for later. */ - pd->ref_cnt = 1; - int r = HashTableAdd(g_db_table, pd, 1); SCMutexUnlock(&g_db_table_mutex); - if (r < 0) - goto error; - - SCHSFreeCompileData(cd); + CompileDataFree(cd); return 0; error: - if (pd) { - PatternDatabaseFree(pd); - } - if (cd) { - SCHSFreeCompileData(cd); - } + SCHSCleanupOnError(pd, cd); return -1; } +/** + * \brief Cache the loaded ruleset + */ +static int SCHSCacheRuleset(void) +{ + SCLogDebug("Caching the loaded ruleset "); + PatternDatabaseCache pd_stats = { 0 }; + SCMutexLock(&g_db_table_mutex); + HashTableIterate(g_db_table, HSSaveCacheIterator, &pd_stats); + SCMutexUnlock(&g_db_table_mutex); + SCLogInfo("%u rule groups cached (%u newly cached) of total %u cacheable groups", + pd_stats.hs_dbs_cache_loaded_cnt + pd_stats.hs_dbs_cache_saved_cnt, + pd_stats.hs_dbs_cache_saved_cnt, pd_stats.hs_cacheable_dbs_cnt); + return 0; +} + /** * \brief Init the mpm thread context. * @@ -1048,6 +1151,7 @@ void MpmHSRegister(void) mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS; mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI; mpm_table[MPM_HS].Prepare = SCHSPreparePatterns; + mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset; mpm_table[MPM_HS].Search = SCHSSearch; mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo; mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats; @@ -1103,7 +1207,7 @@ static int SCHSTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1137,7 +1241,7 @@ static int SCHSTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1174,7 +1278,7 @@ static int SCHSTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1208,7 +1312,7 @@ static int SCHSTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1242,7 +1346,7 @@ static int SCHSTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1274,7 +1378,7 @@ static int SCHSTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcd"; @@ -1318,7 +1422,7 @@ static int SCHSTest07(void) 0, 0, 5, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -1351,7 +1455,7 @@ static int SCHSTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1383,7 +1487,7 @@ static int SCHSTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1415,7 +1519,7 @@ static int SCHSTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "01234567890123456789012345678901234567890123456789" @@ -1458,7 +1562,7 @@ static int SCHSTest11(void) goto end; PmqSetup(&pmq); - if (SCHSPreparePatterns(&mpm_ctx) == -1) + if (SCHSPreparePatterns(&mpm_ctx, false) == -1) goto end; SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); @@ -1502,7 +1606,7 @@ static int SCHSTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyz"; @@ -1536,7 +1640,7 @@ static int SCHSTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; @@ -1570,7 +1674,7 @@ static int SCHSTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; @@ -1604,7 +1708,7 @@ static int SCHSTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; @@ -1638,7 +1742,7 @@ static int SCHSTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; @@ -1672,7 +1776,7 @@ static int SCHSTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; @@ -1711,7 +1815,7 @@ static int SCHSTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcde" @@ -1750,7 +1854,7 @@ static int SCHSTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -1790,7 +1894,7 @@ static int SCHSTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAA" @@ -1829,7 +1933,7 @@ static int SCHSTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1863,7 +1967,7 @@ static int SCHSTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyz"; @@ -1896,7 +2000,7 @@ static int SCHSTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1928,7 +2032,7 @@ static int SCHSTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1961,7 +2065,7 @@ static int SCHSTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -1994,7 +2098,7 @@ static int SCHSTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "works"; @@ -2027,7 +2131,7 @@ static int SCHSTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "tone"; @@ -2060,7 +2164,7 @@ static int SCHSTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "tONE"; diff --git a/src/util-mpm-hs.h b/src/util-mpm-hs.h index 09deb5ff99f9..487abf8fa3ab 100644 --- a/src/util-mpm-hs.h +++ b/src/util-mpm-hs.h @@ -27,48 +27,6 @@ #ifndef SURICATA_UTIL_MPM_HS__H #define SURICATA_UTIL_MPM_HS__H -typedef struct SCHSPattern_ { - /* length of the pattern */ - uint16_t len; - /* flags describing the pattern */ - uint8_t flags; - /* holds the original pattern that was added */ - uint8_t *original_pat; - /* pattern id */ - uint32_t id; - - uint16_t offset; - uint16_t depth; - - /* sid(s) for this pattern */ - uint32_t sids_size; - SigIntId *sids; - - /* only used at ctx init time, when this structure is part of a hash - * table. */ - struct SCHSPattern_ *next; -} SCHSPattern; - -typedef struct SCHSCtx_ { - /* hash used during ctx initialization */ - SCHSPattern **init_hash; - - /* pattern database and pattern arrays. */ - void *pattern_db; - - /* size of database, for accounting. */ - size_t hs_db_size; -} SCHSCtx; - -typedef struct SCHSThreadCtx_ { - /* Hyperscan scratch space region for this thread, capable of handling any - * database that has been compiled. */ - void *scratch; - - /* size of scratch space, for accounting. */ - size_t scratch_size; -} SCHSThreadCtx; - void MpmHSRegister(void); void MpmHSGlobalCleanup(void); diff --git a/src/util-mpm.h b/src/util-mpm.h index 2309b75dee2e..b1a1dccd04d9 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -162,7 +162,8 @@ typedef struct MpmTableElmt_ { */ int (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); - int (*Prepare)(struct MpmCtx_ *); + int (*Prepare)(struct MpmCtx_ *, bool); + int (*CacheRuleset)(void); /** \retval cnt number of patterns that matches: once per pattern max. */ uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t); void (*PrintCtx)(struct MpmCtx_ *); diff --git a/src/util-path.c b/src/util-path.c index 34c4cc75ca79..356c4a7727b9 100644 --- a/src/util-path.c +++ b/src/util-path.c @@ -117,42 +117,6 @@ char *PathMergeAlloc(const char *const dir, const char *const fname) return ret; } -/** - * \brief Wrapper to join a directory and filename and resolve using realpath - * _fullpath is used for WIN32 - * - * \param out_buf output buffer. Up to PATH_MAX will be written. Unchanged on exit failure. - * \param buf_size length of output buffer, must be PATH_MAX - * \param dir the directory - * \param fname the filename - * - * \retval 0 on success - * \retval -1 on failure - */ -int PathJoin(char *out_buf, size_t buf_size, const char *const dir, const char *const fname) -{ - SCEnter(); - if (buf_size != PATH_MAX) { - return -1; - } - if (PathMerge(out_buf, buf_size, dir, fname) != 0) { - SCLogError("Could not join filename to path"); - return -1; - } - char *tmp_buf = SCRealPath(out_buf, NULL); - if (tmp_buf == NULL) { - SCLogError("Error resolving path: %s", strerror(errno)); - return -1; - } - memset(out_buf, 0, buf_size); - size_t ret = strlcpy(out_buf, tmp_buf, buf_size); - free(tmp_buf); - if (ret >= buf_size) { - return -1; - } - return 0; -} - /** * \brief Wrapper around SCMkDir with default mode arguments. */ diff --git a/src/util-path.h b/src/util-path.h index fee58eabd3c4..b2b2624908dd 100644 --- a/src/util-path.h +++ b/src/util-path.h @@ -51,7 +51,6 @@ int PathIsAbsolute(const char *); int PathIsRelative(const char *); int PathMerge(char *out_buf, size_t buf_size, const char *const dir, const char *const fname); char *PathMergeAlloc(const char *const dir, const char *const fname); -int PathJoin(char *out_buf, size_t buf_len, const char *const dir, const char *const fname); int SCDefaultMkDir(const char *path); int SCCreateDirectoryTree(const char *path, const bool final); bool SCPathExists(const char *path); diff --git a/suricata.yaml.in b/suricata.yaml.in index 4bc9e87aa2af..deb38eb442c5 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1702,6 +1702,10 @@ detect: toclient-groups: 3 toserver-groups: 25 sgh-mpm-context: auto + # Cache MPM contexts to the disk to avoid rule compilation at the startup. + # Cache files are created in the standard library directory. + sgh-mpm-caching: yes + sgh-mpm-caching-path: @e_sghcachedir@ inspection-recursion-limit: 3000 # maximum number of times a tx will get logged for rules without app-layer keywords # stream-tx-log-limit: 4