From 4732091c724db655b8a14fa33d1900c7ea55a475 Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Thu, 25 Jul 2024 10:34:21 +0200 Subject: [PATCH 1/7] hashlittle: add a safe variant of hashlittle2 function This variant of hashlittle2() ensures that it avoids accesses beyond the last byte of the string, which will cause warnings from tools like Valgrind or Address Sanitizer. --- src/util-hash-lookup3.c | 204 ++++++++++++++++++++++++++++++++++++++++ src/util-hash-lookup3.h | 5 + 2 files changed, 209 insertions(+) diff --git a/src/util-hash-lookup3.c b/src/util-hash-lookup3.c index 2354c183d219..c2b0fe275828 100644 --- a/src/util-hash-lookup3.c +++ b/src/util-hash-lookup3.c @@ -805,7 +805,211 @@ void hashlittle2( *pc=c; *pb=b; } +/* + * hashlittle2: return 2 32-bit hash values + * + * This is identical to hashlittle(), except it returns two 32-bit hash + * values instead of just one. This is good enough for hash table + * lookup with 2^^64 buckets, or if you want a second hash if you're not + * happy with the first, or if you want a probably-unique 64-bit ID for + * the key. *pc is better mixed than *pb, so use *pc first. If you want + * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)". + */ +void hashlittle2_safe(const void *key, /* the key to hash */ + size_t length, /* length of the key */ + uint32_t *pc, /* IN: primary initval, OUT: primary hash */ + uint32_t *pb) /* IN: secondary initval, OUT: secondary hash */ +{ + uint32_t a, b, c; /* internal state */ + union { + const void *ptr; + size_t i; + } u; /* needed for Mac Powerbook G4 */ + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t)length) + *pc; + c += *pb; + + u.ptr = key; + if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { + const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length > 12) { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a, b, c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * Note that unlike hashlittle() above, we use the "safe" version of this + * block that is #ifdef VALGRIND above, in order to avoid warnings from + * Valgrind or Address Sanitizer. + */ + const uint8_t *k8 = (const uint8_t *)k; + switch (length) { + case 12: + c += k[2]; + b += k[1]; + a += k[0]; + break; + case 11: + c += ((uint32_t)k8[10]) << 16; /* fall through */ + case 10: + c += ((uint32_t)k8[9]) << 8; /* fall through */ + case 9: + c += k8[8]; /* fall through */ + case 8: + b += k[1]; + a += k[0]; + break; + case 7: + b += ((uint32_t)k8[6]) << 16; /* fall through */ + case 6: + b += ((uint32_t)k8[5]) << 8; /* fall through */ + case 5: + b += k8[4]; /* fall through */ + case 4: + a += k[0]; + break; + case 3: + a += ((uint32_t)k8[2]) << 16; /* fall through */ + case 2: + a += ((uint32_t)k8[1]) << 8; /* fall through */ + case 1: + a += k8[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { + const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ + const uint8_t *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length > 12) { + a += k[0] + (((uint32_t)k[1]) << 16); + b += k[2] + (((uint32_t)k[3]) << 16); + c += k[4] + (((uint32_t)k[5]) << 16); + mix(a, b, c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const uint8_t *)k; + switch (length) { + case 12: + c += k[4] + (((uint32_t)k[5]) << 16); + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 11: + c += ((uint32_t)k8[10]) << 16; /* fall through */ + case 10: + c += k[4]; + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 9: + c += k8[8]; /* fall through */ + case 8: + b += k[2] + (((uint32_t)k[3]) << 16); + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 7: + b += ((uint32_t)k8[6]) << 16; /* fall through */ + case 6: + b += k[2]; + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 5: + b += k8[4]; /* fall through */ + case 4: + a += k[0] + (((uint32_t)k[1]) << 16); + break; + case 3: + a += ((uint32_t)k8[2]) << 16; /* fall through */ + case 2: + a += k[0]; + break; + case 1: + a += k8[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + + } else { /* need to read the key one byte at a time */ + const uint8_t *k = (const uint8_t *)key; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) { + a += k[0]; + a += ((uint32_t)k[1]) << 8; + a += ((uint32_t)k[2]) << 16; + a += ((uint32_t)k[3]) << 24; + b += k[4]; + b += ((uint32_t)k[5]) << 8; + b += ((uint32_t)k[6]) << 16; + b += ((uint32_t)k[7]) << 24; + c += k[8]; + c += ((uint32_t)k[9]) << 8; + c += ((uint32_t)k[10]) << 16; + c += ((uint32_t)k[11]) << 24; + mix(a, b, c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch (length) /* all the case statements fall through */ + { + case 12: + c += ((uint32_t)k[11]) << 24; /* fall through */ + case 11: + c += ((uint32_t)k[10]) << 16; /* fall through */ + case 10: + c += ((uint32_t)k[9]) << 8; /* fall through */ + case 9: + c += k[8]; /* fall through */ + case 8: + b += ((uint32_t)k[7]) << 24; /* fall through */ + case 7: + b += ((uint32_t)k[6]) << 16; /* fall through */ + case 6: + b += ((uint32_t)k[5]) << 8; /* fall through */ + case 5: + b += k[4]; /* fall through */ + case 4: + a += ((uint32_t)k[3]) << 24; /* fall through */ + case 3: + a += ((uint32_t)k[2]) << 16; /* fall through */ + case 2: + a += ((uint32_t)k[1]) << 8; /* fall through */ + case 1: + a += k[0]; + break; + case 0: + *pc = c; + *pb = b; + return; /* zero length strings require no mixing */ + } + } + final(a, b, c); + *pc = c; + *pb = b; +} /* * hashbig(): diff --git a/src/util-hash-lookup3.h b/src/util-hash-lookup3.h index 62d3d14270b1..ab7f6d3d88f3 100644 --- a/src/util-hash-lookup3.h +++ b/src/util-hash-lookup3.h @@ -62,6 +62,11 @@ void hashlittle2(const void *key, /* the key to hash */ uint32_t *pc, /* IN: primary initval, OUT: primary hash */ uint32_t *pb); /* IN: secondary initval, OUT: secondary hash */ +/* A variant of hashlittle2() that ensures avoids accesses beyond the last byte + * of the string, which will cause warnings from tools like Valgrind or Address + * Sanitizer. */ +void hashlittle2_safe(const void *key, size_t length, uint32_t *pc, uint32_t *pb); + uint32_t hashbig( const void *key, size_t length, uint32_t initval); #endif /* SURICATA_UTIL_HASH_LOOKUP3_H */ From 66c276d2e1d62183608b0afcd722d58ec49170e7 Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Sat, 5 Oct 2024 20:27:19 +0200 Subject: [PATCH 2/7] util-hash: add iterator for hash function --- src/util-hash.c | 15 +++++++++++++++ src/util-hash.h | 1 + 2 files changed, 16 insertions(+) diff --git a/src/util-hash.c b/src/util-hash.c index ea04f2d7bb94..bdfc1c6f74ff 100644 --- a/src/util-hash.c +++ b/src/util-hash.c @@ -208,6 +208,21 @@ void *HashTableLookup(HashTable *ht, void *data, uint16_t datalen) return NULL; } +// CallbackFn is an iterator, first argument is the data, second is user auxilary data +void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux) +{ + if (ht == NULL || CallbackFn == NULL) + return; + + for (uint32_t i = 0; i < ht->array_size; i++) { + HashTableBucket *hashbucket = ht->array[i]; + while (hashbucket != NULL) { + CallbackFn(hashbucket->data, aux); + hashbucket = hashbucket->next; + } + } +} + uint32_t HashTableGenericHash(HashTable *ht, void *data, uint16_t datalen) { uint8_t *d = (uint8_t *)data; diff --git a/src/util-hash.h b/src/util-hash.h index ec96f5bcac71..40241b3db71a 100644 --- a/src/util-hash.h +++ b/src/util-hash.h @@ -51,6 +51,7 @@ void HashTableFree(HashTable *); int HashTableAdd(HashTable *, void *, uint16_t); int HashTableRemove(HashTable *, void *, uint16_t); void *HashTableLookup(HashTable *, void *, uint16_t); +void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux); uint32_t HashTableGenericHash(HashTable *, void *, uint16_t); char HashTableDefaultCompare(void *, uint16_t, void *, uint16_t); From 53afb5301fa7ecb670304664a8dea37ac7854a2c Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Wed, 24 Jul 2024 22:45:20 +0200 Subject: [PATCH 3/7] util-path: remove dead code --- src/util-path.c | 36 ------------------------------------ src/util-path.h | 1 - 2 files changed, 37 deletions(-) diff --git a/src/util-path.c b/src/util-path.c index 34c4cc75ca79..356c4a7727b9 100644 --- a/src/util-path.c +++ b/src/util-path.c @@ -117,42 +117,6 @@ char *PathMergeAlloc(const char *const dir, const char *const fname) return ret; } -/** - * \brief Wrapper to join a directory and filename and resolve using realpath - * _fullpath is used for WIN32 - * - * \param out_buf output buffer. Up to PATH_MAX will be written. Unchanged on exit failure. - * \param buf_size length of output buffer, must be PATH_MAX - * \param dir the directory - * \param fname the filename - * - * \retval 0 on success - * \retval -1 on failure - */ -int PathJoin(char *out_buf, size_t buf_size, const char *const dir, const char *const fname) -{ - SCEnter(); - if (buf_size != PATH_MAX) { - return -1; - } - if (PathMerge(out_buf, buf_size, dir, fname) != 0) { - SCLogError("Could not join filename to path"); - return -1; - } - char *tmp_buf = SCRealPath(out_buf, NULL); - if (tmp_buf == NULL) { - SCLogError("Error resolving path: %s", strerror(errno)); - return -1; - } - memset(out_buf, 0, buf_size); - size_t ret = strlcpy(out_buf, tmp_buf, buf_size); - free(tmp_buf); - if (ret >= buf_size) { - return -1; - } - return 0; -} - /** * \brief Wrapper around SCMkDir with default mode arguments. */ diff --git a/src/util-path.h b/src/util-path.h index fee58eabd3c4..b2b2624908dd 100644 --- a/src/util-path.h +++ b/src/util-path.h @@ -51,7 +51,6 @@ int PathIsAbsolute(const char *); int PathIsRelative(const char *); int PathMerge(char *out_buf, size_t buf_size, const char *const dir, const char *const fname); char *PathMergeAlloc(const char *const dir, const char *const fname); -int PathJoin(char *out_buf, size_t buf_len, const char *const dir, const char *const fname); int SCDefaultMkDir(const char *path); int SCCreateDirectoryTree(const char *path, const bool final); bool SCPathExists(const char *path); From a22a4bf445ed5daa06a8d2567cf19d73cc0a40b5 Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Wed, 24 Jul 2024 22:46:15 +0200 Subject: [PATCH 4/7] detect-engine: remove commented out code --- src/detect-engine.c | 9 --------- src/detect-engine.h | 1 - 2 files changed, 10 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index d77aa1a8db0a..8ef6a7dd9d59 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -3031,15 +3031,6 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) return 0; } -/* - * getting & (re)setting the internal sig i - */ - -//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx) -//{ -// return de_ctx->signum; -//} - void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx) { de_ctx->signum = 0; diff --git a/src/detect-engine.h b/src/detect-engine.h index b75d124f9cd4..e3b8a578cf35 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -100,7 +100,6 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); -//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *); /* faster as a macro than a inline function on my box -- VJ */ #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum) void DetectEngineResetMaxSigId(DetectEngineCtx *); From e684b4f286033ee03b7d3dc852cf74de8b54b74a Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Mon, 28 Oct 2024 15:41:07 +0100 Subject: [PATCH 5/7] util-mpm-hs: refactor Hyperscan Pattern DB initialization to smaller functions --- src/Makefile.am | 2 + src/util-mpm-hs-core.c | 83 +++++++++++++ src/util-mpm-hs-core.h | 93 ++++++++++++++ src/util-mpm-hs.c | 268 +++++++++++++++++++++++++---------------- src/util-mpm-hs.h | 42 ------- 5 files changed, 343 insertions(+), 145 deletions(-) create mode 100644 src/util-mpm-hs-core.c create mode 100644 src/util-mpm-hs-core.h diff --git a/src/Makefile.am b/src/Makefile.am index 82155d7f825d..f9410ac4f9f9 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -529,6 +529,7 @@ noinst_HEADERS = \ util-mpm-ac-ks.h \ util-mpm.h \ util-mpm-hs.h \ + util-mpm-hs-core.h \ util-optimize.h \ util-pages.h \ util-path.h \ @@ -1076,6 +1077,7 @@ libsuricata_c_a_SOURCES = \ util-mpm-ac-ks-small.c \ util-mpm.c \ util-mpm-hs.c \ + util-mpm-hs-core.c \ util-pages.c \ util-path.c \ util-pidfile.c \ diff --git a/src/util-mpm-hs-core.c b/src/util-mpm-hs-core.c new file mode 100644 index 000000000000..b8663ae220d2 --- /dev/null +++ b/src/util-mpm-hs-core.c @@ -0,0 +1,83 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jim Xu + * \author Justin Viiret + * \author Lukas Sismis + * + * MPM pattern matcher core function for the Hyperscan regex matcher. + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "util-mpm-hs-core.h" + +#ifdef BUILD_HYPERSCAN + +#include + +/** + * Translates Hyperscan error codes to human-readable messages. + * + * \param error_code + * The error code returned by a Hyperscan function. + * \return + * A string describing the error. + */ +const char *HSErrorToStr(hs_error_t error_code) +{ + switch (error_code) { + case HS_SUCCESS: + return "HS_SUCCESS: The engine completed normally"; + case HS_INVALID: + return "HS_INVALID: A parameter passed to this function was invalid"; + case HS_NOMEM: + return "HS_NOMEM: A memory allocation failed"; + case HS_SCAN_TERMINATED: + return "HS_SCAN_TERMINATED: The engine was terminated by callback"; + case HS_COMPILER_ERROR: + return "HS_COMPILER_ERROR: The pattern compiler failed"; + case HS_DB_VERSION_ERROR: + return "HS_DB_VERSION_ERROR: The given database was built for a different version of " + "Hyperscan"; + case HS_DB_PLATFORM_ERROR: + return "HS_DB_PLATFORM_ERROR: The given database was built for a different platform " + "(i.e., CPU type)"; + case HS_DB_MODE_ERROR: + return "HS_DB_MODE_ERROR: The given database was built for a different mode of " + "operation"; + case HS_BAD_ALIGN: + return "HS_BAD_ALIGN: A parameter passed to this function was not correctly aligned"; + case HS_BAD_ALLOC: + return "HS_BAD_ALLOC: The memory allocator did not return correctly aligned memory"; + case HS_SCRATCH_IN_USE: + return "HS_SCRATCH_IN_USE: The scratch region was already in use"; + case HS_ARCH_ERROR: + return "HS_ARCH_ERROR: Unsupported CPU architecture"; + case HS_INSUFFICIENT_SPACE: + return "HS_INSUFFICIENT_SPACE: Provided buffer was too small"; + case HS_UNKNOWN_ERROR: + return "HS_UNKNOWN_ERROR: Unexpected internal error"; + default: + return "Unknown error code"; + } +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h new file mode 100644 index 000000000000..fc7c2d302848 --- /dev/null +++ b/src/util-mpm-hs-core.h @@ -0,0 +1,93 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jim Xu + * \author Justin Viiret + * \author Lukas Sismis + * + * MPM pattern matcher core function for the Hyperscan regex matcher. + */ + +#ifndef SURICATA_UTIL_MPM_HS_CORE__H +#define SURICATA_UTIL_MPM_HS_CORE__H + +#include "suricata-common.h" +#include "suricata.h" + +#ifdef BUILD_HYPERSCAN +#include + +typedef struct SCHSPattern_ { + /* length of the pattern */ + uint16_t len; + /* flags describing the pattern */ + uint8_t flags; + /* holds the original pattern that was added */ + uint8_t *original_pat; + /* pattern id */ + uint32_t id; + + uint16_t offset; + uint16_t depth; + + /* sid(s) for this pattern */ + uint32_t sids_size; + SigIntId *sids; + + /* only used at ctx init time, when this structure is part of a hash + * table. */ + struct SCHSPattern_ *next; +} SCHSPattern; + +typedef struct SCHSCtx_ { + /* hash used during ctx initialization */ + SCHSPattern **init_hash; + + /* pattern database and pattern arrays. */ + void *pattern_db; + + /* size of database, for accounting. */ + size_t hs_db_size; +} SCHSCtx; + +typedef struct SCHSThreadCtx_ { + /* Hyperscan scratch space region for this thread, capable of handling any + * database that has been compiled. */ + void *scratch; + + /* size of scratch space, for accounting. */ + size_t scratch_size; +} SCHSThreadCtx; + +typedef struct PatternDatabase_ { + SCHSPattern **parray; + hs_database_t *hs_db; + uint32_t pattern_cnt; + + /* Reference count: number of MPM contexts using this pattern database. */ + uint32_t ref_cnt; + /* Signals if the matcher has loaded/saved the pattern database to disk */ + bool cached; +} PatternDatabase; + +const char *HSErrorToStr(hs_error_t error_code); + +#endif /* BUILD_HYPERSCAN */ +#endif /* SURICATA_UTIL_MPM_HS_CORE__H */ diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index 82b91b4ec0e8..da02e2194dfe 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -38,6 +38,7 @@ #include "util-unittest-helper.h" #include "util-memcmp.h" #include "util-mpm-hs.h" +#include "util-mpm-hs-core.h" #include "util-memcpy.h" #include "util-hash.h" #include "util-hash-lookup3.h" @@ -379,7 +380,7 @@ typedef struct SCHSCompileData_ { unsigned int pattern_cnt; } SCHSCompileData; -static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt) +static SCHSCompileData *CompileDataAlloc(unsigned int pattern_cnt) { SCHSCompileData *cd = SCCalloc(pattern_cnt, sizeof(SCHSCompileData)); if (cd == NULL) { @@ -422,7 +423,7 @@ static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt) return NULL; } -static void SCHSFreeCompileData(SCHSCompileData *cd) +static void CompileDataFree(SCHSCompileData *cd) { if (cd == NULL) { return; @@ -445,15 +446,6 @@ static void SCHSFreeCompileData(SCHSCompileData *cd) SCFree(cd); } -typedef struct PatternDatabase_ { - SCHSPattern **parray; - hs_database_t *hs_db; - uint32_t pattern_cnt; - - /* Reference count: number of MPM contexts using this pattern database. */ - uint32_t ref_cnt; -} PatternDatabase; - static uint32_t SCHSPatternHash(const SCHSPattern *p, uint32_t hash) { BUG_ON(p->original_pat == NULL); @@ -570,38 +562,20 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt) return pd; } -/** - * \brief Process the patterns added to the mpm, and create the internal tables. - * - * \param mpm_ctx Pointer to the mpm context. - */ -int SCHSPreparePatterns(MpmCtx *mpm_ctx) +static int HSCheckPatterns(MpmCtx *mpm_ctx, SCHSCtx *ctx) { - SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx; - if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) { SCLogDebug("no patterns supplied to this mpm_ctx"); return 0; } + return 1; +} - hs_error_t err; - hs_compile_error_t *compile_err = NULL; - SCHSCompileData *cd = NULL; - PatternDatabase *pd = NULL; - - cd = SCHSAllocCompileData(mpm_ctx->pattern_cnt); - if (cd == NULL) { - goto error; - } - - pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt); - if (pd == NULL) { - goto error; - } - - /* populate the pattern array with the patterns in the hash */ +static void HSPatternArrayPopulate(SCHSCtx *ctx, PatternDatabase *pd) +{ for (uint32_t i = 0, p = 0; i < INIT_HASH_SIZE; i++) { - SCHSPattern *node = ctx->init_hash[i], *nnode = NULL; + SCHSPattern *node = ctx->init_hash[i]; + SCHSPattern *nnode = NULL; while (node != NULL) { nnode = node->next; node->next = NULL; @@ -609,105 +583,207 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx) node = nnode; } } +} +static void HSPatternArrayInit(SCHSCtx *ctx, PatternDatabase *pd) +{ + HSPatternArrayPopulate(ctx, pd); /* we no longer need the hash, so free its memory */ SCFree(ctx->init_hash); ctx->init_hash = NULL; +} - /* Serialise whole database compilation as a relatively easy way to ensure - * dedupe is safe. */ - SCMutexLock(&g_db_table_mutex); - - /* Init global pattern database hash if necessary. */ +static int HSGlobalPatternDatabaseInit(void) +{ if (g_db_table == NULL) { g_db_table = HashTableInit(INIT_DB_HASH_SIZE, PatternDatabaseHash, PatternDatabaseCompare, PatternDatabaseTableFree); if (g_db_table == NULL) { - SCMutexUnlock(&g_db_table_mutex); - goto error; + return -1; } } + return 0; +} +static void HSLogCompileError(hs_compile_error_t *compile_err) +{ + SCLogError("failed to compile hyperscan database"); + if (compile_err) { + SCLogError("compile error: %s", compile_err->message); + hs_free_compile_error(compile_err); + } +} + +static int HSScratchAlloc(const hs_database_t *db) +{ + SCMutexLock(&g_scratch_proto_mutex); + hs_error_t err = hs_alloc_scratch(db, &g_scratch_proto); + SCMutexUnlock(&g_scratch_proto_mutex); + if (err != HS_SUCCESS) { + SCLogError("failed to allocate scratch"); + return -1; + } + return 0; +} + +static int PatternDatabaseGetSize(PatternDatabase *pd, size_t *db_size) +{ + hs_error_t err = hs_database_size(pd->hs_db, db_size); + if (err != HS_SUCCESS) { + SCLogError("failed to query database size: %s", HSErrorToStr(err)); + return -1; + } + return 0; +} + +static void SCHSCleanupOnError(PatternDatabase *pd, SCHSCompileData *cd) +{ + if (pd) { + PatternDatabaseFree(pd); + } + if (cd) { + CompileDataFree(cd); + } +} + +static int CompileDataExtensionsInit(hs_expr_ext_t **ext, const SCHSPattern *p) +{ + if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) { + *ext = SCCalloc(1, sizeof(hs_expr_ext_t)); + if ((*ext) == NULL) { + return -1; + } + if (p->flags & MPM_PATTERN_FLAG_OFFSET) { + (*ext)->flags |= HS_EXT_FLAG_MIN_OFFSET; + (*ext)->min_offset = p->offset + p->len; + } + if (p->flags & MPM_PATTERN_FLAG_DEPTH) { + (*ext)->flags |= HS_EXT_FLAG_MAX_OFFSET; + (*ext)->max_offset = p->offset + p->depth; + } + } + + return 0; +} + +/** + * \brief Initialize the pattern database - try to get existing pd + * from the global hash table, or load it from disk if caching is enabled. + * + * \param PatternDatabase* [in/out] Pointer to the pattern database to use. + * \param SCHSCompileData* [in] Pointer to the compile data. + * \retval 0 On success, negative value on failure. + */ +static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd) +{ /* Check global hash table to see if we've seen this pattern database * before, and reuse the Hyperscan database if so. */ - PatternDatabase *pd_cached = HashTableLookup(g_db_table, pd, 1); - + PatternDatabase *pd_cached = HashTableLookup(g_db_table, *pd, 1); if (pd_cached != NULL) { SCLogDebug("Reusing cached database %p with %" PRIu32 " patterns (ref_cnt=%" PRIu32 ")", pd_cached->hs_db, pd_cached->pattern_cnt, pd_cached->ref_cnt); pd_cached->ref_cnt++; - ctx->pattern_db = pd_cached; - SCMutexUnlock(&g_db_table_mutex); - PatternDatabaseFree(pd); - SCHSFreeCompileData(cd); + PatternDatabaseFree(*pd); + CompileDataFree(cd); + *pd = pd_cached; return 0; } - BUG_ON(ctx->pattern_db != NULL); /* already built? */ + return -1; // not cached +} +static int PatternDatabaseCompile(PatternDatabase *pd, SCHSCompileData *cd) +{ for (uint32_t i = 0; i < pd->pattern_cnt; i++) { const SCHSPattern *p = pd->parray[i]; - cd->ids[i] = i; cd->flags[i] = HS_FLAG_SINGLEMATCH; if (p->flags & MPM_PATTERN_FLAG_NOCASE) { cd->flags[i] |= HS_FLAG_CASELESS; } - cd->expressions[i] = HSRenderPattern(p->original_pat, p->len); + if (CompileDataExtensionsInit(&cd->ext[i], p) != 0) { + return -1; + } + } - if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) { - cd->ext[i] = SCCalloc(1, sizeof(hs_expr_ext_t)); - if (cd->ext[i] == NULL) { - SCMutexUnlock(&g_db_table_mutex); - goto error; - } + hs_compile_error_t *compile_err = NULL; + hs_error_t err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags, cd->ids, + (const hs_expr_ext_t *const *)cd->ext, cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db, + &compile_err); + if (err != HS_SUCCESS) { + HSLogCompileError(compile_err); + return -1; + } - if (p->flags & MPM_PATTERN_FLAG_OFFSET) { - cd->ext[i]->flags |= HS_EXT_FLAG_MIN_OFFSET; - cd->ext[i]->min_offset = p->offset + p->len; - } - if (p->flags & MPM_PATTERN_FLAG_DEPTH) { - cd->ext[i]->flags |= HS_EXT_FLAG_MAX_OFFSET; - cd->ext[i]->max_offset = p->offset + p->depth; - } - } + if (HSScratchAlloc(pd->hs_db) != 0) { + return -1; } - BUG_ON(mpm_ctx->pattern_cnt == 0); + if (HashTableAdd(g_db_table, pd, 1) < 0) { + return -1; + } + pd->ref_cnt = 1; + return 0; +} - err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags, - cd->ids, (const hs_expr_ext_t *const *)cd->ext, - cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db, - &compile_err); +/** + * \brief Process the patterns added to the mpm, and create the internal tables. + * + * \param mpm_ctx Pointer to the mpm context. + */ +int SCHSPreparePatterns(MpmCtx *mpm_ctx) +{ + SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx; - if (err != HS_SUCCESS) { - SCLogError("failed to compile hyperscan database"); - if (compile_err) { - SCLogError("compile error: %s", compile_err->message); - } - hs_free_compile_error(compile_err); + if (HSCheckPatterns(mpm_ctx, ctx) == 0) { + return 0; + } + + SCHSCompileData *cd = CompileDataAlloc(mpm_ctx->pattern_cnt); + PatternDatabase *pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt); + if (cd == NULL || pd == NULL) { + goto error; + } + + HSPatternArrayInit(ctx, pd); + /* Serialise whole database compilation as a relatively easy way to ensure + * dedupe is safe. */ + SCMutexLock(&g_db_table_mutex); + if (HSGlobalPatternDatabaseInit() == -1) { SCMutexUnlock(&g_db_table_mutex); goto error; } - ctx->pattern_db = pd; + if (PatternDatabaseGetCached(&pd, cd) == 0 && pd != NULL) { + ctx->pattern_db = pd; + if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) { + SCMutexUnlock(&g_db_table_mutex); + goto error; + } - SCMutexLock(&g_scratch_proto_mutex); - err = hs_alloc_scratch(pd->hs_db, &g_scratch_proto); - SCMutexUnlock(&g_scratch_proto_mutex); - if (err != HS_SUCCESS) { - SCLogError("failed to allocate scratch"); + if (pd->ref_cnt == 1) { + // freshly allocated + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += ctx->hs_db_size; + } + SCMutexUnlock(&g_db_table_mutex); + return 0; + } + + BUG_ON(ctx->pattern_db != NULL); /* already built? */ + BUG_ON(mpm_ctx->pattern_cnt == 0); + + if (PatternDatabaseCompile(pd, cd) != 0) { SCMutexUnlock(&g_db_table_mutex); goto error; } - err = hs_database_size(pd->hs_db, &ctx->hs_db_size); - if (err != HS_SUCCESS) { - SCLogError("failed to query database size"); + ctx->pattern_db = pd; + if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) { SCMutexUnlock(&g_db_table_mutex); goto error; } @@ -715,26 +791,12 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx) mpm_ctx->memory_cnt++; mpm_ctx->memory_size += ctx->hs_db_size; - SCLogDebug("Built %" PRIu32 " patterns into a database of size %" PRIuMAX - " bytes", mpm_ctx->pattern_cnt, (uintmax_t)ctx->hs_db_size); - - /* Cache this database globally for later. */ - pd->ref_cnt = 1; - int r = HashTableAdd(g_db_table, pd, 1); SCMutexUnlock(&g_db_table_mutex); - if (r < 0) - goto error; - - SCHSFreeCompileData(cd); + CompileDataFree(cd); return 0; error: - if (pd) { - PatternDatabaseFree(pd); - } - if (cd) { - SCHSFreeCompileData(cd); - } + SCHSCleanupOnError(pd, cd); return -1; } diff --git a/src/util-mpm-hs.h b/src/util-mpm-hs.h index 09deb5ff99f9..487abf8fa3ab 100644 --- a/src/util-mpm-hs.h +++ b/src/util-mpm-hs.h @@ -27,48 +27,6 @@ #ifndef SURICATA_UTIL_MPM_HS__H #define SURICATA_UTIL_MPM_HS__H -typedef struct SCHSPattern_ { - /* length of the pattern */ - uint16_t len; - /* flags describing the pattern */ - uint8_t flags; - /* holds the original pattern that was added */ - uint8_t *original_pat; - /* pattern id */ - uint32_t id; - - uint16_t offset; - uint16_t depth; - - /* sid(s) for this pattern */ - uint32_t sids_size; - SigIntId *sids; - - /* only used at ctx init time, when this structure is part of a hash - * table. */ - struct SCHSPattern_ *next; -} SCHSPattern; - -typedef struct SCHSCtx_ { - /* hash used during ctx initialization */ - SCHSPattern **init_hash; - - /* pattern database and pattern arrays. */ - void *pattern_db; - - /* size of database, for accounting. */ - size_t hs_db_size; -} SCHSCtx; - -typedef struct SCHSThreadCtx_ { - /* Hyperscan scratch space region for this thread, capable of handling any - * database that has been compiled. */ - void *scratch; - - /* size of scratch space, for accounting. */ - size_t scratch_size; -} SCHSThreadCtx; - void MpmHSRegister(void); void MpmHSGlobalCleanup(void); From 9b1c453c36abcdc6076ad3f5f97472cb6ef9e62b Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Mon, 28 Oct 2024 15:44:09 +0100 Subject: [PATCH 6/7] util-mpm: prepare MPM codebase for ruleset caching --- Makefile.am | 1 + configure.ac | 4 +++ src/app-layer-detect-proto.c | 2 +- src/app-layer-ftp.c | 3 +- src/app-layer-smtp.c | 2 +- src/detect-engine-loader.c | 5 +++ src/detect-engine-mpm.c | 22 ++++++------- src/detect-engine.c | 30 +++++++++++++++++ src/detect-engine.h | 1 + src/detect.h | 3 ++ src/util-mpm-ac-ks.c | 61 +++++++++++++++++----------------- src/util-mpm-ac.c | 63 ++++++++++++++++++------------------ src/util-mpm-hs.c | 61 +++++++++++++++++----------------- src/util-mpm.h | 3 +- suricata.yaml.in | 4 +++ 15 files changed, 158 insertions(+), 107 deletions(-) diff --git a/Makefile.am b/Makefile.am index 20e50bdc4a03..aafaae0b862d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,6 +37,7 @@ install-conf: install -d "$(DESTDIR)$(e_rundir)" install -m 770 -d "$(DESTDIR)$(e_localstatedir)" install -m 770 -d "$(DESTDIR)$(e_datadir)" + install -m 660 -d "$(DESTDIR)$(e_sghcachedir)" install-rules: if INSTALL_SURICATA_UPDATE diff --git a/configure.ac b/configure.ac index ca964d9039a0..ccc4b85d7a6f 100644 --- a/configure.ac +++ b/configure.ac @@ -2455,6 +2455,7 @@ if test "$WINDOWS_PATH" = "yes"; then e_sysconfdir="${e_winbase}\\\\" e_defaultruledir="$e_winbase\\\\rules\\\\" + e_sghcachedir="$e_winbase\\\\cache\\\\sgh\\\\" e_magic_file="$e_winbase\\\\magic.mgc" e_logdir="$e_winbase\\\\log" e_logfilesdir="$e_logdir\\\\files" @@ -2476,6 +2477,7 @@ else EXPAND_VARIABLE(sysconfdir, e_sysconfdir, "/suricata/") EXPAND_VARIABLE(localstatedir, e_localstatedir, "/run/suricata") EXPAND_VARIABLE(datadir, e_datarulesdir, "/suricata/rules") + EXPAND_VARIABLE(localstatedir, e_sghcachedir, "/lib/suricata/cache/sgh") EXPAND_VARIABLE(localstatedir, e_datadir, "/lib/suricata/data") EXPAND_VARIABLE(localstatedir, e_defaultruledir, "/lib/suricata/rules") @@ -2489,6 +2491,8 @@ AC_SUBST(e_logcertsdir) AC_SUBST(e_sysconfdir) AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR]) AC_SUBST(e_localstatedir) +AC_SUBST(e_sghcachedir) +AC_DEFINE_UNQUOTED([SGH_CACHE_DIR],["$e_sghcachedir"],[Directory path for signature group head cache]) AC_SUBST(e_datadir) AC_DEFINE_UNQUOTED([DATA_DIR],["$e_datadir"],[Our DATA_DIR]) AC_SUBST(e_magic_file) diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index 52e1f2922c02..c4d6cc7eb268 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -1297,7 +1297,7 @@ static int AppLayerProtoDetectPMPrepareMpm(AppLayerProtoDetectPMCtx *ctx) int ret = 0; MpmCtx *mpm_ctx = &ctx->mpm_ctx; - if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx) < 0) + if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx, false) < 0) goto error; goto end; diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index a6a1f632fcb4..fd9b0fc0919f 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -1296,8 +1296,7 @@ static void FTPSetMpmState(void) i /* id */, i /* rule id */ , 0 /* no flags */); } - mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx); - + mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx, false); } static void FTPFreeMpmState(void) diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index b2bf22a53145..dcb79d2bc657 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -1636,7 +1636,7 @@ static void SMTPSetMpmState(void) i /* pattern id */, i /* rule id */ , 0 /* no flags */); } - mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx); + mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx, false); } static void SMTPFreeMpmState(void) diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index 950812a187c9..82a3d3463cc8 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -33,6 +33,7 @@ #include "tm-threads.h" #include "queue.h" +#include "detect-engine.h" #include "detect-engine-loader.h" #include "detect-engine-build.h" #include "detect-engine-analyzer.h" @@ -402,6 +403,10 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc ret = 0; + if (de_ctx->mpm_cache_to_disk && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheRuleset(); + } + end: gettimeofday(&de_ctx->last_reload, NULL); if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) { diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 1c9984ea9541..8adf02ab2778 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -295,7 +295,7 @@ int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx) MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } } @@ -524,7 +524,7 @@ int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx) SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); SCLogDebug("%s: %d", am->name, r); } } @@ -689,7 +689,7 @@ int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx) MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0); if (mpm_ctx != NULL) { if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); SCLogDebug("%s: %d", am->name, r); } } @@ -744,40 +744,40 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx) if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) { mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { - r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk); } } @@ -1621,7 +1621,7 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms) } else { if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) { if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) { - mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx); + mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx, de_ctx->mpm_cache_to_disk); } } } diff --git a/src/detect-engine.c b/src/detect-engine.c index 8ef6a7dd9d59..7354c4afe63a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2481,6 +2481,35 @@ static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx) return -1; } +static bool DetectEngineMpmCachingEnabled(void) +{ + const char *strval = NULL; + if (ConfGet("detect.sgh-mpm-caching", &strval) != 1) + return false; + + int sgh_mpm_caching = 0; + (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching); + return (bool)sgh_mpm_caching; +} + +const char *DetectEngineMpmCachingGetPath(void) +{ + char yamlpath[] = "detect.sgh-mpm-caching-path"; + const char *strval = NULL; + ConfGet(yamlpath, &strval); + + if (strval != NULL) { + return strval; + } + + static bool notified = false; + if (!notified) { + SCLogInfo("%s has no path specified, using %s", yamlpath, SGH_CACHE_DIR); + notified = true; + } + return SGH_CACHE_DIR; +} + static DetectEngineCtx *DetectEngineCtxInitReal( enum DetectEngineType type, const char *prefix, uint32_t tenant_id) { @@ -2512,6 +2541,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal( de_ctx->failure_fatal = (failure_fatal == 1); de_ctx->mpm_matcher = PatternMatchDefaultMatcher(); + de_ctx->mpm_cache_to_disk = DetectEngineMpmCachingEnabled(); de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher(); SCLogConfig("pattern matchers: MPM: %s, SPM: %s", mpm_table[de_ctx->mpm_matcher].name, diff --git a/src/detect-engine.h b/src/detect-engine.h index e3b8a578cf35..f52b6d3df05b 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -100,6 +100,7 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); +const char *DetectEngineMpmCachingGetPath(void); /* faster as a macro than a inline function on my box -- VJ */ #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum) void DetectEngineResetMaxSigId(DetectEngineCtx *); diff --git a/src/detect.h b/src/detect.h index ea81092487b5..3f1714135afe 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1050,6 +1050,9 @@ typedef struct DetectEngineCtx_ { /* number of signatures using filestore, limited as u16 */ uint16_t filestore_cnt; + + /** If enabled, MPM matchers can store compiled pattern databases to disk */ + bool mpm_cache_to_disk; } DetectEngineCtx; /* Engine groups profiles (low, medium, high, custom) */ diff --git a/src/util-mpm-ac-ks.c b/src/util-mpm-ac-ks.c index 60bb1b7a9e42..6eeff3e3a29c 100644 --- a/src/util-mpm-ac-ks.c +++ b/src/util-mpm-ac-ks.c @@ -90,7 +90,7 @@ int SCACTileAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCACTileAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCACTilePreparePatterns(MpmCtx *mpm_ctx); +int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCACTileSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, uint32_t buflen); @@ -861,7 +861,7 @@ static void SCACTilePrepareSearch(MpmCtx *mpm_ctx) * * \param mpm_ctx Pointer to the mpm context. */ -int SCACTilePreparePatterns(MpmCtx *mpm_ctx) +int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) { SCACTileSearchCtx *search_ctx = (SCACTileSearchCtx *)mpm_ctx->ctx; @@ -1403,6 +1403,7 @@ void MpmACTileRegister(void) mpm_table[MPM_AC_KS].AddPattern = SCACTileAddPatternCS; mpm_table[MPM_AC_KS].AddPatternNocase = SCACTileAddPatternCI; mpm_table[MPM_AC_KS].Prepare = SCACTilePreparePatterns; + mpm_table[MPM_AC_KS].CacheRuleset = NULL; mpm_table[MPM_AC_KS].Search = SCACTileSearch; mpm_table[MPM_AC_KS].PrintCtx = SCACTilePrintInfo; #ifdef UNITTESTS @@ -1432,7 +1433,7 @@ static int SCACTileTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1464,7 +1465,7 @@ static int SCACTileTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1499,7 +1500,7 @@ static int SCACTileTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1531,7 +1532,7 @@ static int SCACTileTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1563,7 +1564,7 @@ static int SCACTileTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1593,7 +1594,7 @@ static int SCACTileTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcd"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1635,7 +1636,7 @@ static int SCACTileTest07(void) PmqSetup(&pmq); /* total matches: 135: 6 unique */ - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1662,7 +1663,7 @@ static int SCACTileTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"a", 1); @@ -1692,7 +1693,7 @@ static int SCACTileTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"ab", 2); @@ -1722,7 +1723,7 @@ static int SCACTileTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" @@ -1763,7 +1764,7 @@ static int SCACTileTest11(void) goto end; PmqSetup(&pmq); - if (SCACTilePreparePatterns(&mpm_ctx) == -1) + if (SCACTilePreparePatterns(&mpm_ctx, false) == -1) goto end; result = 1; @@ -1804,7 +1805,7 @@ static int SCACTileTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1836,7 +1837,7 @@ static int SCACTileTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1868,7 +1869,7 @@ static int SCACTileTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1900,7 +1901,7 @@ static int SCACTileTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1932,7 +1933,7 @@ static int SCACTileTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1964,7 +1965,7 @@ static int SCACTileTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2001,7 +2002,7 @@ static int SCACTileTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2033,7 +2034,7 @@ static int SCACTileTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2071,7 +2072,7 @@ static int SCACTileTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2102,7 +2103,7 @@ static int SCACTileTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"AA", 2); @@ -2134,7 +2135,7 @@ static int SCACTileTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2165,7 +2166,7 @@ static int SCACTileTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -2195,7 +2196,7 @@ static int SCACTileTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -2226,7 +2227,7 @@ static int SCACTileTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2257,7 +2258,7 @@ static int SCACTileTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "works"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2288,7 +2289,7 @@ static int SCACTileTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "tone"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2319,7 +2320,7 @@ static int SCACTileTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACTilePreparePatterns(&mpm_ctx); + SCACTilePreparePatterns(&mpm_ctx, false); const char *buf = "tONE"; uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c index 6e115acaa305..c8c01442ea73 100644 --- a/src/util-mpm-ac.c +++ b/src/util-mpm-ac.c @@ -68,7 +68,7 @@ int SCACAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCACAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCACPreparePatterns(MpmCtx *mpm_ctx); +int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCACSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, uint32_t buflen); void SCACPrintInfo(MpmCtx *mpm_ctx); @@ -702,7 +702,7 @@ static void SCACPrepareStateTable(MpmCtx *mpm_ctx) * * \param mpm_ctx Pointer to the mpm context. */ -int SCACPreparePatterns(MpmCtx *mpm_ctx) +int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) { SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx; @@ -1102,6 +1102,7 @@ void MpmACRegister(void) mpm_table[MPM_AC].AddPattern = SCACAddPatternCS; mpm_table[MPM_AC].AddPatternNocase = SCACAddPatternCI; mpm_table[MPM_AC].Prepare = SCACPreparePatterns; + mpm_table[MPM_AC].CacheRuleset = NULL; mpm_table[MPM_AC].Search = SCACSearch; mpm_table[MPM_AC].PrintCtx = SCACPrintInfo; #ifdef UNITTESTS @@ -1130,7 +1131,7 @@ static int SCACTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1162,7 +1163,7 @@ static int SCACTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1197,7 +1198,7 @@ static int SCACTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1229,7 +1230,7 @@ static int SCACTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1261,7 +1262,7 @@ static int SCACTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1291,7 +1292,7 @@ static int SCACTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcd"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1333,7 +1334,7 @@ static int SCACTest07(void) PmqSetup(&pmq); /* total matches: 135: unique matches: 6 */ - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1360,7 +1361,7 @@ static int SCACTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"a", 1); @@ -1390,7 +1391,7 @@ static int SCACTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"ab", 2); @@ -1420,7 +1421,7 @@ static int SCACTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" @@ -1461,7 +1462,7 @@ static int SCACTest11(void) goto end; PmqSetup(&pmq); - if (SCACPreparePatterns(&mpm_ctx) == -1) + if (SCACPreparePatterns(&mpm_ctx, false) == -1) goto end; result = 1; @@ -1502,7 +1503,7 @@ static int SCACTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1534,7 +1535,7 @@ static int SCACTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1566,7 +1567,7 @@ static int SCACTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1598,7 +1599,7 @@ static int SCACTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1630,7 +1631,7 @@ static int SCACTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1662,7 +1663,7 @@ static int SCACTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1699,7 +1700,7 @@ static int SCACTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1731,7 +1732,7 @@ static int SCACTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1769,7 +1770,7 @@ static int SCACTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1800,7 +1801,7 @@ static int SCACTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"AA", 2); @@ -1832,7 +1833,7 @@ static int SCACTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1863,7 +1864,7 @@ static int SCACTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -1893,7 +1894,7 @@ static int SCACTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2); @@ -1924,7 +1925,7 @@ static int SCACTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1955,7 +1956,7 @@ static int SCACTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "works"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -1986,7 +1987,7 @@ static int SCACTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "tone"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2016,7 +2017,7 @@ static int SCACTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf = "tONE"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, @@ -2083,7 +2084,7 @@ static int SCACTest30(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"xyz", 3, 0, 0, 0, 0, MPM_PATTERN_FLAG_ENDSWITH); PmqSetup(&pmq); - SCACPreparePatterns(&mpm_ctx); + SCACPreparePatterns(&mpm_ctx, false); const char *buf1 = "abcdefghijklmnopqrstuvwxyz"; uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf1, strlen(buf1)); diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index da02e2194dfe..9246741cdf80 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -56,7 +56,7 @@ int SCHSAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int SCHSAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); -int SCHSPreparePatterns(MpmCtx *mpm_ctx); +int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk); uint32_t SCHSSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PrefilterRuleStore *pmq, const uint8_t *buf, const uint32_t buflen); void SCHSPrintInfo(MpmCtx *mpm_ctx); @@ -735,7 +735,7 @@ static int PatternDatabaseCompile(PatternDatabase *pd, SCHSCompileData *cd) * * \param mpm_ctx Pointer to the mpm context. */ -int SCHSPreparePatterns(MpmCtx *mpm_ctx) +int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) { SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx; @@ -1110,6 +1110,7 @@ void MpmHSRegister(void) mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS; mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI; mpm_table[MPM_HS].Prepare = SCHSPreparePatterns; + mpm_table[MPM_HS].CacheRuleset = NULL; mpm_table[MPM_HS].Search = SCHSSearch; mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo; mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats; @@ -1165,7 +1166,7 @@ static int SCHSTest01(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1199,7 +1200,7 @@ static int SCHSTest02(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1236,7 +1237,7 @@ static int SCHSTest03(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1270,7 +1271,7 @@ static int SCHSTest04(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1304,7 +1305,7 @@ static int SCHSTest05(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghjiklmnopqrstuvwxyz"; @@ -1336,7 +1337,7 @@ static int SCHSTest06(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcd"; @@ -1380,7 +1381,7 @@ static int SCHSTest07(void) 0, 0, 5, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -1413,7 +1414,7 @@ static int SCHSTest08(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1445,7 +1446,7 @@ static int SCHSTest09(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1477,7 +1478,7 @@ static int SCHSTest10(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "01234567890123456789012345678901234567890123456789" @@ -1520,7 +1521,7 @@ static int SCHSTest11(void) goto end; PmqSetup(&pmq); - if (SCHSPreparePatterns(&mpm_ctx) == -1) + if (SCHSPreparePatterns(&mpm_ctx, false) == -1) goto end; SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); @@ -1564,7 +1565,7 @@ static int SCHSTest12(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyz"; @@ -1598,7 +1599,7 @@ static int SCHSTest13(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCD"; @@ -1632,7 +1633,7 @@ static int SCHSTest14(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE"; @@ -1666,7 +1667,7 @@ static int SCHSTest15(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF"; @@ -1700,7 +1701,7 @@ static int SCHSTest16(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzABC"; @@ -1734,7 +1735,7 @@ static int SCHSTest17(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyzAB"; @@ -1773,7 +1774,7 @@ static int SCHSTest18(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcde" @@ -1812,7 +1813,7 @@ static int SCHSTest19(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -1852,7 +1853,7 @@ static int SCHSTest20(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "AAAAA" @@ -1891,7 +1892,7 @@ static int SCHSTest21(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1925,7 +1926,7 @@ static int SCHSTest22(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "abcdefghijklmnopqrstuvwxyz"; @@ -1958,7 +1959,7 @@ static int SCHSTest23(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -1990,7 +1991,7 @@ static int SCHSTest24(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); uint32_t cnt = @@ -2023,7 +2024,7 @@ static int SCHSTest25(void) MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -2056,7 +2057,7 @@ static int SCHSTest26(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "works"; @@ -2089,7 +2090,7 @@ static int SCHSTest27(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "tone"; @@ -2122,7 +2123,7 @@ static int SCHSTest28(void) MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0); PmqSetup(&pmq); - SCHSPreparePatterns(&mpm_ctx); + SCHSPreparePatterns(&mpm_ctx, false); SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx); const char *buf = "tONE"; diff --git a/src/util-mpm.h b/src/util-mpm.h index 2309b75dee2e..b1a1dccd04d9 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -162,7 +162,8 @@ typedef struct MpmTableElmt_ { */ int (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); - int (*Prepare)(struct MpmCtx_ *); + int (*Prepare)(struct MpmCtx_ *, bool); + int (*CacheRuleset)(void); /** \retval cnt number of patterns that matches: once per pattern max. */ uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t); void (*PrintCtx)(struct MpmCtx_ *); diff --git a/suricata.yaml.in b/suricata.yaml.in index 4bc9e87aa2af..deb38eb442c5 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1702,6 +1702,10 @@ detect: toclient-groups: 3 toserver-groups: 25 sgh-mpm-context: auto + # Cache MPM contexts to the disk to avoid rule compilation at the startup. + # Cache files are created in the standard library directory. + sgh-mpm-caching: yes + sgh-mpm-caching-path: @e_sghcachedir@ inspection-recursion-limit: 3000 # maximum number of times a tx will get logged for rules without app-layer keywords # stream-tx-log-limit: 4 From 8f01d2ef1268d7a7fc49a6c1648cd1825fa672c2 Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Mon, 28 Oct 2024 15:46:17 +0100 Subject: [PATCH 7/7] hyperscan: add caching mechanism for hyperscan contexts Cache Hyperscan serialized databases to disk to prevent compilation of the same databases when Suricata is run again with the same ruleset. Hyperscan binary files are stored per rulegroup in the designated folder, by default in the cached library folder. Since caching is per signature group heads, some chunk of the ruleset can change and it still can reuse part of the unchanged signature groups. Loading *fresh* ET Open ruleset: 19 seconds Loading *cached* ET Open ruleset: 07 seconds Ticket: 7170 --- doc/userguide/performance/hyperscan.rst | 26 ++- src/Makefile.am | 2 + src/util-mpm-hs-cache.c | 255 ++++++++++++++++++++++++ src/util-mpm-hs-cache.h | 37 ++++ src/util-mpm-hs-core.h | 34 ++-- src/util-mpm-hs.c | 47 ++++- 6 files changed, 384 insertions(+), 17 deletions(-) create mode 100644 src/util-mpm-hs-cache.c create mode 100644 src/util-mpm-hs-cache.h diff --git a/doc/userguide/performance/hyperscan.rst b/doc/userguide/performance/hyperscan.rst index 055fa7f21b75..9b39bc22e95b 100644 --- a/doc/userguide/performance/hyperscan.rst +++ b/doc/userguide/performance/hyperscan.rst @@ -81,4 +81,28 @@ if it is present on the system in case of the "auto" setting. If the current suricata installation does not have hyperscan -support, refer to :ref:`installation` \ No newline at end of file +support, refer to :ref:`installation` + +Hyperscan caching +~~~~~~~~~~~~~~~~~ + +Upon startup, Hyperscan compiles and optimizes the ruleset into its own +internal structure. Suricata optimizes the startup process by saving +the Hyperscan internal structures to disk and loading them on the next start. +This prevents the recompilation of the ruleset and results in faster +initialization. If the ruleset is changed, new necessary cache files are +automatically created. + +To enable this function, in `suricata.yaml` configure: + +:: + + # Cache MPM contexts to the disk to avoid rule compilation at the startup. + # Cache files are created in the standard library directory. + sgh-mpm-caching: yes + sgh-mpm-caching-path: /var/lib/suricata/cache/hs + + +**Note**: +You might need to create and adjust permissions to the default caching folder +path, especially if you are running Suricata as a non-root user. diff --git a/src/Makefile.am b/src/Makefile.am index f9410ac4f9f9..56e761a5c2e9 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -529,6 +529,7 @@ noinst_HEADERS = \ util-mpm-ac-ks.h \ util-mpm.h \ util-mpm-hs.h \ + util-mpm-hs-cache.h \ util-mpm-hs-core.h \ util-optimize.h \ util-pages.h \ @@ -1077,6 +1078,7 @@ libsuricata_c_a_SOURCES = \ util-mpm-ac-ks-small.c \ util-mpm.c \ util-mpm-hs.c \ + util-mpm-hs-cache.c \ util-mpm-hs-core.c \ util-pages.c \ util-path.c \ diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c new file mode 100644 index 000000000000..83107385696b --- /dev/null +++ b/src/util-mpm-hs-cache.c @@ -0,0 +1,255 @@ +/* Copyright (C) 2007-2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Lukas Sismis + * + * MPM pattern matcher that calls the Hyperscan regex matcher. + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "detect-engine.h" +#include "util-debug.h" +#include "util-hash-lookup3.h" +#include "util-mpm-hs-core.h" +#include "util-mpm-hs-cache.h" +#include "util-path.h" + +#ifdef BUILD_HYPERSCAN + +#include + +static const char *HSCacheConstructFPath(uint64_t hs_db_hash) +{ + static char hash_file_path[PATH_MAX]; + + char hash_file_path_suffix[] = "_v1.hs"; + char filename[PATH_MAX]; + uint64_t r = + snprintf(filename, sizeof(filename), "%020lu%s", hs_db_hash, hash_file_path_suffix); + if (r != (uint64_t)(20 + strlen(hash_file_path_suffix))) + return NULL; + + r = PathMerge( + hash_file_path, sizeof(hash_file_path), DetectEngineMpmCachingGetPath(), filename); + if (r) + return NULL; + + return hash_file_path; +} + +static char *HSReadStream(const char *file_path, size_t *buffer_sz) +{ + FILE *file = fopen(file_path, "rb"); + if (!file) { + SCLogDebug("Failed to open file %s: %s", file_path, strerror(errno)); + return NULL; + } + + // Seek to the end of the file to determine its size + fseek(file, 0, SEEK_END); + long file_sz = ftell(file); + if (file_sz < 0) { + SCLogDebug("Failed to determine file size of %s: %s", file_path, strerror(errno)); + fclose(file); + return NULL; + } + + char *buffer = (char *)SCCalloc(file_sz, sizeof(char)); + if (!buffer) { + SCLogWarning("Failed to allocate memory"); + fclose(file); + return NULL; + } + + // Rewind file pointer and read the file into the buffer + rewind(file); + size_t bytes_read = fread(buffer, 1, file_sz, file); + if (bytes_read != (size_t)file_sz) { + SCLogDebug("Failed to read the entire file %s: %s", file_path, strerror(errno)); + SCFree(buffer); + fclose(file); + return NULL; + } + + *buffer_sz = file_sz; + fclose(file); + return buffer; +} + +/** + * Function to hash the searched pattern, only things relevant to Hyperscan + * compilation are hashed. + */ +static void SCHSCachePatternHash(const SCHSPattern *p, uint32_t *h1, uint32_t *h2) +{ + BUG_ON(p->original_pat == NULL); + hashlittle2_safe(&p->len, sizeof(p->len), h1, h2); + hashlittle2_safe(&p->flags, sizeof(p->flags), h1, h2); + hashlittle2_safe(p->original_pat, p->len, h1, h2); + hashlittle2_safe(&p->offset, sizeof(p->offset), h1, h2); + hashlittle2_safe(&p->depth, sizeof(p->depth), h1, h2); +} + +int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash) +{ + const char *hash_file_static = HSCacheConstructFPath(hs_db_hash); + if (hash_file_static == NULL) + return -1; + + SCLogDebug("Loading the cached HS DB from %s", hash_file_static); + if (!SCPathExists(hash_file_static)) + return -1; + + FILE *db_cache = fopen(hash_file_static, "r"); + char *buffer = NULL; + int ret = 0; + if (db_cache) { + size_t buffer_size; + buffer = HSReadStream(hash_file_static, &buffer_size); + if (!buffer) { + SCLogWarning("Hyperscan cached DB file %s cannot be read", hash_file_static); + ret = -1; + goto freeup; + } + + hs_error_t error = hs_deserialize_database(buffer, buffer_size, hs_db); + if (error != HS_SUCCESS) { + SCLogWarning("Failed to deserialize Hyperscan database of %s: %s", hash_file_static, + HSErrorToStr(error)); + ret = -1; + goto freeup; + } + + ret = 0; + goto freeup; + } + +freeup: + if (db_cache) + fclose(db_cache); + if (buffer) + SCFree(buffer); + return ret; +} + +static int HSSaveCache(hs_database_t *hs_db, uint64_t hs_db_hash) +{ + static bool notified = false; + char *db_stream = NULL; + size_t db_size; + int ret = -1; + + hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size); + if (err != HS_SUCCESS) { + SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err)); + goto cleanup; + } + + const char *hash_file_static = HSCacheConstructFPath(hs_db_hash); + SCLogDebug("Caching the compiled HS at %s", hash_file_static); + if (SCPathExists(hash_file_static)) { + // potentially signs that it might not work as expected as we got into + // hash collision. If this happens with older and not used caches it is + // fine. + // It is problematic when one ruleset yields two colliding MPM groups. + SCLogWarning("Overwriting cache file %s. If the problem persists consider switching off " + "the caching", + hash_file_static); + } + + if (SCCreateDirectoryTree(DetectEngineMpmCachingGetPath(), true) != 0) { + if (!notified) { + SCLogWarning("Failed to create Hyperscan cache folder, make sure " + "the parent folder is writeable " + "or adjust sgh-mpm-caching-path setting (%s)", + DetectEngineMpmCachingGetPath()); + notified = true; + } + return -1; + } + + FILE *db_cache_out = fopen(hash_file_static, "w"); + if (!db_cache_out) { + if (!notified) { + SCLogWarning("Failed to create Hyperscan cache file, make sure the folder exist and is " + "writable or adjust sgh-mpm-caching-path setting (%s)", + hash_file_static); + notified = true; + } + goto cleanup; + } + size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out); + if (r > 0 && (size_t)r != db_size) { + SCLogWarning("Failed to write to file: %s", hash_file_static); + if (r != db_size) { + // possibly a corrupted DB cache was created + r = remove(hash_file_static); + if (r != 0) { + SCLogWarning("Failed to remove corrupted cache file: %s", hash_file_static); + } + } + } + ret = fclose(db_cache_out); + if (ret != 0) { + SCLogWarning("Failed to close file: %s", hash_file_static); + goto cleanup; + } + + ret = 0; +cleanup: + if (db_stream) + SCFree(db_stream); + return ret; +} + +uint64_t HSHashDb(const PatternDatabase *pd) +{ + uint64_t cached_hash = 0; + uint32_t *hash = (uint32_t *)(&cached_hash); + hashword2(&pd->pattern_cnt, 1, &hash[0], &hash[1]); + for (uint32_t i = 0; i < pd->pattern_cnt; i++) { + SCHSCachePatternHash(pd->parray[i], &hash[0], &hash[1]); + } + + return cached_hash; +} + +void HSSaveCacheIterator(void *data, void *aux) +{ + PatternDatabase *pd = (PatternDatabase *)data; + PatternDatabaseCache *pd_stats = (PatternDatabaseCache *)aux; + if (pd->no_cache) + return; + + // count only cacheable DBs + pd_stats->hs_cacheable_dbs_cnt++; + if (pd->cached) { + pd_stats->hs_dbs_cache_loaded_cnt++; + return; + } + + if (HSSaveCache(pd->hs_db, HSHashDb(pd)) == 0) { + pd->cached = true; // for rule reloads + pd_stats->hs_dbs_cache_saved_cnt++; + } +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-cache.h b/src/util-mpm-hs-cache.h new file mode 100644 index 000000000000..cc17d9846cc7 --- /dev/null +++ b/src/util-mpm-hs-cache.h @@ -0,0 +1,37 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Lukas Sismis + * + * Hyperscan caching logic for faster database compilation. + */ + +#ifndef SURICATA_UTIL_MPM_HS_CACHE__H +#define SURICATA_UTIL_MPM_HS_CACHE__H + +#include "util-mpm-hs-core.h" + +#ifdef BUILD_HYPERSCAN +int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash); +uint64_t HSHashDb(const PatternDatabase *pd); +void HSSaveCacheIterator(void *data, void *aux); +#endif /* BUILD_HYPERSCAN */ + +#endif /* SURICATA_UTIL_MPM_HS_CACHE__H */ diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h index fc7c2d302848..699dd69568a3 100644 --- a/src/util-mpm-hs-core.h +++ b/src/util-mpm-hs-core.h @@ -35,44 +35,44 @@ #include typedef struct SCHSPattern_ { - /* length of the pattern */ + /** length of the pattern */ uint16_t len; - /* flags describing the pattern */ + /** flags describing the pattern */ uint8_t flags; - /* holds the original pattern that was added */ + /** holds the original pattern that was added */ uint8_t *original_pat; - /* pattern id */ + /** pattern id */ uint32_t id; uint16_t offset; uint16_t depth; - /* sid(s) for this pattern */ + /** sid(s) for this pattern */ uint32_t sids_size; SigIntId *sids; - /* only used at ctx init time, when this structure is part of a hash + /** only used at ctx init time, when this structure is part of a hash * table. */ struct SCHSPattern_ *next; } SCHSPattern; typedef struct SCHSCtx_ { - /* hash used during ctx initialization */ + /** hash used during ctx initialization */ SCHSPattern **init_hash; - /* pattern database and pattern arrays. */ + /** pattern database and pattern arrays. */ void *pattern_db; - /* size of database, for accounting. */ + /** size of database, for accounting. */ size_t hs_db_size; } SCHSCtx; typedef struct SCHSThreadCtx_ { - /* Hyperscan scratch space region for this thread, capable of handling any + /** Hyperscan scratch space region for this thread, capable of handling any * database that has been compiled. */ void *scratch; - /* size of scratch space, for accounting. */ + /** size of scratch space, for accounting. */ size_t scratch_size; } SCHSThreadCtx; @@ -81,12 +81,20 @@ typedef struct PatternDatabase_ { hs_database_t *hs_db; uint32_t pattern_cnt; - /* Reference count: number of MPM contexts using this pattern database. */ + /** Reference count: number of MPM contexts using this pattern database. */ uint32_t ref_cnt; - /* Signals if the matcher has loaded/saved the pattern database to disk */ + /** Signals if the matcher has loaded/saved the pattern database to disk */ bool cached; + /** Matcher will not cache this pattern DB */ + bool no_cache; } PatternDatabase; +typedef struct PatternDatabaseCache_ { + uint32_t hs_cacheable_dbs_cnt; + uint32_t hs_dbs_cache_loaded_cnt; + uint32_t hs_dbs_cache_saved_cnt; +} PatternDatabaseCache; + const char *HSErrorToStr(hs_error_t error_code); #endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index 9246741cdf80..7d6dc44736f5 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -33,16 +33,19 @@ #include "detect-engine-build.h" #include "conf.h" +#include "util-conf.h" #include "util-debug.h" #include "util-unittest.h" #include "util-unittest-helper.h" #include "util-memcmp.h" #include "util-mpm-hs.h" +#include "util-mpm-hs-cache.h" #include "util-mpm-hs-core.h" #include "util-memcpy.h" #include "util-hash.h" #include "util-hash-lookup3.h" #include "util-hyperscan.h" +#include "util-path.h" #ifdef BUILD_HYPERSCAN @@ -551,6 +554,7 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt) pd->pattern_cnt = pattern_cnt; pd->ref_cnt = 0; pd->hs_db = NULL; + pd->cached = false; /* alloc the pattern array */ pd->parray = (SCHSPattern **)SCCalloc(pd->pattern_cnt, sizeof(SCHSPattern *)); @@ -675,7 +679,7 @@ static int CompileDataExtensionsInit(hs_expr_ext_t **ext, const SCHSPattern *p) * \param SCHSCompileData* [in] Pointer to the compile data. * \retval 0 On success, negative value on failure. */ -static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd) +static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd, bool disk_cache) { /* Check global hash table to see if we've seen this pattern database * before, and reuse the Hyperscan database if so. */ @@ -690,6 +694,26 @@ static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd) CompileDataFree(cd); *pd = pd_cached; return 0; + } else if (disk_cache) { + pd_cached = *pd; + uint64_t db_lookup_hash = HSHashDb(pd_cached); + if (HSLoadCache(&pd_cached->hs_db, db_lookup_hash) == 0) { + pd_cached->ref_cnt = 1; + pd_cached->cached = true; + if (HSScratchAlloc(pd_cached->hs_db) != 0) { + goto recover; + } + if (HashTableAdd(g_db_table, pd_cached, 1) < 0) { + goto recover; + } + CompileDataFree(cd); + return 0; + + recover: + pd_cached->ref_cnt = 0; + pd_cached->cached = false; + return -1; + } } return -1; // not cached @@ -750,6 +774,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) } HSPatternArrayInit(ctx, pd); + pd->no_cache = cache_to_disk ? false : true; /* Serialise whole database compilation as a relatively easy way to ensure * dedupe is safe. */ SCMutexLock(&g_db_table_mutex); @@ -758,7 +783,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) goto error; } - if (PatternDatabaseGetCached(&pd, cd) == 0 && pd != NULL) { + if (PatternDatabaseGetCached(&pd, cd, cache_to_disk) == 0 && pd != NULL) { ctx->pattern_db = pd; if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) { SCMutexUnlock(&g_db_table_mutex); @@ -800,6 +825,22 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk) return -1; } +/** + * \brief Cache the loaded ruleset + */ +static int SCHSCacheRuleset(void) +{ + SCLogDebug("Caching the loaded ruleset "); + PatternDatabaseCache pd_stats = { 0 }; + SCMutexLock(&g_db_table_mutex); + HashTableIterate(g_db_table, HSSaveCacheIterator, &pd_stats); + SCMutexUnlock(&g_db_table_mutex); + SCLogInfo("%u rule groups cached (%u newly cached) of total %u cacheable groups", + pd_stats.hs_dbs_cache_loaded_cnt + pd_stats.hs_dbs_cache_saved_cnt, + pd_stats.hs_dbs_cache_saved_cnt, pd_stats.hs_cacheable_dbs_cnt); + return 0; +} + /** * \brief Init the mpm thread context. * @@ -1110,7 +1151,7 @@ void MpmHSRegister(void) mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS; mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI; mpm_table[MPM_HS].Prepare = SCHSPreparePatterns; - mpm_table[MPM_HS].CacheRuleset = NULL; + mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset; mpm_table[MPM_HS].Search = SCHSSearch; mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo; mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats;