From 0bf86a7c15823ee80061317a997842211558bfa4 Mon Sep 17 00:00:00 2001 From: "Hong, Yang A" Date: Wed, 1 Mar 2023 14:42:27 +0000 Subject: [PATCH 1/4] scratch: remove quick validity check Roll back fix for github issue #350 About Scratch Usage: For compile time, scratch space is strongly recommended to be allocated immediately after database generation. For runtime, besides using scratch for corresponding database, Hyperscan also allows user to use larger scratch space allocated for another database. When multiple concurrent threads need to use the same databases and a new scratch space is required, cloning the largest one is always safe. This is realized based on API hs_scratch_size() and hs_clone_scratch(). Behaviors beyond above are discouraged and results are undefined. --- src/runtime.c | 37 +++++++++++++++++++++---------------- src/scratch.c | 4 +--- src/scratch.h | 3 +-- src/state.h | 5 +---- src/stream_compress_impl.h | 3 +-- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/runtime.c b/src/runtime.c index 3c2d65338..a055e5f4f 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -90,7 +90,7 @@ u8 *getHistory(char *state, const struct RoseEngine *t, u64a offset) { * callers. */ static really_inline -char validScratch(const struct hs_scratch *s, u32 crc) { +char validScratch(const struct RoseEngine *t, const struct hs_scratch *s) { if (!ISALIGNED_CL(s)) { DEBUG_PRINTF("bad alignment %p\n", s); return 0; @@ -101,12 +101,18 @@ char validScratch(const struct hs_scratch *s, u32 crc) { return 0; } - /* add quick rose sanity checks by db crc*/ - if (s->db_crc != crc) { - DEBUG_PRINTF("Improper scratch for current db\n"); + if (t->mode == HS_MODE_BLOCK && t->stateOffsets.end > s->bStateSize) { + DEBUG_PRINTF("bad state size\n"); return 0; } + if (t->queueCount > s->queueCount) { + DEBUG_PRINTF("bad queue count\n"); + return 0; + } + + /* TODO: add quick rose sanity checks */ + return 1; } @@ -329,7 +335,7 @@ hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data, return HS_DB_MODE_ERROR; } - if (unlikely(!validScratch(scratch, db->crc32))) { + if (unlikely(!validScratch(rose, scratch))) { return HS_INVALID; } @@ -503,7 +509,7 @@ void maintainHistoryBuffer(const struct RoseEngine *rose, char *state, static really_inline void init_stream(struct hs_stream *s, const struct RoseEngine *rose, - char init_history, u32 crc) { + char init_history) { char *state = getMultiState(s); if (init_history) { @@ -518,7 +524,6 @@ void init_stream(struct hs_stream *s, const struct RoseEngine *rose, s->rose = rose; s->offset = 0; - s->crc32 = crc; setStreamStatus(state, 0); roseInitState(rose, state); @@ -563,7 +568,7 @@ hs_error_t HS_CDECL hs_open_stream(const hs_database_t *db, return HS_NOMEM; } - init_stream(s, rose, 1, db->crc32); + init_stream(s, rose, 1); *stream = s; return HS_SUCCESS; @@ -751,7 +756,7 @@ hs_error_t HS_CDECL hs_reset_and_copy_stream(hs_stream_t *to_id, } if (onEvent) { - if (!scratch || !validScratch(scratch, to_id->crc32)) { + if (!scratch || !validScratch(to_id->rose, scratch)) { return HS_INVALID; } if (unlikely(markScratchInUse(scratch))) { @@ -977,7 +982,7 @@ hs_error_t HS_CDECL hs_scan_stream(hs_stream_t *id, const char *data, hs_scratch_t *scratch, match_event_handler onEvent, void *context) { if (unlikely(!id || !scratch || !data || - !validScratch(scratch, id->crc32))) { + !validScratch(id->rose, scratch))) { return HS_INVALID; } @@ -999,7 +1004,7 @@ hs_error_t HS_CDECL hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch, } if (onEvent) { - if (!scratch || !validScratch(scratch, id->crc32)) { + if (!scratch || !validScratch(id->rose, scratch)) { return HS_INVALID; } if (unlikely(markScratchInUse(scratch))) { @@ -1029,7 +1034,7 @@ hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags, } if (onEvent) { - if (!scratch || !validScratch(scratch, id->crc32)) { + if (!scratch || !validScratch(id->rose, scratch)) { return HS_INVALID; } if (unlikely(markScratchInUse(scratch))) { @@ -1044,7 +1049,7 @@ hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags, } // history already initialised - init_stream(id, id->rose, 0, id->crc32); + init_stream(id, id->rose, 0); return HS_SUCCESS; } @@ -1123,7 +1128,7 @@ hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db, return HS_DB_MODE_ERROR; } - if (unlikely(!validScratch(scratch, db->crc32))) { + if (unlikely(!validScratch(rose, scratch))) { return HS_INVALID; } @@ -1133,7 +1138,7 @@ hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db, hs_stream_t *id = (hs_stream_t *)(scratch->bstate); - init_stream(id, rose, 1, db->crc32); /* open stream */ + init_stream(id, rose, 1); /* open stream */ for (u32 i = 0; i < count; i++) { DEBUG_PRINTF("block %u/%u offset=%llu len=%u\n", i, count, id->offset, @@ -1248,7 +1253,7 @@ hs_error_t HS_CDECL hs_reset_and_expand_stream(hs_stream_t *to_stream, const struct RoseEngine *rose = to_stream->rose; if (onEvent) { - if (!scratch || !validScratch(scratch, to_stream->crc32)) { + if (!scratch || !validScratch(to_stream->rose, scratch)) { return HS_INVALID; } if (unlikely(markScratchInUse(scratch))) { diff --git a/src/scratch.c b/src/scratch.c index 5849380d6..9f6d77cdc 100644 --- a/src/scratch.c +++ b/src/scratch.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2022, Intel Corporation + * Copyright (c) 2015-2023, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -373,7 +373,6 @@ hs_error_t HS_CDECL hs_alloc_scratch(const hs_database_t *db, hs_scratch_free((*scratch)->scratch_alloc); } - proto->db_crc = db->crc32; hs_error_t alloc_ret = alloc_scratch(proto, scratch); hs_scratch_free(proto_tmp); /* kill off temp used for sizing */ if (alloc_ret != HS_SUCCESS) { @@ -381,7 +380,6 @@ hs_error_t HS_CDECL hs_alloc_scratch(const hs_database_t *db, return alloc_ret; } } else { - (*scratch)->db_crc = db->crc32; hs_scratch_free(proto_tmp); /* kill off temp used for sizing */ unmarkScratchInUse(*scratch); } diff --git a/src/scratch.h b/src/scratch.h index efaa68841..e3cd92452 100644 --- a/src/scratch.h +++ b/src/scratch.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2022, Intel Corporation + * Copyright (c) 2015-2023, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -171,7 +171,6 @@ struct match_deduper { */ struct ALIGN_CL_DIRECTIVE hs_scratch { u32 magic; - u32 db_crc; /**< identity of a scratch space, for validity check */ u8 in_use; /**< non-zero when being used by an API call. */ u32 queueCount; u32 activeQueueArraySize; /**< size of active queue array fatbit in bytes */ diff --git a/src/state.h b/src/state.h index 567001ea8..68600a910 100644 --- a/src/state.h +++ b/src/state.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2022, Intel Corporation + * Copyright (c) 2015-2023, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -57,9 +57,6 @@ struct hs_stream { /** \brief The current stream offset. */ u64a offset; - - /** \brief Identity of hs_stream, for scratch validity check. */ - u32 crc32; }; #define getMultiState(hs_s) ((char *)(hs_s) + sizeof(*(hs_s))) diff --git a/src/stream_compress_impl.h b/src/stream_compress_impl.h index ceea14a6f..f02543efa 100644 --- a/src/stream_compress_impl.h +++ b/src/stream_compress_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022, Intel Corporation + * Copyright (c) 2017-2023, Intel Corporation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -116,7 +116,6 @@ size_t JOIN(sc_, FN_SUFFIX)(const struct RoseEngine *rose, = ((STREAM_QUAL char *)stream) + sizeof(struct hs_stream); COPY_FIELD(stream->offset); - COPY_FIELD(stream->crc32); ASSIGN(stream->rose, rose); COPY(stream_body + ROSE_STATE_OFFSET_STATUS_FLAGS, 1); From ef16059c3ed74ad9d7690f222a2089579eb1eb1f Mon Sep 17 00:00:00 2001 From: "Hong, Yang A" Date: Thu, 23 Mar 2023 06:43:46 +0000 Subject: [PATCH 2/4] remove invalid nfa dump info --- src/nfa/nfa_dump_dispatch.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nfa/nfa_dump_dispatch.cpp b/src/nfa/nfa_dump_dispatch.cpp index b498fd956..bc8c175d3 100644 --- a/src/nfa/nfa_dump_dispatch.cpp +++ b/src/nfa/nfa_dump_dispatch.cpp @@ -75,7 +75,6 @@ namespace ue2 { DISPATCH_CASE(LBR_NFA_VERM, LbrVerm, dbnt_func); \ DISPATCH_CASE(LBR_NFA_NVERM, LbrNVerm, dbnt_func); \ DISPATCH_CASE(LBR_NFA_SHUF, LbrShuf, dbnt_func); \ - DISPATCH_CASE(LBR_NFA_VSHUF, LbrVShuf, dbnt_func); \ DISPATCH_CASE(LBR_NFA_TRUF, LbrTruf, dbnt_func); \ DISPATCH_CASE(CASTLE_NFA, Castle, dbnt_func); \ DISPATCH_CASE(SHENG_NFA, Sheng, dbnt_func); \ From bc9a37c0ffa610e5d1b9a636fbbbb6aff30c5978 Mon Sep 17 00:00:00 2001 From: "Hong, Yang A" Date: Wed, 19 Apr 2023 09:18:45 +0000 Subject: [PATCH 3/4] changelog: updates for 5.4.2 release --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 481f8fcf4..09b4a95cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ This is a list of notable changes to Hyperscan, in reverse chronological order. +## [5.4.2] 2023-04-19 +- Roll back bugfix for github issue #350: Besides using scratch for + corresponding database, Hyperscan also allows user to use larger scratch + allocated for another database. Users can leverage this property to achieve + safe scratch usage in multi-database scenarios. Behaviors beyond these are + discouraged and results are undefined. +- Fix hsdump issue due to invalid nfa type. + ## [5.4.1] 2023-02-20 - The Intel Hyperscan team is pleased to provide a bug fix release to our open source library. Intel also maintains an upgraded version available through your Intel sales representative. From 64731738a6634e8ab427a63c9c3618c2bf3cf172 Mon Sep 17 00:00:00 2001 From: "Hong, Yang A" Date: Wed, 19 Apr 2023 09:19:57 +0000 Subject: [PATCH 4/4] Bump version number for release --- CMakeLists.txt | 2 +- src/hs.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd6d2defb..7757916d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ project (hyperscan C CXX) set (HS_MAJOR_VERSION 5) set (HS_MINOR_VERSION 4) -set (HS_PATCH_VERSION 1) +set (HS_PATCH_VERSION 2) set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) diff --git a/src/hs.h b/src/hs.h index ca3d6dec4..3d3c5cdea 100644 --- a/src/hs.h +++ b/src/hs.h @@ -43,7 +43,7 @@ #define HS_MAJOR 5 #define HS_MINOR 4 -#define HS_PATCH 1 +#define HS_PATCH 2 #include "hs_compile.h" #include "hs_runtime.h"