Merge pull request intel#3 from G-Core/gcore-linux-rex

Gcore linux rex

.gitignore

@@ -25,6 +25,15 @@ autojunk
 .libs
 bin
 
+# kernel binaries
+*.o.d
+*.cmd
+*.ko
+*.mod
+*.mod.c
+modules.order
+Module.symvers
+
 # Merge files created by git.
 *.orig
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 2.8.11)
+cmake_minimum_required (VERSION 3.12)
 project (hyperscan C CXX)
 
 set (HS_MAJOR_VERSION 5)
@@ -505,7 +505,7 @@ if (NOT WIN32)
     endforeach()
 
     configure_file(libhs.pc.in libhs.pc @ONLY) # only replace @ quoted vars
-    install(FILES ${CMAKE_BINARY_DIR}/libhs.pc
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libhs.pc
         DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 endif()
 
@@ -579,7 +579,7 @@ set (hs_exec_common_SRCS
 
 set (hs_exec_SRCS
     ${hs_HEADERS}
-    src/hs_version.h
+    ${PROJECT_BINARY_DIR}/hs_version.h
     src/ue2common.h
     src/allocator.h
     src/crc32.c
@@ -730,13 +730,13 @@ set (hs_exec_avx2_SRCS
 
 SET (hs_compile_SRCS
     ${hs_HEADERS}
+    ${PROJECT_BINARY_DIR}/hs_version.h
     src/crc32.h
     src/database.h
     src/grey.cpp
     src/grey.h
     src/hs.cpp
     src/hs_internal.h
-    src/hs_version.h
     src/scratch.h
     src/state.h
     src/ue2common.h
@@ -1412,3 +1412,5 @@ option(BUILD_EXAMPLES "Build Hyperscan example code (default TRUE)" TRUE)
 if(NOT WIN32 AND BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
+
+option(TEST_XDPSCAN "Build Hyperscan unit tests for XDP module" FALSE)

README.md

@@ -1,43 +1,34 @@
-# Hyperscan
+# eBPF REGEX helper
 
-Hyperscan is a high-performance multiple regex matching library. It follows the
-regular expression syntax of the commonly-used libpcre library, but is a
-standalone library with its own C API.
-
-Hyperscan uses hybrid automata techniques to allow simultaneous matching of
-large numbers (up to tens of thousands) of regular expressions and for the
-matching of regular expressions across streams of data.
-
-Hyperscan is typically used in a DPI library stack.
+The `linux-rex` is a loadable kernel module providing eBPF helper functions
+for processing regular expressions. It uses Hyperscan as a runtime and
+configuration tool.
 
-# Documentation
+License: GPLv2
 
-Information on building the Hyperscan library and using its API is available in
-the [Developer Reference Guide](http://intel.github.io/hyperscan/dev-reference/).
+## Documentation
 
-# License
+Refer to [the project wiki pages](https://github.com/G-Core/linux-regex-module/wiki)
+to find all the necessary documentation.
 
-Hyperscan is licensed under the BSD License. See the LICENSE file in the
-project repository.
+## Talks and videos
 
-# Versioning
+The `linux-rex` module [was introduced on the Netdev 0x16, Technical Conference
+on Linux Networking](https://netdevconf.info/0x16/session.html?When-regular-expressions-meet-XDP#)
 
-The `master` branch on Github will always contain the most recent release of
-Hyperscan. Each version released to `master` goes through QA and testing before
-it is released; if you're a user, rather than a developer, this is the version
-you should be using.
+## Hyperscan
 
-Further development towards the next release takes place on the `develop`
-branch.
+Hyperscan is a high-performance multiple regex matching library. It follows the
+regular expression syntax of the commonly-used libpcre library, but is a
+standalone library with its own C API.
 
-# Get Involved
+Hyperscan uses hybrid automata techniques to allow simultaneous matching of
+large numbers (up to tens of thousands) of regular expressions and for the
+matching of regular expressions across streams of data.
 
-The official homepage for Hyperscan is at [www.hyperscan.io](https://www.hyperscan.io).
+Hyperscan is typically used in a DPI library stack.
 
-If you have questions or comments, we encourage you to [join the mailing
-list](https://lists.01.org/mailman/listinfo/hyperscan). Bugs can be filed by
-sending email to the list, or by creating an issue on Github.
+More information can be found at
+[Hyperscan project repo](https://github.com/intel/hyperscan)
 
-If you wish to contact the Hyperscan team at Intel directly, without posting
-publicly to the mailing list, send email to
-[hyperscan@intel.com](mailto:hyperscan@intel.com).
+License: BSD

cmake/FindLibBpf.cmake

@@ -0,0 +1,35 @@
+# - Try to find libbpf
+# Once done this will define
+#  LIBBPF_FOUND        - System has libbpf
+#  LIBBPF_INCLUDE_DIRS - The libbpf include directories
+#  LIBBPF_LIBRARIES    - The libraries needed to use libbpf
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PC_LIBBPF QUIET libbpf)
+pkg_check_modules(PC_LIBELF QUIET libelf)
+pkg_check_modules(PC_ZLIB QUIET zlib)
+
+find_path(LIBBPF_INCLUDE_DIR
+  NAMES bpf/bpf.h
+  HINTS ${PC_LIBBPF_INCLUDE_DIRS}
+  )
+find_library(LIBBPF_LIBRARY
+  NAMES bpf
+  HINTS ${PC_LIBBPF_LIBRARY_DIRS}
+  )
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set LIBBPF_FOUND
+# to TRUE if all listed variables are TRUE and the requested version
+# matches.
+find_package_handle_standard_args(LibBpf REQUIRED_VARS
+                                  LIBBPF_LIBRARY LIBBPF_INCLUDE_DIR
+                                  VERSION_VAR LIBBPF_VERSION
+                                  )
+
+if(LIBBPF_FOUND)
+  set(LIBBPF_LIBRARIES     ${LIBBPF_LIBRARY} ${PC_LIBELF_LIBRARIES} ${PC_ZLIB_LIBRARIES})
+  set(LIBBPF_INCLUDE_DIRS  ${LIBBPF_INCLUDE_DIR} ${PC_LIBELF_INCLUDE_DIRS} ${PC_ZLIB_INCLUDE_DIRS})
+endif()
+
+mark_as_advanced(LIBBPF_INCLUDE_DIR LIBBPF_LIBRARY)

cmake/build_wrapper.sh

@@ -17,7 +17,7 @@ KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX)
 LIBC_SO=$("$@" --print-file-name=libc.so.6)
 cp ${KEEPSYMS_IN} ${KEEPSYMS}
 # get all symbols from libc and turn them into patterns
-nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ ]*\).*/^\1$/' >> ${KEEPSYMS}
+nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS}
 # build the object
 "$@"
 # rename the symbols in the object

debian.dkms/README.Debian

@@ -0,0 +1 @@
+dkms build linux-rex/0.1

debian.dkms/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -xe
+
+kernel_source_dir="$1"
+linux_image=/boot/"vmlinuz-${kernelver}"
+shift 1
+
+# Make our own source tree and extract vmlinux into it.
+subdirs=$(ls -A "${kernel_source_dir}"/)
+mkdir -p linux
+for d in $subdirs; do
+    ln -s "${kernel_source_dir}"/"$d" linux/"$d"
+done
+
+linux/scripts/extract-vmlinux "${linux_image}" \
+    > linux/vmlinux
+
+exec make -C linux "$@"

debian.dkms/changelog

@@ -0,0 +1,5 @@
+linux-rex (0.1) unstable; urgency=medium
+
+  * Initial release
+
+ -- Sergey Nizovtsev <sn@tempesta-tech.com>  Tue, 14 Jul 2022 16:24:30 +0300

debian.dkms/clean

@@ -0,0 +1,7 @@
+*.o
+Module.symvers
+*.ko
+.*.cmd
+xdp_rex.mod
+xdp_rex.mod.c
+modules.order

debian.dkms/control

@@ -0,0 +1,17 @@
+Source: linux-rex
+Section: kernel
+Priority: optional
+Maintainer: Ivan Koveshnikov <ivan.koveshnikov@gcore.com>
+Build-Depends: debhelper-compat (= 13), dkms
+Standards-Version: 4.6.0
+Homepage: https://github.com/G-Core/linux-regex-module/
+Rules-Requires-Root: no
+
+Package: rex-dkms
+Architecture: all
+Depends: dwarves, zstd, ${misc:Depends}
+Description: Hyperscan loadable module for linux.
+  .
+  A eBPF helper to process regular expressions inside eBPF context.
+  Only a patched Linux kernel 5.17 is supported only.
+  Refer to https://github.com/G-Core/linux-regex-module/ for more documentation.

debian.dkms/dkms

@@ -0,0 +1,9 @@
+PACKAGE_NAME="linux-rex"
+PACKAGE_VERSION="#MODULE_VERSION#"
+BUILD_EXCLUSIVE_KERNEL="^5[.]17[.]"
+BUILD_EXCLUSIVE_ARCH=x86_64
+MAKE[0]="./build.sh ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build -j${parallel_jobs}"
+CLEAN="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build clean"
+AUTOINSTALL=yes
+BUILT_MODULE_NAME[0]="xdp_rex"
+DEST_MODULE_LOCATION[0]=/extra

debian.dkms/rex-dkms.install

@@ -0,0 +1,140 @@
+debian/build.sh				${env:DKMS_DIR}
+src/Kbuild				${env:DKMS_DIR}
+src/kmod				${env:DKMS_DIR}
+src/allocator.h				${env:DKMS_DIR}
+src/alloc.c				${env:DKMS_DIR}
+src/crc32.h				${env:DKMS_DIR}
+src/hs_common.h				${env:DKMS_DIR}
+src/hs_compile.h			${env:DKMS_DIR}
+src/hs.h				${env:DKMS_DIR}
+src/hs_internal.h			${env:DKMS_DIR}
+src/hs_runtime.h			${env:DKMS_DIR}
+src/report.h				${env:DKMS_DIR}
+src/state.h				${env:DKMS_DIR}
+src/ue2common.h				${env:DKMS_DIR}
+src/scratch.{c,h}			${env:DKMS_DIR}
+src/runtime.c				${env:DKMS_DIR}
+src/database.{c,h}			${env:DKMS_DIR}
+src/hs_version.c			${env:DKMS_DIR}
+src/stream_compress.{c,h}		${env:DKMS_DIR}
+src/stream_compress_impl.h		${env:DKMS_DIR}
+src/smallwrite/smallwrite_internal.h	${env:DKMS_DIR}/smallwrite
+src/fdr/flood_runtime.h			${env:DKMS_DIR}/fdr
+src/fdr/fdr_internal.h			${env:DKMS_DIR}/fdr
+src/fdr/fdr_loadval.h			${env:DKMS_DIR}/fdr
+src/fdr/fdr_confirm_runtime.h		${env:DKMS_DIR}/fdr
+src/fdr/fdr_confirm.h			${env:DKMS_DIR}/fdr
+src/fdr/fdr.{c,h}			${env:DKMS_DIR}/fdr
+src/fdr/teddy_avx2.c			${env:DKMS_DIR}/fdr
+src/fdr/teddy_internal.h		${env:DKMS_DIR}/fdr
+src/fdr/teddy_runtime_common.h		${env:DKMS_DIR}/fdr
+src/fdr/teddy.{c,h}			${env:DKMS_DIR}/fdr
+src/hwlm/hwlm_internal.h		${env:DKMS_DIR}/hwlm
+src/hwlm/hwlm.{c,h}			${env:DKMS_DIR}/hwlm
+src/hwlm/noodle_internal.h		${env:DKMS_DIR}/hwlm
+src/hwlm/noodle_engine.{c,h}		${env:DKMS_DIR}/hwlm
+src/hwlm/noodle_engine_*.c		${env:DKMS_DIR}/hwlm
+src/nfa/vermicelli.h			${env:DKMS_DIR}/nfa
+src/nfa/vermicelli_run.h		${env:DKMS_DIR}/nfa
+src/nfa/vermicelli_sse.h		${env:DKMS_DIR}/nfa
+src/nfa/nfa_internal.h			${env:DKMS_DIR}/nfa
+src/nfa/nfa_rev_api.h			${env:DKMS_DIR}/nfa
+src/nfa/accel.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/castle_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/castle.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/gough_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/gough.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/lbr_internal.h			${env:DKMS_DIR}/nfa
+src/nfa/lbr_common_impl.h		${env:DKMS_DIR}/nfa
+src/nfa/lbr.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/limex.h				${env:DKMS_DIR}/nfa
+src/nfa/limex_common_impl.h		${env:DKMS_DIR}/nfa
+src/nfa/limex_context.h			${env:DKMS_DIR}/nfa
+src/nfa/limex_exceptional.h		${env:DKMS_DIR}/nfa
+src/nfa/limex_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/limex_limits.h			${env:DKMS_DIR}/nfa
+src/nfa/limex_ring.h			${env:DKMS_DIR}/nfa
+src/nfa/limex_runtime_impl.h		${env:DKMS_DIR}/nfa
+src/nfa/limex_runtime.h			${env:DKMS_DIR}/nfa
+src/nfa/limex_shuffle.h			${env:DKMS_DIR}/nfa
+src/nfa/limex_state_impl.h		${env:DKMS_DIR}/nfa
+src/nfa/limex_64.c			${env:DKMS_DIR}/nfa
+src/nfa/limex_accel.{c,h}		${env:DKMS_DIR}/nfa
+src/nfa/limex_native.c			${env:DKMS_DIR}/nfa
+src/nfa/limex_simd128.c			${env:DKMS_DIR}/nfa
+src/nfa/limex_simd256.c			${env:DKMS_DIR}/nfa
+src/nfa/limex_simd384.c			${env:DKMS_DIR}/nfa
+src/nfa/limex_simd512.c			${env:DKMS_DIR}/nfa
+src/nfa/mcclellan_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/mcclellan_common_impl.h		${env:DKMS_DIR}/nfa
+src/nfa/mcclellan.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/mcsheng_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/mcsheng.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/mcsheng_data.c			${env:DKMS_DIR}/nfa
+src/nfa/callback.h			${env:DKMS_DIR}/nfa
+src/nfa/mpv_internal.h			${env:DKMS_DIR}/nfa
+src/nfa/mpv.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/nfa_api.h			${env:DKMS_DIR}/nfa
+src/nfa/nfa_api_util.h			${env:DKMS_DIR}/nfa
+src/nfa/nfa_api_queue.h			${env:DKMS_DIR}/nfa
+src/nfa/nfa_api_dispatch.c		${env:DKMS_DIR}/nfa
+src/nfa/repeat_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/repeat.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/sheng_defs.h			${env:DKMS_DIR}/nfa
+src/nfa/sheng_impl.h			${env:DKMS_DIR}/nfa
+src/nfa/sheng_impl4.h			${env:DKMS_DIR}/nfa
+src/nfa/sheng_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/sheng.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/shufti.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/tamarama_internal.h		${env:DKMS_DIR}/nfa
+src/nfa/tamarama.{c,h}			${env:DKMS_DIR}/nfa
+src/nfa/truffle.{c,h}			${env:DKMS_DIR}/nfa
+src/rose/rose.h				${env:DKMS_DIR}/rose
+src/rose/rose_types.h			${env:DKMS_DIR}/rose
+src/rose/rose_common.h			${env:DKMS_DIR}/rose
+src/rose/rose_internal.h		${env:DKMS_DIR}/rose
+src/rose/rose_program.h			${env:DKMS_DIR}/rose
+src/rose/counting_miracle.h		${env:DKMS_DIR}/rose
+src/rose/infix.h			${env:DKMS_DIR}/rose
+src/rose/miracle.h			${env:DKMS_DIR}/rose
+src/rose/runtime.h			${env:DKMS_DIR}/rose
+src/rose/validate_mask.h		${env:DKMS_DIR}/rose
+src/rose/validate_shufti.h		${env:DKMS_DIR}/rose
+src/rose/block.c			${env:DKMS_DIR}/rose
+src/rose/catchup.{c,h}			${env:DKMS_DIR}/rose
+src/rose/init.{c,h}			${env:DKMS_DIR}/rose
+src/rose/match.{c,h}			${env:DKMS_DIR}/rose
+src/rose/program_runtime.{c,h}		${env:DKMS_DIR}/rose
+src/rose/stream.c			${env:DKMS_DIR}/rose
+src/rose/stream_long_lit.h		${env:DKMS_DIR}/rose
+src/rose/stream_long_lit_hash.h		${env:DKMS_DIR}/rose
+src/som/som_operation.h			${env:DKMS_DIR}/som
+src/som/som_runtime.{c,h}		${env:DKMS_DIR}/som
+src/som/som_stream.{c,h}		${env:DKMS_DIR}/som
+src/util/arch.h				${env:DKMS_DIR}/util
+src/util/bitutils.h			${env:DKMS_DIR}/util
+src/util/compare.h			${env:DKMS_DIR}/util
+src/util/copybytes.h			${env:DKMS_DIR}/util
+src/util/cpuid_inline.h			${env:DKMS_DIR}/util
+src/util/exhaust.h			${env:DKMS_DIR}/util
+src/util/fatbit.h			${env:DKMS_DIR}/util
+src/util/intrinsics.h			${env:DKMS_DIR}/util
+src/util/join.h				${env:DKMS_DIR}/util
+src/util/logical.h			${env:DKMS_DIR}/util
+src/util/pack_bits.h			${env:DKMS_DIR}/util
+src/util/partial_store.h		${env:DKMS_DIR}/util
+src/util/popcount.h			${env:DKMS_DIR}/util
+src/util/pqueue.h			${env:DKMS_DIR}/util
+src/util/pqueue.h			${env:DKMS_DIR}/util
+src/util/scatter.h			${env:DKMS_DIR}/util
+src/util/scatter_runtime.h		${env:DKMS_DIR}/util
+src/util/unaligned.h			${env:DKMS_DIR}/util
+src/util/uniform_ops.h			${env:DKMS_DIR}/util
+src/util/cpuid_flags.{c,h}		${env:DKMS_DIR}/util
+src/util/masked_move.{c,h}		${env:DKMS_DIR}/util
+src/util/multibit_internal.h		${env:DKMS_DIR}/util
+src/util/multibit_compress.h		${env:DKMS_DIR}/util
+src/util/multibit.{c,h}			${env:DKMS_DIR}/util
+src/util/simd_utils.{c,h}		${env:DKMS_DIR}/util
+src/util/simd_types.h			${env:DKMS_DIR}/util
+src/util/state_compress.{c,h}		${env:DKMS_DIR}/util