Initial commit

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.github/.templateMarker

@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"

.github/workflows/CI.yml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,10 @@
+__pycache__
+*.py[co]
+/*.egg-info
+*.srctrlbm
+*.srctrldb
+build
+dist
+.eggs
+monkeytype.sqlite3
+/.ipynb_checkpoints

.gitlab-ci.yml

@@ -0,0 +1,51 @@
+image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest
+
+variables:
+  DOCKER_DRIVER: overlay2
+  SAST_ANALYZER_IMAGE_TAG: latest
+  SAST_DISABLE_DIND: "true"
+  SAST_CONFIDENCE_LEVEL: 5
+  CODECLIMATE_VERSION: latest
+
+include:
+  - template: SAST.gitlab-ci.yml
+  - template: Code-Quality.gitlab-ci.yml
+  - template: License-Management.gitlab-ci.yml
+
+build:
+  tags:
+    - shared
+    - linux
+  stage: build
+  variables:
+    GIT_DEPTH: "1"
+    PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+
+  before_script:
+    - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables`
+    - apt-get update
+    # todo:
+    #- apt-get -y install 
+    #- pip3 install --upgrade 
+    #- python3 ./fix_python_modules_paths.py
+
+  script:
+    - python3 -m build -nw bdist_wheel
+    - mv ./dist/*.whl ./dist/signatureCandidatesAutoDiscover-0.CI-py3-none-any.whl
+    - pip3 install --upgrade ./dist/*.whl
+    - coverage run --source=signatureCandidatesAutoDiscover -m --branch pytest --junitxml=./rspec.xml ./tests/test.py
+    - coverage report -m
+    - coverage xml
+
+  coverage: "/^TOTAL(?:\\s+\\d+){4}\\s+(\\d+%).+/"
+
+  cache:
+    paths:
+      - $PYTHONUSERBASE
+
+  artifacts:
+    paths:
+      - dist
+    reports:
+      junit: ./rspec.xml
+      cobertura: ./coverage.xml

Code_Of_Conduct.md

@@ -0,0 +1 @@
+No codes of conduct!

MANIFEST.in

@@ -0,0 +1,4 @@
+include UNLICENSE
+include *.md
+include tests
+include .editorconfig

ReadMe.md

@@ -0,0 +1,40 @@
+signatureCandidatesAutoDiscover.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/)
+==================================
+~~[wheel (GitLab)](https://gitlab.com/KOLANICH-tools/signatureCandidatesAutoDiscover.py/-/jobs/artifacts/master/raw/dist/signatureCandidatesAutoDiscover-0.CI-py3-none-any.whl?job=build)~~
+[wheel (GHA via `nightly.link`)](https://nightly.link/KOLANICH-tools/signatureCandidatesAutoDiscover.py/workflows/CI/master/signatureCandidatesAutoDiscover-0.CI-py3-none-any.whl)
+~~![GitLab Build Status](https://gitlab.com/KOLANICH-tools/signatureCandidatesAutoDiscover.py/badges/master/pipeline.svg)~~
+~~![GitLab Coverage](https://gitlab.com/KOLANICH-tools/signatureCandidatesAutoDiscover.py/badges/master/coverage.svg)~~
+~~![GitHub Actions](https://github.com/KOLANICH-tools/signatureCandidatesAutoDiscover.py/workflows/CI/badge.svg)](https://github.com/KOLANICH-tools/signatureCandidatesAutoDiscover.py/actions/)~~
+[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH-tools/signatureCandidatesAutoDiscover.py.svg)](https://libraries.io/github/KOLANICH-tools/signatureCandidatesAutoDiscover.py)
+[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py)
+
+This is a tool that helps you to automatically discover signatures used in file formats and/or protocols using disassembly listings of the software and the dataset of the files used by it.
+
+It relies on the following assumptions, causing the limitations of the tool:
+1. in order to create a valid file in a certain format using signatures software has to write the signature somewhere.
+2. the software is not obfuscated or packed and the decompiler/disassembler has done its work correctly
+3. the signature is usually `4` bytes, so `uint32_t`. 4 bytes give enough low probability of false identification of file format.
+4. when using in-memory structures, including memory-mapped files the signature is usually aligned **within ith own struct** (it may be not aligned relative to root struct base). It makes appending it easier.
+5. when reading signature from files using stream API (`fread` and so on) it is usually convenient for a programmer to read the block as whole rather than read it byte-by-byte in a random order.
+6. when comparing/writing signatures read this way the compiler will optimize compares and writes by using the corresponding integer types.
+7. the compiler will put the signatures into immediate values into the instructions
+8. signatures should have low probability to occur by chance.
+
+
+So the principle of the tool is simple:
+1. Read the disassembly/decompilation of the software and identify the instructions doing 4-byte integer assignments and comparisons. Collect their operands.
+2. Because certain low-entropy integers like `0x000000FF` will likely occur by chance, filter them out heuristically.
+3. Check the presence of the remaining candidates within files, count occurences, print the listing.
+4. Remove the integers seen only once within the dataset.
+5. Print the rest as a nice table.
+
+
+## How to use
+
+0. Get prior knowledge that the format in question uses signatures.
+1. Create a dataset of files containing the signatures.
+2. Collect enough different implemetations of the software dealing with the format. Disassemble and/or decompile it with `retdec` or other decompiler.
+3. Execute the tool within the directory with decompilation results, providing it with the glob expression to the files containing the data.
+4. The tool will give you the list of signature candidates with their counts of occurences within the dataset and different representations convenient for grepping within disassembly listings and decompilation results.
+
+

UNLICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org/>

pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.2.0", "setuptools_scm[toml]>=3.4.3"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "signatureCandidatesAutoDiscover"
+readme = "ReadMe.md"
+description = "A tool for automatic discovery of candidates for signatures used within a file format."
+authors = [{name = "KOLANICH"}]
+classifiers = [
+	"Development Status :: 4 - Beta",
+	"Environment :: Other Environment",
+	"Intended Audience :: Developers",
+	"License :: Public Domain",
+	"Operating System :: OS Independent",
+	"Programming Language :: Python",
+	"Programming Language :: Python :: 3",
+	"Programming Language :: Python :: 3 :: Only",
+	"Topic :: Software Development :: Libraries :: Python Modules",
+]
+keywords = ["signatureCandidatesAutoDiscover"]
+license = {text = "Unlicense"}
+requires-python = ">=3.4"
+dynamic = ["version"]
+
+[project.scripts]
+signatureCandidatesAutoDiscover = "signatureCandidatesAutoDiscover:CLI.run"
+
+[project.urls]
+Homepage = "https://codeberg.org/KOLANICH-tools/signatureCandidatesAutoDiscover.py"
+
+[tool.setuptools]
+zip-safe = true
+py-modules = ["signatureCandidatesAutoDiscover"]
+
+[tool.setuptools_scm]