feat: generic anomaly detection (#226)

Explain what this PR does.

---------

Signed-off-by: Nandita Koppisetty <nandita.iitkgp@gmail.com>

udf/anomaly-detection/.codecov.yml

@@ -0,0 +1,10 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 20%
+    patch:
+      default:
+        target: auto
+        threshold: 20%

udf/anomaly-detection/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+branch = True
+parallel = True
+source = numalogic-prometheus
+omit = tests/*

udf/anomaly-detection/.dockerignore

@@ -0,0 +1,16 @@
+tests/
+manifests/
+docs/
+.github
+CHANGELOG.md
+CODE_OF_CONDUCT.md
+CONTRIBUTING.md
+README.md
+LICENSE
+.gitignore
+.idea/
+.codecov.yml
+.coveragerc
+.flake8
+.hack/
+.venv/

udf/anomaly-detection/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, F821
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist
+max-complexity = 10
+max-line-length = 110

udf/anomaly-detection/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+prometheus-serde/vendor
+prometheus-serde/dist
+prometheus-serde/coverage.out
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Jetbrains project
+.idea/
+
+# Mac related
+.DS_Store

udf/anomaly-detection/Dockerfile

@@ -0,0 +1,57 @@
+####################################################################################################
+# builder: install needed dependencies
+####################################################################################################
+
+FROM python:3.10-slim-bullseye AS builder
+
+ENV PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PIP_NO_CACHE_DIR=on \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100 \
+  POETRY_VERSION=1.4.2 \
+  POETRY_HOME="/opt/poetry" \
+  POETRY_VIRTUALENVS_IN_PROJECT=true \
+  POETRY_NO_INTERACTION=1 \
+  PYSETUP_PATH="/opt/pysetup" \
+  VENV_PATH="/opt/pysetup/.venv"
+
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        curl \
+        wget \
+        # deps for building python deps
+        build-essential \
+    && apt-get install -y git \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* \
+    \
+    # install dumb-init
+    && wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \
+    && chmod +x /dumb-init \
+    && curl -sSL https://install.python-poetry.org | python3 -
+
+####################################################################################################
+# udf: used for running the udf vertices
+####################################################################################################
+FROM builder AS udf
+
+WORKDIR $PYSETUP_PATH
+COPY ./pyproject.toml ./poetry.lock ./
+COPY requirements ./requirements
+
+RUN poetry install --no-cache --no-root && \
+    poetry run pip install --no-cache -r requirements/requirements-torch.txt && \
+    rm -rf ~/.cache/pypoetry/
+
+ADD . /app
+WORKDIR /app
+
+RUN chmod +x entry.sh
+
+ENTRYPOINT ["/dumb-init", "--"]
+CMD ["/app/entry.sh"]
+
+EXPOSE 5000

udf/anomaly-detection/Makefile

@@ -0,0 +1,40 @@
+# Check Python
+PYTHON:=$(shell command -v python 2> /dev/null)
+ifndef PYTHON
+PYTHON:=$(shell command -v python3 2> /dev/null)
+endif
+ifndef PYTHON
+$(error "Python is not available, please install.")
+endif
+
+clean:
+	@rm -rf build dist .eggs *.egg-info
+	@rm -rf .benchmarks .coverage coverage.xml htmlcov report.xml .tox
+	@find . -type d -name '.mypy_cache' -exec rm -rf {} +
+	@find . -type d -name '__pycache__' -exec rm -rf {} +
+	@find . -type d -name '*pytest_cache*' -exec rm -rf {} +
+	@find . -type f -name "*.py[co]" -exec rm -rf {} +
+
+format: clean
+	poetry run black src/ tests/
+	poetry run black starter.py
+
+lint: format
+	poetry run flake8 .
+
+# install all dependencies
+setup:
+	poetry install --with dev --all-extras
+
+test:
+	poetry run pytest -v tests/
+
+requirements:
+	poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+requirements-dev:
+	poetry export -f requirements.txt --with dev --output requirements-dev.txt --without-hashes
+
+pipeline:
+	kustomize build manifests/prerequisites | kubectl apply -f -
+	kustomize build manifests/ | kubectl apply -f -

udf/anomaly-detection/README.md

@@ -0,0 +1,3 @@
+# Anomaly Detection
+
+![anomalydetection](./docs/images/anomalydetection.png)

udf/anomaly-detection/config/default-configs/config.yaml

@@ -0,0 +1,28 @@
+configs:
+  - name: "service-mesh"
+    composite_keys: [ "destination_asset_id", "env" ]
+    metrics: [ "error_rate" , "error_count" ]
+  - name: "service-mesh-s2s"
+    composite_keys: ["source_asset_id, destination_asset_id", "env"]
+    metrics: [ "error_rate", "error_count" ]
+  - name: "default-argorollouts"
+    composite_keys: [ "namespace", "name", "app", "rollouts_pod_template_hash" ]
+    metrics: [ "namespace_app_rollouts_http_request_error_rate"]
+    metric_configs:
+      - metric: "namespace_app_rollouts_http_request_error_rate"
+        static_threshold:
+          weight: 0.7
+  - name: "default-argocd"
+    composite_keys: [ "namespace", "name" ]
+    metrics: [ "namespace_app_http_server_requests_errors",
+               "namespace_app_http_server_requests_error_rate",
+               "namespace_app_http_server_requests_latency",
+               "namespace_app_cpu_utilization",
+               "namespace_app_memory_utilization" ]
+    metric_configs:
+      - metric: "namespace_app_cpu_utilization"
+        static_threshold:
+          upper_limit: 80
+      - metric: "namespace_app_memory_utilization"
+        static_threshold:
+          upper_limit: 80

udf/anomaly-detection/config/default-configs/numalogic_config.yaml

@@ -0,0 +1,21 @@
+model:
+  name: "SparseVanillaAE"
+  conf:
+    seq_len: 12
+    n_features: 1
+    encoder_layersizes:
+      - 16
+      - 8
+    decoder_layersizes:
+      - 8
+      - 16
+    dropout_p: 0.25
+trainer:
+  max_epochs: 30
+preprocess:
+  - name: "StandardScaler"
+threshold:
+  name: "StdDevThreshold"
+postprocess:
+  name: "TanhNorm"
+  stateful: false

udf/anomaly-detection/config/default-configs/pipeline_config.yaml

@@ -0,0 +1,5 @@
+redis_conf:
+  host: "isbsvc-redis-isbs-redis-svc.numalogic-rollouts.svc"
+  port: 26379
+  expiry: 360
+  master_name: "mymaster"

udf/anomaly-detection/config/user-configs/config.yaml

@@ -0,0 +1,13 @@
+configs:
+  - name: "dev-devx-o11yfuzzygqlfederation-usw2-qal"
+    composite_keys: ["namespace", "name", "hash_id"]
+    metrics: ["namespace_rollout_api_error_rate", "namespace_rollout_api_latency"]
+  - name: "dev-devx-o11yfuzzygqlfederation-usw2-pprd"
+    composite_keys: [ "namespace", "name", "hash_id" ]
+    metrics: ["namespace_rollout_api_error_rate", "namespace_rollout_api_latency"]
+  - name: "dev-devx-o11yfuzzygqlfederation-usw2-stg"
+    composite_keys: [ "namespace", "name", "hash_id" ]
+    metrics: ["namespace_rollout_api_error_rate", "namespace_rollout_api_latency"]
+  - name: "dev-devx-o11yfuzzygqlfederation-usw2-prd"
+    composite_keys: [ "namespace", "name", "hash_id" ]
+    metrics: ["namespace_rollout_api_error_rate", "namespace_rollout_api_latency"]

udf/anomaly-detection/entry.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eux
+
+python starter.py