Dockerized

Dockerfile

@@ -0,0 +1,34 @@
+# Use an official Python runtime as a parent image
+# Use bullseye with Python pre-installed
+FROM python:3.9-bullseye
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy all the things...
+ADD . /app/
+
+# Add essential packages and psycopg2 prerequisites then upgrade pip
+RUN apt-get update && apt-get install -y \
+    gcc \
+    python3-dev \
+    libpq-dev \
+&& pip install --upgrade pip
+
+# Install python packages and remove unnecessary packages
+RUN pip install --no-cache-dir -r requirements.txt \
+    && apt-get autoremove -y gcc python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Make Log file
+RUN mkdir -p /app/logs
+
+
+# Env Variables
+ENV APP_PORT=3000
+
+# Expose APP_PORT of the container to the outside
+EXPOSE $APP_PORT
+
+# Run the command to start things...
+CMD ["python", "app/main.py"]

app/database/clickhouse/process_tests.py

@@ -6,6 +6,7 @@
 
 """
 from .connect import client as clickhouse_client
+import traceback
 import json
 import html
 from datetime import datetime
@@ -82,8 +83,12 @@ def insert_axe_into_clickhouse(data):
             try:
                 client.execute(query)
             except Exception as e:
+            # Log the relevant parts of the exception
+                exception_traceback = traceback.format_exc()
                 logger.error(f'Failed to insert data into ClickHouse. HTML being processed: {html}')
-                logger.exception("Exception: ")
+                logger.error(f'Failed Query:\n{query}')
+                logger.error(f'Exception: {str(e)}')
+                logger.debug(f'Exception Traceback:\n{exception_traceback}')
 
 
     # close the client connection

app/database/postgres/fetch_unprocessed.py

@@ -10,14 +10,15 @@
 session = SessionLocal()
 
 
-def fetch_unprocessed_rules(limit=10000):
+def fetch_unprocessed_rules(limit=1000):
     """Fetches all rule_id that are not processed yet."""
     result = session.execute(text("""
         SELECT id as rule_id
         FROM axe.rules
         WHERE imported = false
         LIMIT :limit
      """), {'limit': limit})
+    logger.info(f'Importing {limit} unprocessed rules from Postgres')
 
     # Fetch all records from the query execution result
     records = result.fetchall()

app/database/postgres/process_tests.py

@@ -79,5 +79,6 @@ def mark_rule_as_processed(rule_id):
     session.execute(text(query), {'rule_id': rule_id})
     session.commit()
 
+
 if __name__ == "__main__":
     main()

app/main.py

@@ -5,8 +5,11 @@
 
 """
 import time
-from .processes import execute_axes
-from .database import fetch_unprocessed_rules, mark_axe_rule_as_processed
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from app.processes import execute_axes
+from app.database import fetch_unprocessed_rules, mark_axe_rule_as_processed
 
 # rule_id = 15
 # 1363