Project upgrade to 2.0.0 with k8s support

.gitlab-ci.yml

@@ -0,0 +1,267 @@
+stages:
+  - test
+  - build
+  - scan
+  - push
+  - deploy
+
+variables:
+  CONTAINER_REGISTRY: "registry.digitalocean.com/htw"
+
+.build_template: &build_definition
+  stage: build
+  tags:
+    - ubuntu-runner
+  script:
+    - cd $CI_PROJECT_DIR/$SERVICE_PATH
+    - docker build -t $SERVICE_NAME:$CI_PIPELINE_ID .
+
+test_backend:
+  stage: test
+  variables:
+    SERVICE_PATH: "./backend"
+    RUNNING_TESTS: "true"
+  tags:
+    - ubuntu-runner
+  script:
+    - export RUNNING_TESTS=${RUNNING_TESTS}
+    - python3.10 -m pip --version || (wget https://bootstrap.pypa.io/get-pip.py && python3.10 get-pip.py --user)
+    - export PATH=$PATH:~/.local/bin
+    - cd $SERVICE_PATH
+    - pip install -r requirements.txt
+    - pytest
+
+test_frontend:
+  stage: test
+  variables:
+    SERVICE_PATH: "./frontend"
+  tags:
+    - ubuntu-runner
+  script:
+    - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
+    - source ~/.nvm/nvm.sh
+    - nvm install 18
+    - nvm use 18
+    - cd $SERVICE_PATH
+    - npm cache clean --force
+    - npm ci
+    - npm test
+
+.scan_template: &scan_definition
+  stage: scan
+  tags:
+    - ubuntu-runner
+  script:
+    - trivy image --severity HIGH,CRITICAL --no-progress $SERVICE_NAME:$CI_PIPELINE_ID
+
+.push_template: &push_definition
+  stage: push
+  tags:
+    - ubuntu-runner
+  script:
+    - docker tag $SERVICE_NAME:$CI_PIPELINE_ID $CONTAINER_REGISTRY/$SERVICE_NAME:$CI_PIPELINE_ID
+    - docker push $CONTAINER_REGISTRY/$SERVICE_NAME:$CI_PIPELINE_ID
+
+.deploy_template: &deploy_definition
+  stage: deploy
+  tags:
+    - ubuntu-runner
+  script:
+    - export KUBECONFIG=$KUBECONFIG_DATA
+    - kubectl config set-context --current --namespace=example-namespace-name
+    - cd $CI_PROJECT_DIR/$SERVICE_PATH/k8s
+    - if [ -f config.yml ]; then kubectl apply -f config.yml; fi
+    - if [ -f pv-pvc.yml ]; then kubectl apply -f pv-pvc.yml; fi
+    - if [ -f ingress.yml ]; then kubectl apply -f ingress.yml; fi
+    - sed -i "s|registry/app:tag|$CONTAINER_REGISTRY/$SERVICE_NAME:$CI_PIPELINE_ID|g" deployment.yml
+    - kubectl apply -f deployment.yml
+    - kubectl apply -f service.yml
+
+# Frontend Service
+
+build_frontend:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-frontend"
+    SERVICE_PATH: "./frontend"
+
+scan_frontend:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-frontend"
+
+push_frontend:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-frontend"
+
+deploy_frontend:
+  <<: *deploy_definition
+  variables:
+    SERVICE_NAME: "ee-frontend"
+    SERVICE_PATH: "./frontend"
+
+# Backend Service
+
+build_backend:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-backend"
+    SERVICE_PATH: "./backend"
+
+scan_backend:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-backend"
+
+push_backend:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-backend"
+
+deploy_backend:
+  stage: deploy
+  tags:
+    - ubuntu-runner
+  variables:
+    SERVICE_NAME: "ee-backend"
+    SERVICE_PATH: "./backend"
+  script:
+    # Custom script steps for the backend deployment
+    - export KUBECONFIG=$KUBECONFIG_DATA
+    - kubectl config set-context --current --namespace=example-namespace-name
+    - |
+      if kubectl get secret ee-backend-secrets &> /dev/null; then
+        kubectl delete secret ee-backend-secrets
+      fi
+      kubectl create secret generic ee-backend-secrets \
+        --from-literal=OPENAI_API_KEY=$OPENAI_API_KEY \
+        --from-literal=POCKETBASE_ADMIN_EMAIL=$POCKETBASE_ADMIN_EMAIL \
+        --from-literal=POCKETBASE_ADMIN_PASSWORD=$POCKETBASE_ADMIN_PASSWORD
+
+    # Deploy definition steps
+    - cd $CI_PROJECT_DIR/$SERVICE_PATH/k8s
+    - if [ -f config.yml ]; then kubectl apply -f config.yml; fi
+    - if [ -f pv-pvc.yml ]; then kubectl apply -f pv-pvc.yml; fi
+    - if [ -f ingress.yml ]; then kubectl apply -f ingress.yml; fi
+    - sed -i "s|registry/app:tag|$CONTAINER_REGISTRY/$SERVICE_NAME:$CI_PIPELINE_ID|g" deployment.yml
+    - kubectl apply -f deployment.yml
+    - kubectl apply -f service.yml
+
+# Qdrant Service
+
+build_qdrant:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-qdrant"
+    SERVICE_PATH: "./qdrant"
+
+scan_qdrant:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-qdrant"
+
+push_qdrant:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-qdrant"
+
+deploy_qdrant:
+  <<: *deploy_definition
+  variables:
+    SERVICE_NAME: "ee-qdrant"
+    SERVICE_PATH: "./qdrant"
+
+# Pocketbase Service
+
+build_pocketbase:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-pocketbase"
+    SERVICE_PATH: "./pocketbase"
+
+scan_pocketbase:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-pocketbase"
+
+push_pocketbase:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-pocketbase"
+
+deploy_pocketbase:
+  stage: deploy
+  tags:
+    - ubuntu-runner
+  variables:
+    SERVICE_NAME: "ee-pocketbase"
+    SERVICE_PATH: "./pocketbase"
+  script:
+    # Custom script steps for the pocketbase deployment
+    - export KUBECONFIG=$KUBECONFIG_DATA
+    - kubectl config set-context --current --namespace=example-namespace-name
+    - |
+      if kubectl get secret ee-pocketbase-secrets &> /dev/null; then
+        kubectl delete secret ee-pocketbase-secrets
+      fi
+      kubectl create secret generic ee-pocketbase-secrets \
+        --from-literal=POCKETBASE_ADMIN_EMAIL=$POCKETBASE_ADMIN_EMAIL \
+        --from-literal=POCKETBASE_ADMIN_PASSWORD=$POCKETBASE_ADMIN_PASSWORD
+
+    # Deploy definition steps
+    - cd $CI_PROJECT_DIR/$SERVICE_PATH/k8s
+    - if [ -f config.yml ]; then kubectl apply -f config.yml; fi
+    - if [ -f pv-pvc.yml ]; then kubectl apply -f pv-pvc.yml; fi
+    - if [ -f ingress.yml ]; then kubectl apply -f ingress.yml; fi
+    - sed -i "s|registry/app:tag|$CONTAINER_REGISTRY/$SERVICE_NAME:$CI_PIPELINE_ID|g" deployment.yml
+    - kubectl apply -f deployment.yml
+    - kubectl apply -f service.yml
+
+# Prometheus Service
+
+build_prometheus:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-prometheus"
+    SERVICE_PATH: "./prometheus"
+
+scan_prometheus:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-prometheus"
+
+push_prometheus:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-prometheus"
+
+deploy_prometheus:
+  <<: *deploy_definition
+  variables:
+    SERVICE_NAME: "ee-prometheus"
+    SERVICE_PATH: "./prometheus"
+
+# Grafana Service
+
+build_grafana:
+  <<: *build_definition
+  variables:
+    SERVICE_NAME: "ee-grafana"
+    SERVICE_PATH: "./grafana"
+
+scan_grafana:
+  <<: *scan_definition
+  variables:
+    SERVICE_NAME: "ee-grafana"
+
+push_grafana:
+  <<: *push_definition
+  variables:
+    SERVICE_NAME: "ee-grafana"
+
+deploy_grafana:
+  <<: *deploy_definition
+  variables:
+    SERVICE_NAME: "ee-grafana"
+    SERVICE_PATH: "./grafana"

README.md

@@ -4,10 +4,20 @@
 
 This is the Repository for the unified services for the Epistle Engine with Github actions and Docker support.
 
+## Important!
+
+You should be using the Chrome/Edge Browser for testing and using the application.
+
+Because in the frontend env file: PUBLIC_SECURE=true to test it. Only if you are need to test it on safari change it to false.
+
+Also On the Cluster it is set to TRUE. So you can only test it with Chrome/Edge.
+
+for init-pocketbase.sh to run, you need the file to be in LF line endings, so please do not edit the file or open it. If you have the error: init-pocketbase.sh not found, edit the file to have LF line endings and not CRLF!
+
 ## Status
 
-[![Test Backend](https://github.com/kamyabnazari/epistle-engine/actions/workflows/test-backend.yml/badge.svg)](https://github.com/kamyabnazari/epistle-engine/actions/workflows/test-backend.yml)
-[![Test Frontend](https://github.com/kamyabnazari/epistle-engine/actions/workflows/test-frontend.yml/badge.svg)](https://github.com/kamyabnazari/epistle-engine/actions/workflows/test-frontend.yml)
+Test Backend
+Test Frontend
 
 ## About the Project
 
@@ -65,6 +75,14 @@ Branch Example: `kn-feature-name-example`
 
 - [Qdrant](https://qdrant.io/) [Vector Search Engine]
 
+### Prometheus
+
+- [Prometheus](https://prometheus.io/) [Obeservation Tool]
+
+### Grafana
+
+- [Grafana](https://grafana.com/) [Dashboard Tool]
+
 ## Getting Started
 
 Install all the prerequisites and follow the instructions in the README.md files of the services.
@@ -90,6 +108,14 @@ These all also have to be installed on the system in order to generate PDFs.
 
 Please create the services after another in order and follow the instructions in the README.md files.
 
+When you run them individually you have to use localhost to access and connect them to each other.
+
+Or Use Docker Compose to create all of it and connect them together.
+
+```bash
+docker-compose up --build
+```
+
 Please setup in the following order:
 
 ### 1. Setup [Pocketbase](pocketbase/README.md)
@@ -99,3 +125,7 @@ Please setup in the following order:
 ### 3. Setup [Backend](backend/README.md)
 
 ### 4. Setup [Frontend](frontend/README.md)
+
+### 5. Setup [Prometheus](prometheus/README.md)
+
+### 6. Setup [Grafana](grafana/README.md)

backend/Dockerfile

@@ -1,20 +1,31 @@
 # Dockerfile for the backend service testing
-FROM python:3
+FROM python:3.10
+
+# Set environment variable
+ENV XDG_RUNTIME_DIR=/tmp/runtime-root
 
 # Install system dependencies
 RUN apt-get update && \
-    apt-get install -y libcairo2-dev pkg-config texlive-base texlive-latex-recommended wkhtmltopdf && \
+    apt-get install -y \
+    libcairo2-dev \
+    pkg-config \
+    texlive-base \
+    texlive-latex-recommended \
+    wkhtmltopdf && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Set the working directory inside the container
 WORKDIR /app
+
+# Copy the local src files to the container
 ADD . /app
 
 # Install python dependencies
 RUN pip install -r requirements.txt --no-cache-dir
 
-# Make sure our start script is executable
-RUN chmod +x /app/start.sh
-
+# Expose port for the application
 EXPOSE 5003
-CMD ["./start.sh"]
+
+# Command to run the application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5003", "--timeout-keep-alive", "360"]

backend/README.md

@@ -10,18 +10,19 @@ Create a `.env` file in the root of the backend directory with the following var
 OPENAI_API_KEY={your openai api key}
 POCKETBASE_ADMIN_EMAIL={your pocketbase admin email}
 POCKETBASE_ADMIN_PASSWORD={your pocketbase admin password}
-PUBLIC_POCKETBASE_URL=http://localhost:8090
-PUBLIC_FRONTEND_URL=http://localhost:5173
-PUBLIC_QDRANT_URL=http://localhost:6333
-QDRANT__SERVICE_API_KEY={Only for Deployed Qdrant}
+PUBLIC_POCKETBASE_URL=http://ee-pocketbase:8090
+PUBLIC_FRONTEND_URL=http://ee-frontend:3000
+PUBLIC_QDRANT_URL=http://ee-qdrant:6333
+PUBLIC_PROMETHEUS_URL=http://ee-prometheus:9090
+RUNNING_TESTS=false // if you want to run tests set to true
 ```
 
 ### Running the backend
 
 To install the required packages for this plugin and run the service locally, run the following commands:
 
-```bash
-pip install --upgrade -r requirements.txt
+```
+pip install -r requirements.txt
 
 uvicorn main:app --reload --host 0.0.0.0 --port 5003 --timeout-keep-alive 360
 ```