Threat detection

.github/workflows/prod.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       release_version:
-        required: true  
+        required: true
       dashboard:
         type: boolean
         default: true
@@ -23,11 +23,16 @@ on:
         type: boolean
         default: true
         description: Internal
-      protection:
+      threat_detection:
         type: boolean
         default: true
-        description: Protection        
-
+        description: Threat Detection Client
+
+      threat_detection_backend:
+        type: boolean
+        default: true
+        description: Threat Deteection Backend
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # This workflow contains a single job called "build"
@@ -37,16 +42,21 @@ jobs:
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v2
         with:
-          distribution: 'adopt'
-          java-version: '8'
+          distribution: "adopt"
+          java-version: "8"
           architecture: x64
       - uses: actions/setup-node@v2
         with:
-          node-version: '17'
+          node-version: "17"
+      - uses: bufbuild/buf-action@v1
+        with:
+          setup_only: true
+      - name: Generate Proto files
+        run: make proto-gen
       - name: Download Akto templates zip and PII files
         working-directory: ./apps/dashboard/src/main/resources
         run: |
@@ -74,7 +84,7 @@ jobs:
           DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
           DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
         run: |
-          docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD 
+          docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
 
       - name: Build, tag, and push the image to DockerHub
         id: build-image-dockerhub
@@ -92,36 +102,40 @@ jobs:
             docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG_1 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG_2 . --push
             echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG"   
           fi
-          
+
           if [[ "${{ github.event.inputs.testing }}" == "true" ]]; then
             cd apps/testing
             docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG_2 . --push
             echo "::set-output name=image::$ECR_REGISTRY/akto-api-testing:$IMAGE_TAG"       
           fi
-          
+
           if [[ "${{ github.event.inputs.testing }}" == "true" ]]; then
             cd apps/testing-cli
             docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG_2 . --push
             echo "::set-output name=image::$ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG"
           fi
-          
+
           if [[ "${{ github.event.inputs.billing }}" == "true" ]]; then
             cd apps/billing
             docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG_2 . --push
             echo "::set-output name=image::$ECR_REGISTRY/akto-billing:$IMAGE_TAG"
           fi
-          
+
           if [[ "${{ github.event.inputs.internal }}" == "true" ]]; then
             cd apps/internal
             docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG_2 . --push
             echo "::set-output name=image::$ECR_REGISTRY/akto-internal:$IMAGE_TAG"
           fi
-
-          if [[ "${{ github.event.inputs.protection }}" == "true" ]]; then
-            cd apps/api-threat-detection
-            docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_2 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_3 . --push
-            echo "::set-output name=image::$ECR_REGISTRY/akto-api-protection:$IMAGE_TAG"
-          fi  
+
+          if [[ "${{ github.event.inputs.threat_detection}}" == "true" ]]; then
+            cd apps/threat-detection
+            docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-threat-detection:$IMAGE_TAG . --push
+          fi
+
+          if [[ "${{ github.event.inputs.threat_detection_backend}}" == "true" ]]; then
+            cd ../threat-detection-backend
+            docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-threat-detection-backend:$IMAGE_TAG . --push
+          fi
       - name: Configure AWS Credentials for ECR
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -132,23 +146,9 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
         with:
-          mask-password: 'true'
+          mask-password: "true"
           registry-type: public
 
-      - name: Build, tag, and push docker image to Amazon ECR
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: akto-api-security
-          REGISTRY_ALIAS: p7q3h0z2
-          IMAGE_TAG: local
-          IMAGE_TAG2: latest
-        run: |
-          docker buildx create --use
-          # Build a docker container and push it to DockerHub 
-          cd apps/api-threat-detection
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$REGISTRY_ALIAS/akto-api-protection:$IMAGE_TAG -t $ECR_REGISTRY/$REGISTRY_ALIAS/akto-api-protection:$IMAGE_TAG2 . --push
-          echo "::set-output name=image::$ECR_REGISTRY/akto-api-protection:$IMAGE_TAG"
-
       - name: Push git tag
         id: tag_version
         uses: mathieudutour/github-tag-action@v6.1

.github/workflows/staging.yml

@@ -26,6 +26,11 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: "17"
+      - uses: bufbuild/buf-action@v1
+        with:
+          setup_only: true
+      - name: Generate Proto files
+        run: make proto-gen
       - name: Convert github branch name to be compatible with docker tag name convention and generate tag name
         id: docker_tag
         run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT
@@ -36,20 +41,15 @@ jobs:
           wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json
           wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json
           wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json
-      - name: Prepare Dashboard polaris UI
-        working-directory: ./apps/dashboard/web/polaris_web
-        run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build
+
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}}
           aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}}
           aws-region: ap-south-1
 
-      - name: Deploy polaris site to S3 bucket
-        run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete
-
-      - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}}
+      - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} -DskipTests
       - name: DockerHub login
         env:
           DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
@@ -79,10 +79,12 @@ jobs:
           docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push
           cd ../internal
           docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push
-          cd ../api-threat-detection
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG . --push
           cd ../source-code-analyser
           docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/source-code-analyser:$IMAGE_TAG . --push
+          cd ../threat-detection
+          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-threat-detection:$IMAGE_TAG . --push
+          cd ../threat-detection-backend
+          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-threat-detection-backend:$IMAGE_TAG . --push
 
       - name: Set up JDK 11
         uses: actions/setup-java@v1

.gitignore

@@ -22,3 +22,4 @@ https:
 **/data-zoo-data
 **/data-zoo-logs
 **/bin
+.factorypath

Makefile

@@ -0,0 +1,8 @@
+proto-gen:
+	sh ./scripts/proto-gen.sh
+
+build: proto-gen
+	mvn install -DskipTests
+
+build-clean: proto-gen
+	mvn clean install -DskipTests