CI/CD - ECR/ECS Python Deployment Test

Signed-off-by: David Deal <ddeal@linuxfoundation.org>
communitybridge · Aug 31, 2023 · 789e2eb · 789e2eb
1 parent fc31b43
commit 789e2eb
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 19 deletions.
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
@@ -10,8 +10,11 @@ on:
 
 env:
   AWS_REGION: us-east-1
+  AWS_ECR_REGION: us-east-1
   AWS_PROFILE: lf-cla
   STAGE: dev
+  REPOSITORY: lfx-easycla-dev
+  ECR_HOST: ${{secrets.AWS_ACCOUNT_ID}}.dkr.ecr.us-east-1.amazonaws.com
 
 jobs:
   build-deploy-dev:
@@ -25,6 +28,13 @@ jobs:
           go-version: '1.20.1'
       - name: Go Version
         run: go version
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Available Build Platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
       - name: Setup Node
         uses: actions/setup-node@v3
         with:
@@ -88,6 +98,7 @@ jobs:
       - name: Python Setup
         working-directory: cla-backend
         run: |
+          pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-test.txt
 
@@ -107,6 +118,27 @@ jobs:
           AUTH0_PLATFORM_CLIENT_SECRET: ${{ secrets.AUTH0_PLATFORM_CLIENT_SECRET }}
           AUTH0_PLATFORM_AUDIENCE: https://api-gw.dev.platform.linuxfoundation.org/
 
+      - name: Build Docker Image
+        run:  |
+          # Create a new builder, named container, that uses the Docker container driver
+          echo "Creating a new builder container..."
+          docker buildx create --name container --driver=docker-container
+          
+          echo "Building image with tag: ${ECR_HOST}/${REPOSITORY}:${{github.sha}}"
+          docker buildx build --platform=linux/arm64 --builder=container --tag ${ECR_HOST}/${REPOSITORY}:${{github.sha}} .
+          echo "Building image with tag: ${ECR_HOST}/${REPOSITORY}:latest"
+          docker buildx build --platform=linux/arm64 --builder=container --tag ${ECR_HOST}/${REPOSITORY}:latest .
+
+          # Note, unlike when using the default docker driver, images built with the docker-container driver must be explicitly loaded into the local image store. Use the --load flag
+          echo "Loading the image into the image store..."
+          docker buildx build --platform=linux/arm64 --builder=container --load --tag ${ECR_HOST}/${REPOSITORY}:${{github.sha}} .
+          docker buildx build --platform=linux/arm64 --builder=container --load --tag ${ECR_HOST}/${REPOSITORY}:latest .
+      - name: Publish Docker Image
+        run: |
+          aws ecr get-login-password --region ${AWS_ECR_REGION} | docker login --username AWS --password-stdin ${ECR_HOST}
+          docker push ${ECR_HOST}/${REPOSITORY}:${{github.sha}}
+          docker push ${ECR_HOST}/${REPOSITORY}:latest
+
       - name: Go Setup
         working-directory: cla-backend-go
         run: |

diff --git a/cla-backend/Dockerfile b/cla-backend/Dockerfile
@@ -0,0 +1,16 @@
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+FROM public.ecr.aws/lambda/python:3.7
+
+# Copy requirements.txt
+COPY requirements.txt ${LAMBDA_TASK_ROOT}
+
+# Copy function code
+COPY *.py cla ${LAMBDA_TASK_ROOT}
+
+# Install the specified packages
+RUN pip install -r requirements.txt
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "wsgi_handler.handler" ]
diff --git a/cla-backend/requirements.txt b/cla-backend/requirements.txt
@@ -43,7 +43,8 @@ pip-autoremove==0.10.0
 pipdeptree==2.9.6
 platformdirs==3.10.0
 pluggy==0.13.1
-psycopg2==2.9.7
+#psycopg2==2.9.7
+psycopg2-binary==2.9.7
 py==1.11.0
 pyasn1==0.4.8
 pycparser==2.21

diff --git a/cla-backend/serverless.yml b/cla-backend/serverless.yml
@@ -49,10 +49,6 @@ custom:
     number: 3
   userEventsSNSTopicARN: arn:aws:sns:us-east-2:${aws:accountId}:userservice-triggers-${sls:stage}-user-sns-topic
 
-  # For the python requirements add layer:true to put them into a Lambda Layer.
-  pythonRequirements:
-    layer: true
-
   certificate:
     arn:
       # From env Certificate Manager -
@@ -111,6 +107,10 @@ custom:
     dev: admin@dev.lfcla.com
     staging: admin@staging.lfcla.com
     prod: admin@lfx.linuxfoundation.org
+  ecr_img:
+    dev: 395594542180.dkr.ecr.us-east-1.amazonaws.com/lfx-easycla-${sls:stage}:latest
+    staging: 844390194980.dkr.ecr.us-east-1.amazonaws.com/lfx-easycla-${sls:stage}:latest
+    prod: 716487311010.dkr.ecr.us-east-1.amazonaws.com/lfx-easycla-${sls:stage}:latest
 
 provider:
   name: aws
@@ -121,6 +121,12 @@ provider:
   logRetentionInDays: 14
   lambdaHashingVersion: '20201221' # Resolution of lambda version hashes was improved with better algorithm, which will be used in next major release. Switch to it now by setting "provider.lambdaHashingVersion" to "20201221"
 
+  ecr:
+    # In this section you can define images that will be built locally and uploaded to ECR
+    images:
+      easyclaPythonAppImage:
+        uri: ${self:custom.ecr_img.${sls:stage}
+
   apiGateway:
     # https://www.serverless.com/framework/docs/deprecations/#AWS_API_GATEWAY_NAME_STARTING_WITH_SERVICE
     shouldStartNameWithService: true
@@ -392,7 +398,7 @@ provider:
     Owner: "David Deal"
 
 plugins:
-  - serverless-python-requirements
+  #- serverless-python-requirements
   - serverless-wsgi
   - serverless-plugin-tracing
   # Serverless Finch does s3 uploading. Called with 'sls client deploy'.
@@ -596,8 +602,12 @@ functions:
   apiv1:
     handler: wsgi_handler.handler
     description: "EasyCLA Python API handler for the /v1 endpoints"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'wsgi_handler.handler'
+      #entryPoint:
+      #  - '/lambda-entrypoint.sh'
     events:
       - http:
           method: ANY
@@ -607,8 +617,12 @@ functions:
   apiv2:
     handler: wsgi_handler.handler
     description: "EasyCLA Python API handler for the /v2 endpoints"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'wsgi_handler.handler'
+    #    layers:
+    #      - Ref: PythonRequirementsLambdaLayer
     events:
       - http:
           method: ANY
@@ -618,8 +632,12 @@ functions:
   salesforceprojects:
     handler: cla.salesforce.get_projects
     description: "EasyCLA API Callback Handler for fetching all SalesForce projects"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'cla.salesforce.get_projects'
+#    layers:
+#      - Ref: PythonRequirementsLambdaLayer
     events:
       - http:
           method: ANY
@@ -629,8 +647,12 @@ functions:
   salesforceprojectbyID:
     handler: cla.salesforce.get_project
     description: "EasyCLA API Callback Handler for fetching SalesForce projects by ID"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'cla.salesforce.get_project'
+#    layers:
+#      - Ref: PythonRequirementsLambdaLayer
     events:
       - http:
           method: ANY
@@ -641,8 +663,12 @@ functions:
   githubinstall:
     handler: wsgi_handler.handler
     description: "EasyCLA API Callback Handler for GitHub bot installations"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'wsgi_handler.handler'
+#    layers:
+#      - Ref: PythonRequirementsLambdaLayer
     events:
       - http:
           method: ANY
@@ -652,14 +678,17 @@ functions:
   githubactivity:
     handler: wsgi_handler.handler
     description: "EasyCLA API Callback Handler for GitHub activity"
-    layers:
-      - Ref: PythonRequirementsLambdaLayer
+    image:
+      name: easyclaPythonAppImage
+      command:
+        - 'wsgi_handler.handler'
+#    layers:
+#      - Ref: PythonRequirementsLambdaLayer
     events:
       - http:
           method: POST
           path: v2/github/activity
 
-
 resources:
   Conditions:
     # Helper functions since we conditionally create some resources