feat: Python MPL (#805)

.github/actions/polymorph_codegen/action.yml

@@ -82,6 +82,12 @@ runs:
       run: |
         make polymorph_dotnet
 
+    - name: Regenerate Python code using smithy-dafny
+      working-directory: ./${{ inputs.library }}
+      shell: bash
+      run: |
+        make polymorph_python
+
     - name: Check regenerated code against commited code
       # Composite action inputs seem to not actually support booleans properly for some reason
       if: inputs.diff-generated-code == 'true'

.github/workflows/daily_ci.yml

@@ -43,6 +43,12 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  daily-ci-python:
+    needs: getVersion
+    if: github.event_name != 'schedule' || github.repository_owner == 'aws'
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
   daily-interop-test:
     needs: getVersion
     if: github.event_name != 'schedule' || github.repository_owner == 'aws'

.github/workflows/library_codegen.yml

@@ -25,6 +25,10 @@ jobs:
         # Note dotnet is only used for formatting generated code
         # in this workflow
         dotnet-version: ["6.0.x"]
+        # Note python is only used for formatting generated code
+        # in this workflow.
+        # The workflow installs black and docformatter and runs them on generated code.
+        python-version: [3.11]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     defaults:
@@ -66,6 +70,16 @@ jobs:
       - name: Install Smithy-Dafny codegen dependencies
         uses: ./.github/actions/install_smithy_dafny_codegen_dependencies
 
+      - name: Setup Python, black, and docformatter for code formatting
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade black
+          python -m pip install --upgrade docformatter
+
       - uses: ./.github/actions/polymorph_codegen
         with:
           dafny: ${{ env.DAFNY_VERSION }}

.github/workflows/library_interop_tests.yml

@@ -25,7 +25,7 @@ jobs:
             ubuntu-latest,
             macos-12,
           ]
-        language: [java, net]
+        language: [java, net, python]
         # https://taskei.amazon.dev/tasks/CrypTool-5284
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
@@ -57,6 +57,16 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Dafny
         uses: dafny-lang/setup-dafny-action@v1.6.1
         with:
@@ -89,6 +99,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python
+
       - name: Setup gradle
         if: matrix.language == 'java'
         uses: gradle/gradle-build-action@v2
@@ -128,8 +147,8 @@ jobs:
             ubuntu-latest,
             macos-12,
           ]
-        encrypting_language: [java, net]
-        decrypting_language: [java, net]
+        encrypting_language: [java, net, python]
+        decrypting_language: [java, net, python]
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
     permissions:
@@ -167,6 +186,16 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Dafny
         uses: dafny-lang/setup-dafny-action@v1.6.1
         with:
@@ -199,6 +228,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.decrypting_language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python
+
       - name: Download Encrypt Manifest Artifact
         uses: actions/download-artifact@v4
         with:

.github/workflows/library_python_tests.yml

@@ -0,0 +1,92 @@
+# This workflow performs tests in Python.
+name: Library Python tests
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+      regenerate-code:
+        description: "Regenerate code using smithy-dafny"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  testPython:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          [
+            StandardLibrary,
+            AwsCryptographyPrimitives,
+            ComAmazonawsKms,
+            ComAmazonawsDynamodb,
+            AwsCryptographicMaterialProviders,
+            TestVectorsAwsCryptographicMaterialProviders,
+          ]
+        python-version: ["3.11"]
+        os: [
+            # TODO fix Dafny-generated tests on Windows;
+            # the sys.path workaround for generated Dafny doesn't work on Windows.
+            # Note: only tests use the sys.path workaround, not source code.
+            # Windows source code is tested downstream (ex. ESDK-Python CI).
+            # windows-latest,
+            ubuntu-latest,
+            macos-12,
+          ]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Support longpaths on Git checkout
+        run: |
+          git config --global core.longpaths true
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-MPL-Dafny-Role-us-west-2
+          role-session-name: PythonTests
+
+      - uses: actions/checkout@v4
+      # The specification submodule is private so we don't have access, but we don't need
+      # it to verify the Dafny code. Instead we manually pull the submodules we DO need.
+      - run: git submodule update --init libraries
+      - run: git submodule update --init smithy-dafny
+
+      - name: Setup Dafny
+        uses: dafny-lang/setup-dafny-action@v1.7.0
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Setup Python ${{ matrix.python-version }} for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Build ${{ matrix.library }} implementation
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
+      - name: Test ${{ matrix.library }}
+        working-directory: ./${{ matrix.library }}
+        shell: bash
+        run: |
+          make test_python

.github/workflows/manual.yml

@@ -37,6 +37,11 @@ jobs:
     with:
       dafny: ${{ inputs.dafny }}
       regenerate-code: ${{ inputs.regenerate-code }}
+  manual-ci-python:
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
+      regenerate-code: ${{ inputs.regenerate-code }}
   manual-interop-test:
     uses: ./.github/workflows/library_interop_tests.yml
     with:

.github/workflows/nightly_dafny.yml

@@ -37,6 +37,11 @@ jobs:
     with:
       dafny: "nightly-latest"
       regenerate-code: true
+  dafny-nightly-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
 
   cut-issue-on-failure:
     runs-on: ubuntu-latest

.github/workflows/pull.yml

@@ -34,6 +34,11 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  pr-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
   pr-interop-test:
     needs: getVersion
     uses: ./.github/workflows/library_interop_tests.yml

.github/workflows/push.yml

@@ -36,6 +36,11 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  push-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
   pr-interop-test:
     needs: getVersion
     uses: ./.github/workflows/library_interop_tests.yml

.gitignore

@@ -7,6 +7,11 @@ test/**/Output/*
 /package-lock.json
 /node_modules
 
+# Python docs build Artifacts
+build
+doc/build
+AwsCryptographicMaterialProviders/runtimes/python/doc/generated/*
+
 # Duvet output
 specification_compliance_report.html