ci: Add integration tests (#33)

* ci: Add integration tests * Add missing data file * Reference correct file * Ensure bucket exists * Run minio in the backgroud * Rename release workflow * Fix line break in docker command * Checkout code before runnin minio * Add s3 region
MeltanoLabs · Feb 24, 2023 · 2d23e04 · 2d23e04
1 parent 2041db1
commit 2d23e04
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 79 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,35 +1,36 @@
-on: {}
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+
 jobs:
-  releases-matrix:
-    name: Release Go Binaries
+  goreleaser:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-        - goos: linux
-          goarch: "386"
-        - goos: linux
-          goarch: "amd64"
-        - goos: linux
-          goarch: "arm64"
-        - goos: darwin
-          goarch: "amd64"
-        - goos: darwin
-          goarch: "arm64"
-        - goos: windows
-          goarch: "386"
-        - goos: windows
-          goarch: "amd64"
     steps:
-    - name: Checkout code
+    - name: Checkout
       uses: actions/checkout@v3
-    - name: Release binaries
-      uses: wangyoucao577/go-release-action@v1.36
       with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        goos: ${{ matrix.goos }}
-        goarch: ${{ matrix.goarch }}
-        goversion: "1.20"
-        project_path: ./
-        binary_name: target-jsonl-blob
-        compress_assets: false
+        fetch-depth: 0
+
+    - run: git fetch --force --tags
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
+        cache: true
+
+    - name: Run GoReleaser
+      uses: goreleaser/goreleaser-action@v4
+      with:
+        distribution: goreleaser
+        version: latest
+        args: release --clean --skip-validate --debug
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/releasev2.yml b/.github/workflows/releasev2.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,8 +1,14 @@
 name: Test
+
 on:
   pull_request: {}
   push:
     branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     strategy:
@@ -11,13 +17,67 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-        cache: true
-        cache-dependency-path: go.sum
-    - name: Test
-      run: go test ./... -cover
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache: true
+          cache-dependency-path: go.sum
+      - name: Test
+        run: go test ./... -cover
+
+  integration-test:
+    name: Integration test - ${{ matrix.fs }}
+    strategy:
+      matrix:
+        include:
+          - fs: local
+            config:
+              bucket: file://./output/my-bucket
+          - fs: s3
+            config:
+              bucket: s3://my-bucket?endpoint=http://127.0.0.1:9000&disableSSL=true&s3ForcePathStyle=true&region=us-west-1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Start MinIO
+        if: ${{ matrix.fs == 's3' }}
+        run: >
+          docker run -d
+          --name minio
+          -p 9000:9000
+          -p 9090:9090
+          -v $PWD/output:/data
+          -e "MINIO_REGION_NAME=eu-west-1"
+          -e "MINIO_ROOT_USER=minioadmin"
+          -e "MINIO_ROOT_PASSWORD=minioadmin"
+          quay.io/minio/minio server /data --console-address ":9090"
+
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Build
+        run: |
+          go build
+          ./target-jsonl-blob --version
+
+      - name: Write config
+        run: |
+          echo '${{ toJSON(matrix.config) }}' > config.json
+
+      - run: mkdir output/my-bucket
+
+      - name: Test
+        env:
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin
+        run: |
+          ./target-jsonl-blob --config config.json --input resources/data/data.jsonl
diff --git a/.gitignore b/.gitignore
@@ -2,9 +2,6 @@
 *.json
 !*.example.*
 
-# Data files
-**/*.jsonl
-
 # Binaries for programs and plugins
 target-jsonl-blob
 *.exe

diff --git a/resources/data/data.jsonl b/resources/data/data.jsonl
@@ -0,0 +1,13 @@
+{"type": "BAD"}
+{"type": "SCHEMA", "stream": "test", "schema": {"properties": {"id": "integer"}}}
+{"type": "RECORD", "stream": "test", "record": {"id": 1}, "time_extracted": "2022-01-01T00:00:00Z"}
+{"type": "RECORD", "stream": "test", "record": {"id": 2}, "time_extracted": "2022-01-01T00:00:00Z"}
+{"type": "STATE", "value": {"bookmark": "2022-01-01"}}
+{"type": "SCHEMA", "stream": "test2", "schema": {"properties": {"name": "string"}}}
+{"type": "RECORD", "stream": "test2", "record": {"name": "A"}}
+{"type": "RECORD", "stream": "test2", "record": {"name": "B"}}
+{"type": "STATE", "value": {"test": "2022-01-01", "test2": "2022-02-01"}}
+{"type": "SCHEMA", "stream": "test", "schema": {"properties": {"id": "integer"}}}
+{"type": "RECORD", "stream": "test", "record": {"id": 3}}
+{"type": "RECORD", "stream": "test", "record": {"id": 4}}
+{"type": "STATE", "value": {"test": "2022-03-01", "test2": "2022-02-01"}}