Fix/test data download (#132)

* Changed the git actions to get the files from gdrive * Fixed typo * Also removed charset from requirements * Fix windows test action * Fixed unzip on windows test * Fixed unzip on windows test * Fixed unzip on windows test * Changed windows to wget * Changed windows to wget * Changed windows to wget
OceanStreamIO · Dec 5, 2023 · 3ecd1ca · 3ecd1ca
1 parent 3aeb05e
commit 3ecd1ca
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 1 deletion.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -6,6 +6,7 @@ on:
 
 env:
   NUM_WORKERS: 2
+  TEST_DATA_FOLDER: ${{ github.workspace }}/test_data
 
 jobs:
   test:
@@ -62,6 +63,11 @@ jobs:
 
           # Check data endpoint
           curl http://localhost:8080/data/
+      # Add the following steps for downloading and unzipping test data
+      - name: Download Test Data from Google Drive
+        run: |
+          wget --load-cookies /tmp/cookies.txt "https://drive.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://drive.google.com/uc?export=download&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8" -O test_data.zip && rm -rf /tmp/cookies.txt
+          unzip -n test_data.zip -d ${{ env.TEST_DATA_FOLDER }}
       - name: Finding changed files
         id: files
         uses: Ana06/get-changed-files@v2.2.0

diff --git a/.github/workflows/windows-utils.yaml b/.github/workflows/windows-utils.yaml
@@ -11,6 +11,7 @@ on:
 
 env:
   CONDA_ENV: echopype
+  TEST_DATA_FOLDER: ${{ github.workspace }}\\test_data
 
 jobs:
   windows-test:
@@ -64,6 +65,32 @@ jobs:
         - name: Install echopype
           run: |
             python -m pip install -e .
+        # Add steps for downloading and unzipping test data
+        - name: Create Test Data Directory
+          run: New-Item -ItemType Directory -Force -Path ${{ env.TEST_DATA_FOLDER }}
+        - name: Install wget
+          run: choco install wget
+          shell: powershell
+        #- name: Download Test Data from Google Drive
+        #  run: |
+        #    $downloadLink = "https://drive.google.com/uc?export=download&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8"
+        #    $downloadPage = Invoke-WebRequest -Uri $downloadLink -OutFile "download.html"
+        #    $confirmCode = (Select-String -Path "download.html" -Pattern 'confirm=([0-9A-Za-z_]+)' -AllMatches).Matches.Groups[1].Value
+        #    $downloadFileLink = "https://drive.google.com/uc?export=download&confirm=$confirmCode&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8"
+        #    Invoke-WebRequest -Uri $downloadFileLink -OutFile "test_data.zip"
+        - name: Download Test Data from Google Drive
+          run: |
+            Remove-Item alias:wget
+            wget --quiet --save-cookies cookies.txt --keep-session-cookies --no-check-certificate 'https://drive.google.com/uc?export=download&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8' -O- | Out-String | Select-String -Pattern 'confirm=([0-9A-Za-z_]+)' | %{ $_.Matches.Groups[1].Value } > confirm.txt
+            $confirmCode = Get-Content confirm.txt
+            wget --load-cookies cookies.txt "https://drive.google.com/uc?export=download&confirm=$confirmCode&id=1ofiSQ4zDwXfHE65tow4_jDIceBYHNW_8" -O test_data.zip
+            Remove-Item cookies.txt -Force
+            Remove-Item confirm.txt -Force
+          shell: powershell
+
+        - name: Unzip Test Data
+          run: Expand-Archive -LiteralPath "test_data.zip" -DestinationPath ${{ env.TEST_DATA_FOLDER }} -Force
+
         - name: Running all Tests
           run: |
             pytest -vvv -rx --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings echopype/tests/utils |& tee ci_${{ matrix.python-version }}_test_log.log
diff --git a/requirements.txt b/requirements.txt
@@ -17,5 +17,4 @@ xarray-datatree==0.0.6
 psutil>=5.9.1
 geopy
 flox>=0.7.2,<1.0.0
-charset-normalizer<3.2
 dask-image