Download-upload cycle test (#291)

* Remove empty server-side tests

* Add download-upload cycle test

* Lint fix

* Separate cycle test and run every hour

* Lint fix

* Write file structure comparison method

* Fix api and test

* Lint fix

* Modify CI test triggers

* Change to once per day

* Remove redundant pull_request trigger

.github/workflows/download_upload_cycle.yml

@@ -0,0 +1,43 @@
+name: download_upload_cycle
+on:
+  push:
+  schedule:
+    - cron: "0 0 * * *"
+jobs:
+  client:
+    env:
+      DJANGO_SUPERUSER_PASSWORD: django-password
+      DJANGO_SUPERUSER_USERNAME: admin
+      DJANGO_SUPERUSER_EMAIL: admin@noemail.com
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install tox
+        run: |
+          pip install --upgrade pip
+          pip install tox
+
+      - name: Build images
+        run: docker-compose build
+      - name: Run migrations
+        run: docker-compose run --rm django ./manage.py migrate
+      - name: Create super user
+        run: docker-compose run -e DJANGO_SUPERUSER_PASSWORD=$DJANGO_SUPERUSER_PASSWORD --rm django ./manage.py createsuperuser --noinput --email=$DJANGO_SUPERUSER_EMAIL
+      - name: Start server
+        run: docker-compose up -d
+
+      - uses: ifaxity/wait-on-action@v1
+        with:
+          resource: 'http://localhost:8000/api/docs/swagger/'
+
+      - name: Run tox
+        run: tox -e download_upload_cycle
+        working-directory: ./swcc
+
+      - name: Stop server
+        run: docker-compose down --volumes
+        if: ${{ always() }}

.github/workflows/test.yml

@@ -1,6 +1,6 @@
 name: test
 on:
-  pull_request:
+  push:
   schedule:
     - cron: "0 0 * * *"
 jobs:

shapeworks_cloud/core/filters.py

@@ -107,6 +107,7 @@ class OptimizedParticlesFilter(FilterSet):
     class Meta:
         models = models.OptimizedParticles
         fields = [
+            'project',
             'shape_model',
             'groomed_segmentation',
             'groomed_mesh',

shapeworks_cloud/core/models.py

@@ -113,7 +113,7 @@ def create_new_file(self):
         )
 
     def get_download_paths(self):
-        ret = {self.file.name.split('/')[-1]: self.file.url}
+        ret = {}
         with self.file.open() as f:
             data = json.load(f)['data']
 

swcc/swcc/models/project.py

@@ -362,6 +362,7 @@ def create(self) -> Project:
 
     def download(self, folder: Union[Path, str]):
         session = current_session()
+        self.file.download(folder)
         r: requests.Response = session.get(f'{self._endpoint}/{self.id}/download/')
         raise_for_status(r)
         data = r.json()

swcc/tests/test_download_upload.py

@@ -0,0 +1,101 @@
+# This test requires that https://www.shapeworks-cloud.org/#/ is running
+# This will fetch all datasets populated on the public server and attempt to
+# upload them all to the local server
+import filecmp
+import os
+import random
+from tempfile import TemporaryDirectory
+
+from swcc import models
+from swcc.api import swcc_session
+
+SAMPLE_SIZE = 3
+
+
+def project_as_dict_repr(project):
+    project_repr = dict(project)
+    project_repr['dataset'] = dict(project_repr['dataset'])
+
+    # remove keys that are expected to differ between servers
+    del project_repr['id']
+    del project_repr['file']
+    del project_repr['dataset']['id']
+    del project_repr['dataset']['creator']
+    del project_repr['last_cached_analysis']
+    return project_repr
+
+
+class DirCmp(filecmp.dircmp):
+    def phase3(self):
+        fcomp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False)
+        self.same_files, self.diff_files, self.funny_files = fcomp
+
+
+def is_same(dir1, dir2):
+    compared = DirCmp(dir1, dir2)
+    different = (
+        compared.left_only or compared.right_only or compared.diff_files or compared.funny_files
+    )
+    if different:
+        return False
+    for subdir in compared.common_dirs:
+        if not is_same(os.path.join(dir1, subdir), os.path.join(dir2, subdir)):
+            return False
+    return True
+
+
+def public_server_download(download_dir):
+    with swcc_session() as public_server_session:
+        public_server_session.login('testuser@noemail.nil', 'cicdtest')
+        all_projects = list(models.Project.list())
+        project_subset = (
+            random.sample(all_projects, SAMPLE_SIZE)
+            if len(all_projects) >= SAMPLE_SIZE
+            else all_projects
+        )
+        for project in project_subset:
+            project.download(download_dir)
+        return project_subset
+
+
+def test_download_upload_cycle(session):
+    with TemporaryDirectory() as temp_dir:
+        # Download from public server
+        download_dir = f'{temp_dir}/download'
+        os.mkdir(download_dir)
+        all_projects = public_server_download(download_dir)
+
+        new_projects = []
+        # Upload to local server
+        for project in all_projects:
+            d = models.Dataset(
+                name=project.dataset.name,
+                description=project.dataset.description,
+                license=project.dataset.license,
+                acknowledgement=project.dataset.acknowledgement,
+                keywords=project.dataset.keywords,
+                contributors=project.dataset.contributors,
+                publications=project.dataset.publications,
+            ).force_create()
+            new_projects.append(
+                models.Project(
+                    dataset=d,
+                    file=project.file.path,
+                    description=project.description,
+                ).create()
+            )
+
+        # Redownload from local server
+        redownload_dir = f'{temp_dir}/redownload'
+        os.mkdir(redownload_dir)
+        for project in new_projects:
+            project.download(redownload_dir)
+
+        # Test for file structure congruence
+        assert is_same(download_dir, redownload_dir)
+
+    for local_project, remote_project in zip(models.Project.list(), all_projects):
+        local_repr = project_as_dict_repr(local_project)
+        remote_repr = project_as_dict_repr(remote_project)
+        assert local_repr == remote_repr
+    assert len(list(models.Project.list())) == len(all_projects)

swcc/tox.ini

@@ -53,7 +53,20 @@ deps =
     pytest-factoryboy
     pytest-mock
 commands =
-    pytest {posargs}
+    pytest -k "not cycle" {posargs}
+
+[testenv:download_upload_cycle]
+passenv =
+    DJANGO_BASE_URL
+    DJANGO_SUPERUSER_EMAIL
+    DJANGO_SUPERUSER_PASSWORD
+deps =
+    pytest
+    click
+    pytest-factoryboy
+    pytest-mock
+commands =
+    pytest -k "cycle" {posargs}
 
 [testenv:py37]
 install_command = pip install --ignore-requires-python {opts} {packages}