From ec6fa335e3c076fad43cbda2c0d063a8748b9e67 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Wed, 5 Apr 2023 16:01:15 -0400 Subject: [PATCH] Download-upload cycle test (#291) * Remove empty server-side tests * Add download-upload cycle test * Lint fix * Separate cycle test and run every hour * Lint fix * Write file structure comparison method * Fix api and test * Lint fix * Modify CI test triggers * Change to once per day * Remove redundant pull_request trigger --- .github/workflows/download_upload_cycle.yml | 43 ++++++++ .github/workflows/test.yml | 2 +- shapeworks_cloud/core/filters.py | 1 + shapeworks_cloud/core/models.py | 2 +- shapeworks_cloud/core/tests/__init__.py | 0 shapeworks_cloud/core/tests/conftest.py | 14 --- shapeworks_cloud/core/tests/factories.py | 22 ---- shapeworks_cloud/core/tests/test_factories.py | 9 -- shapeworks_cloud/core/tests/test_models.py | 1 - swcc/swcc/models/project.py | 1 + swcc/tests/test_download_upload.py | 101 ++++++++++++++++++ swcc/tox.ini | 15 ++- 12 files changed, 162 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/download_upload_cycle.yml delete mode 100644 shapeworks_cloud/core/tests/__init__.py delete mode 100644 shapeworks_cloud/core/tests/conftest.py delete mode 100644 shapeworks_cloud/core/tests/factories.py delete mode 100644 shapeworks_cloud/core/tests/test_factories.py delete mode 100644 shapeworks_cloud/core/tests/test_models.py create mode 100644 swcc/tests/test_download_upload.py diff --git a/.github/workflows/download_upload_cycle.yml b/.github/workflows/download_upload_cycle.yml new file mode 100644 index 00000000..ae2703cf --- /dev/null +++ b/.github/workflows/download_upload_cycle.yml @@ -0,0 +1,43 @@ +name: download_upload_cycle +on: + push: + schedule: + - cron: "0 0 * * *" +jobs: + client: + env: + DJANGO_SUPERUSER_PASSWORD: django-password + DJANGO_SUPERUSER_USERNAME: admin + DJANGO_SUPERUSER_EMAIL: admin@noemail.com + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install tox + run: | + pip install --upgrade pip + pip install tox + + - name: Build images + run: docker-compose build + - name: Run migrations + run: docker-compose run --rm django ./manage.py migrate + - name: Create super user + run: docker-compose run -e DJANGO_SUPERUSER_PASSWORD=$DJANGO_SUPERUSER_PASSWORD --rm django ./manage.py createsuperuser --noinput --email=$DJANGO_SUPERUSER_EMAIL + - name: Start server + run: docker-compose up -d + + - uses: ifaxity/wait-on-action@v1 + with: + resource: 'http://localhost:8000/api/docs/swagger/' + + - name: Run tox + run: tox -e download_upload_cycle + working-directory: ./swcc + + - name: Stop server + run: docker-compose down --volumes + if: ${{ always() }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a456f23..abae7133 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: test on: - pull_request: + push: schedule: - cron: "0 0 * * *" jobs: diff --git a/shapeworks_cloud/core/filters.py b/shapeworks_cloud/core/filters.py index 602428b7..b22faedf 100644 --- a/shapeworks_cloud/core/filters.py +++ b/shapeworks_cloud/core/filters.py @@ -107,6 +107,7 @@ class OptimizedParticlesFilter(FilterSet): class Meta: models = models.OptimizedParticles fields = [ + 'project', 'shape_model', 'groomed_segmentation', 'groomed_mesh', diff --git a/shapeworks_cloud/core/models.py b/shapeworks_cloud/core/models.py index 2b464f37..3c2bb0d2 100644 --- a/shapeworks_cloud/core/models.py +++ b/shapeworks_cloud/core/models.py @@ -113,7 +113,7 @@ def create_new_file(self): ) def get_download_paths(self): - ret = {self.file.name.split('/')[-1]: self.file.url} + ret = {} with self.file.open() as f: data = json.load(f)['data'] diff --git a/shapeworks_cloud/core/tests/__init__.py b/shapeworks_cloud/core/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shapeworks_cloud/core/tests/conftest.py b/shapeworks_cloud/core/tests/conftest.py deleted file mode 100644 index 80693026..00000000 --- a/shapeworks_cloud/core/tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import pytest -from rest_framework.test import APIClient - - -@pytest.fixture -def api_client() -> APIClient: - return APIClient() - - -@pytest.fixture -def authenticated_api_client(user) -> APIClient: - client = APIClient() - client.force_authenticate(user=user) - return client diff --git a/shapeworks_cloud/core/tests/factories.py b/shapeworks_cloud/core/tests/factories.py deleted file mode 100644 index bfcf44f5..00000000 --- a/shapeworks_cloud/core/tests/factories.py +++ /dev/null @@ -1,22 +0,0 @@ -# import factory.django -# import factory.fuzzy - -# from shapeworks_cloud.core import models - - -# class GroomedDatasetFactory(factory.django.DjangoModelFactory): -# class Meta: -# model = models.GroomedDataset - -# name = factory.Faker('sentence') - - -# class GroomedSegmentationFactory(factory.django.DjangoModelFactory): -# class Meta: -# model = models.GroomedSegmentation - -# name = factory.Faker('file_name', extension='nrrd') -# blob = factory.django.FileField(data=b'fakeimagebytes', filename='fake.nrrd') -# mesh = factory.django.FileField(data=b'fakemeshbytes', filename='fake.vtp') -# dataset = factory.SubFactory(GroomedDatasetFactory) -# index = factory.Sequence(lambda n: n) diff --git a/shapeworks_cloud/core/tests/test_factories.py b/shapeworks_cloud/core/tests/test_factories.py deleted file mode 100644 index 73af570e..00000000 --- a/shapeworks_cloud/core/tests/test_factories.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - - -@pytest.mark.django_db -def test_dataset(groomed_dataset): - assert type(groomed_dataset.name) is str - - -# TODO real tests diff --git a/shapeworks_cloud/core/tests/test_models.py b/shapeworks_cloud/core/tests/test_models.py deleted file mode 100644 index 2ae28399..00000000 --- a/shapeworks_cloud/core/tests/test_models.py +++ /dev/null @@ -1 +0,0 @@ -pass diff --git a/swcc/swcc/models/project.py b/swcc/swcc/models/project.py index 0f3ba2a4..be1ef043 100644 --- a/swcc/swcc/models/project.py +++ b/swcc/swcc/models/project.py @@ -362,6 +362,7 @@ def create(self) -> Project: def download(self, folder: Union[Path, str]): session = current_session() + self.file.download(folder) r: requests.Response = session.get(f'{self._endpoint}/{self.id}/download/') raise_for_status(r) data = r.json() diff --git a/swcc/tests/test_download_upload.py b/swcc/tests/test_download_upload.py new file mode 100644 index 00000000..3bea08cf --- /dev/null +++ b/swcc/tests/test_download_upload.py @@ -0,0 +1,101 @@ +# This test requires that https://www.shapeworks-cloud.org/#/ is running +# This will fetch all datasets populated on the public server and attempt to +# upload them all to the local server +import filecmp +import os +import random +from tempfile import TemporaryDirectory + +from swcc import models +from swcc.api import swcc_session + +SAMPLE_SIZE = 3 + + +def project_as_dict_repr(project): + project_repr = dict(project) + project_repr['dataset'] = dict(project_repr['dataset']) + + # remove keys that are expected to differ between servers + del project_repr['id'] + del project_repr['file'] + del project_repr['dataset']['id'] + del project_repr['dataset']['creator'] + del project_repr['last_cached_analysis'] + return project_repr + + +class DirCmp(filecmp.dircmp): + def phase3(self): + fcomp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) + self.same_files, self.diff_files, self.funny_files = fcomp + + +def is_same(dir1, dir2): + compared = DirCmp(dir1, dir2) + different = ( + compared.left_only or compared.right_only or compared.diff_files or compared.funny_files + ) + if different: + return False + for subdir in compared.common_dirs: + if not is_same(os.path.join(dir1, subdir), os.path.join(dir2, subdir)): + return False + return True + + +def public_server_download(download_dir): + with swcc_session() as public_server_session: + public_server_session.login('testuser@noemail.nil', 'cicdtest') + all_projects = list(models.Project.list()) + project_subset = ( + random.sample(all_projects, SAMPLE_SIZE) + if len(all_projects) >= SAMPLE_SIZE + else all_projects + ) + for project in project_subset: + project.download(download_dir) + return project_subset + + +def test_download_upload_cycle(session): + with TemporaryDirectory() as temp_dir: + # Download from public server + download_dir = f'{temp_dir}/download' + os.mkdir(download_dir) + all_projects = public_server_download(download_dir) + + new_projects = [] + # Upload to local server + for project in all_projects: + d = models.Dataset( + name=project.dataset.name, + description=project.dataset.description, + license=project.dataset.license, + acknowledgement=project.dataset.acknowledgement, + keywords=project.dataset.keywords, + contributors=project.dataset.contributors, + publications=project.dataset.publications, + ).force_create() + new_projects.append( + models.Project( + dataset=d, + file=project.file.path, + description=project.description, + ).create() + ) + + # Redownload from local server + redownload_dir = f'{temp_dir}/redownload' + os.mkdir(redownload_dir) + for project in new_projects: + project.download(redownload_dir) + + # Test for file structure congruence + assert is_same(download_dir, redownload_dir) + + for local_project, remote_project in zip(models.Project.list(), all_projects): + local_repr = project_as_dict_repr(local_project) + remote_repr = project_as_dict_repr(remote_project) + assert local_repr == remote_repr + assert len(list(models.Project.list())) == len(all_projects) diff --git a/swcc/tox.ini b/swcc/tox.ini index 2b5171e2..d866ed1a 100644 --- a/swcc/tox.ini +++ b/swcc/tox.ini @@ -53,7 +53,20 @@ deps = pytest-factoryboy pytest-mock commands = - pytest {posargs} + pytest -k "not cycle" {posargs} + +[testenv:download_upload_cycle] +passenv = + DJANGO_BASE_URL + DJANGO_SUPERUSER_EMAIL + DJANGO_SUPERUSER_PASSWORD +deps = + pytest + click + pytest-factoryboy + pytest-mock +commands = + pytest -k "cycle" {posargs} [testenv:py37] install_command = pip install --ignore-requires-python {opts} {packages}