From 46a4812b21cc0bdd48bf44ea71a1a412374e7284 Mon Sep 17 00:00:00 2001 From: Brian Clarke <9725212+bfclarke@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:12:03 +0200 Subject: [PATCH 1/7] remove anaconda defaults channel --- deeprvat_annotations.yml | 1 - deeprvat_env.yaml | 1 - deeprvat_env_no_gpu.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/deeprvat_annotations.yml b/deeprvat_annotations.yml index 47f6e1f5..bd297949 100644 --- a/deeprvat_annotations.yml +++ b/deeprvat_annotations.yml @@ -1,7 +1,6 @@ name: deeprvat_annotations channels: - conda-forge - - defaults - bioconda dependencies: - python=3.9.16 diff --git a/deeprvat_env.yaml b/deeprvat_env.yaml index b8b26123..2bd29790 100644 --- a/deeprvat_env.yaml +++ b/deeprvat_env.yaml @@ -4,7 +4,6 @@ channels: - nvidia - conda-forge - bioconda - - defaults dependencies: - click=8.1 - cudatoolkit=11.8 diff --git a/deeprvat_env_no_gpu.yml b/deeprvat_env_no_gpu.yml index 54a16d4a..fd721c97 100644 --- a/deeprvat_env_no_gpu.yml +++ b/deeprvat_env_no_gpu.yml @@ -3,7 +3,6 @@ channels: - pytorch - conda-forge - bioconda - - defaults dependencies: - click=8.1 - dask=2023.5 From fe82e65e2447a32715d0fe673f216a9b9ff7b47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCck?= Date: Thu, 24 Oct 2024 11:45:40 +0200 Subject: [PATCH 2/7] sanity testing gh actions --- deeprvat_env_no_gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deeprvat_env_no_gpu.yml b/deeprvat_env_no_gpu.yml index fd721c97..03dc210d 100644 --- a/deeprvat_env_no_gpu.yml +++ b/deeprvat_env_no_gpu.yml @@ -3,6 +3,7 @@ channels: - pytorch - conda-forge - bioconda + - default dependencies: - click=8.1 - dask=2023.5 From c2a7319fe465be9006f36ea05a4f2c1514452816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCck?= Date: Thu, 24 Oct 2024 12:04:55 +0200 Subject: [PATCH 3/7] pin down version of mkl --- deeprvat_env_no_gpu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deeprvat_env_no_gpu.yml b/deeprvat_env_no_gpu.yml index 03dc210d..bfefd719 100644 --- a/deeprvat_env_no_gpu.yml +++ b/deeprvat_env_no_gpu.yml @@ -3,13 +3,12 @@ channels: - pytorch - conda-forge - bioconda - - default dependencies: - click=8.1 - dask=2023.5 - fastparquet=0.5 - h5py=3.1 - - mkl!=2024.1.0 + - mkl==2022.1.0 - numcodecs=0.11 - numpy=1.21 - optuna=2.10 From e8c27ad66f5c51511248acf8c06cf7026b594dbd Mon Sep 17 00:00:00 2001 From: Kayla Meyer Date: Thu, 24 Oct 2024 15:30:06 +0200 Subject: [PATCH 4/7] disable reference tests for gha testing --- .github/workflows/pipeline-tests.yml | 62 ++++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pipeline-tests.yml b/.github/workflows/pipeline-tests.yml index e57f4f9e..9b5d6b4a 100644 --- a/.github/workflows/pipeline-tests.yml +++ b/.github/workflows/pipeline-tests.yml @@ -34,7 +34,7 @@ jobs: prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ Pipeline-Tests-RunTraining: - needs: [Smoke-RunTraining, Reference-Output-Training] + needs: [Smoke-RunTraining] #, Reference-Output-Training] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/run_training.snakefile @@ -44,35 +44,35 @@ jobs: download_training_outputs: true run_training_results_check: true - Reference-Output-Training: - uses: ./.github/workflows/run-pipeline.yml - with: - branch: main - pipeline_file: ./pipelines/run_training.snakefile - environment_file: ./deeprvat_env_no_gpu.yml - prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ - dry_run: false - upload_training_outputs: true - - Reference-Output-Pretrained: - uses: ./.github/workflows/run-pipeline.yml - with: - branch: main - pipeline_file: ./pipelines/association_testing_pretrained.snakefile - environment_file: ./deeprvat_env_no_gpu.yml - prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ - dry_run: false - upload_pretrained_outputs: true + # Reference-Output-Training: + # uses: ./.github/workflows/run-pipeline.yml + # with: + # branch: main + # pipeline_file: ./pipelines/run_training.snakefile + # environment_file: ./deeprvat_env_no_gpu.yml + # prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ + # dry_run: false + # upload_training_outputs: true + + # Reference-Output-Pretrained: + # uses: ./.github/workflows/run-pipeline.yml + # with: + # branch: main + # pipeline_file: ./pipelines/association_testing_pretrained.snakefile + # environment_file: ./deeprvat_env_no_gpu.yml + # prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ + # dry_run: false + # upload_pretrained_outputs: true - Reference-Output-Pretrained-Regenie: - uses: ./.github/workflows/run-pipeline.yml - with: - branch: main - pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile - environment_file: ./deeprvat_env_no_gpu.yml - prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ - dry_run: false - upload_regenie_outputs: true + # Reference-Output-Pretrained-Regenie: + # uses: ./.github/workflows/run-pipeline.yml + # with: + # branch: main + # pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile + # environment_file: ./deeprvat_env_no_gpu.yml + # prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ + # dry_run: false + # upload_regenie_outputs: true # Association Testing Pretrained Pipeline Smoke-Association-Testing-Pretrained: @@ -83,7 +83,7 @@ jobs: prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ Pipeline-Tests-Training-Association-Testing: - needs: [Smoke-Association-Testing-Pretrained, Reference-Output-Pretrained] + needs: [Smoke-Association-Testing-Pretrained] #, Reference-Output-Pretrained] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/association_testing_pretrained.snakefile @@ -103,7 +103,7 @@ jobs: prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ Pipeline-Tests-Association-Testing-Pretrained-Regenie: - needs: [Smoke-Association-Testing-Pretrained-Regenie, Reference-Output-Pretrained-Regenie] + needs: [Smoke-Association-Testing-Pretrained-Regenie] #, Reference-Output-Pretrained-Regenie] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile From 8bf9102e73195125d365df8ee277e99774be67c5 Mon Sep 17 00:00:00 2001 From: Kayla Meyer Date: Thu, 24 Oct 2024 15:46:56 +0200 Subject: [PATCH 5/7] disable results checking for gha testing --- .github/workflows/pipeline-tests.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pipeline-tests.yml b/.github/workflows/pipeline-tests.yml index 9b5d6b4a..b137d961 100644 --- a/.github/workflows/pipeline-tests.yml +++ b/.github/workflows/pipeline-tests.yml @@ -41,8 +41,8 @@ jobs: environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ dry_run: false - download_training_outputs: true - run_training_results_check: true + #download_training_outputs: true + #run_training_results_check: true # Reference-Output-Training: # uses: ./.github/workflows/run-pipeline.yml @@ -90,9 +90,9 @@ jobs: environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ dry_run: false - download_pretrained_outputs: true - run_burden_results_check: true - run_association_results_check: true + #download_pretrained_outputs: true + #run_burden_results_check: true + #run_association_results_check: true # Association Testing Pretrained Regenie Smoke-Association-Testing-Pretrained-Regenie: @@ -110,9 +110,8 @@ jobs: environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ dry_run: false - download_regenie_outputs: true - #run_burden_results_check: true - run_regenie_association_results_check: true + #download_regenie_outputs: true + #run_regenie_association_results_check: true # Association Testing Training Smoke-Association-Testing-Training: From b9dc7f3478d7b1e935d01d786becbd6863aa6081 Mon Sep 17 00:00:00 2001 From: Kayla Meyer Date: Thu, 24 Oct 2024 16:01:26 +0200 Subject: [PATCH 6/7] reactivate reference results checking tests --- .github/workflows/pipeline-tests.yml | 78 ++++++++++++++-------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pipeline-tests.yml b/.github/workflows/pipeline-tests.yml index b137d961..f810140c 100644 --- a/.github/workflows/pipeline-tests.yml +++ b/.github/workflows/pipeline-tests.yml @@ -34,45 +34,45 @@ jobs: prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ Pipeline-Tests-RunTraining: - needs: [Smoke-RunTraining] #, Reference-Output-Training] + needs: [Smoke-RunTraining, Reference-Output-Training] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/run_training.snakefile environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ dry_run: false - #download_training_outputs: true - #run_training_results_check: true - - # Reference-Output-Training: - # uses: ./.github/workflows/run-pipeline.yml - # with: - # branch: main - # pipeline_file: ./pipelines/run_training.snakefile - # environment_file: ./deeprvat_env_no_gpu.yml - # prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ - # dry_run: false - # upload_training_outputs: true - - # Reference-Output-Pretrained: - # uses: ./.github/workflows/run-pipeline.yml - # with: - # branch: main - # pipeline_file: ./pipelines/association_testing_pretrained.snakefile - # environment_file: ./deeprvat_env_no_gpu.yml - # prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ - # dry_run: false - # upload_pretrained_outputs: true + download_training_outputs: true + run_training_results_check: true + + Reference-Output-Training: + uses: ./.github/workflows/run-pipeline.yml + with: + branch: main + pipeline_file: ./pipelines/run_training.snakefile + environment_file: ./deeprvat_env_no_gpu.yml + prerun_cmd: cp ./tests/deeprvat/training_association_testing/deeprvat_config.yaml ./example/ + dry_run: false + upload_training_outputs: true + + Reference-Output-Pretrained: + uses: ./.github/workflows/run-pipeline.yml + with: + branch: main + pipeline_file: ./pipelines/association_testing_pretrained.snakefile + environment_file: ./deeprvat_env_no_gpu.yml + prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ + dry_run: false + upload_pretrained_outputs: true - # Reference-Output-Pretrained-Regenie: - # uses: ./.github/workflows/run-pipeline.yml - # with: - # branch: main - # pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile - # environment_file: ./deeprvat_env_no_gpu.yml - # prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ - # dry_run: false - # upload_regenie_outputs: true + Reference-Output-Pretrained-Regenie: + uses: ./.github/workflows/run-pipeline.yml + with: + branch: main + pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile + environment_file: ./deeprvat_env_no_gpu.yml + prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ + dry_run: false + upload_regenie_outputs: true # Association Testing Pretrained Pipeline Smoke-Association-Testing-Pretrained: @@ -83,16 +83,16 @@ jobs: prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ Pipeline-Tests-Training-Association-Testing: - needs: [Smoke-Association-Testing-Pretrained] #, Reference-Output-Pretrained] + needs: [Smoke-Association-Testing-Pretrained, Reference-Output-Pretrained] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/association_testing_pretrained.snakefile environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/pretrained/deeprvat_config.yaml ./example/ dry_run: false - #download_pretrained_outputs: true - #run_burden_results_check: true - #run_association_results_check: true + download_pretrained_outputs: true + run_burden_results_check: true + run_association_results_check: true # Association Testing Pretrained Regenie Smoke-Association-Testing-Pretrained-Regenie: @@ -103,15 +103,15 @@ jobs: prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ Pipeline-Tests-Association-Testing-Pretrained-Regenie: - needs: [Smoke-Association-Testing-Pretrained-Regenie] #, Reference-Output-Pretrained-Regenie] + needs: [Smoke-Association-Testing-Pretrained-Regenie, Reference-Output-Pretrained-Regenie] uses: ./.github/workflows/run-pipeline.yml with: pipeline_file: ./pipelines/association_testing_pretrained_regenie.snakefile environment_file: ./deeprvat_env_no_gpu.yml prerun_cmd: cp ./tests/deeprvat/regenie/pretrained/deeprvat_config.yaml ./example/ dry_run: false - #download_regenie_outputs: true - #run_regenie_association_results_check: true + download_regenie_outputs: true + run_regenie_association_results_check: true # Association Testing Training Smoke-Association-Testing-Training: From 8aecaa04f1af4de226c924711a3a6be0d9a1f82a Mon Sep 17 00:00:00 2001 From: Kayla Meyer Date: Fri, 25 Oct 2024 10:16:00 +0200 Subject: [PATCH 7/7] pin mkl version in regular deeprvat env file --- deeprvat_env.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deeprvat_env.yaml b/deeprvat_env.yaml index 2bd29790..a50346da 100644 --- a/deeprvat_env.yaml +++ b/deeprvat_env.yaml @@ -10,6 +10,7 @@ dependencies: - dask=2023.5 - fastparquet=0.5 - h5py=3.1 + - mkl==2022.1.0 - numcodecs=0.11 - numpy=1.21 - optuna=2.10