Merge pull request #490 from DeepRegNet/489-add-option-in-travis-ci-t…

…o-run-tests-on-demo

Issue #489: test demos depending on job type and commit msg test-demo

.travis.yml

@@ -2,7 +2,7 @@ language: python
 os: linux
 dist: bionic
 python:
-  - "3.7" # current default Python on Travis CI
+  - "3.7" # same as environment.yml
 install:
   - pip install -e . # install deepreg package with corresponding requirements
   - pip install codecov
@@ -29,3 +29,43 @@ jobs:
         - deepreg_train -g "" --config_path config/unpaired_labeled_ddf.yaml --log_dir test
         - deepreg_predict -g "" --ckpt_path logs/test/save/weights-epoch2.ckpt --mode test -b 3
         - deepreg_warp --image data/test/nifti/unit_test/moving_image.nii.gz --ddf data/test/nifti/unit_test/ddf.nii.gz --out logs/test_warp/out.nii.gz
+    - stage: "Demos"
+      name: "test_single_config_demo[grouped_mask_prostate_longitudinal]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[grouped_mask_prostate_longitudinal]
+    - name: "test_single_config_demo[grouped_mr_heart]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[grouped_mr_heart]
+    - name: "test_single_config_demo[paired_ct_lung]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[paired_ct_lung]
+    - name: "test_single_config_demo[paired_mrus_brain]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[paired_mrus_brain]
+    - name: "test_single_config_demo[paired_mrus_prostate]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[paired_mrus_prostate]
+    - name: "test_single_config_demo[unpaired_ct_lung]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[unpaired_ct_lung]
+    - name: "test_single_config_demo[unpaired_mr_brain]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[unpaired_mr_brain]
+    - name: "test_single_config_demo[unpaired_us_prostate_cv]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_single_config_demo[unpaired_us_prostate_cv]
+    - name: "test_unpaired_ct_abdomen[comb]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_unpaired_ct_abdomen[comb]
+    - name: "test_unpaired_ct_abdomen[unsup]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_unpaired_ct_abdomen[unsup]
+    - name: "test_unpaired_ct_abdomen[weakly]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_unpaired_ct_abdomen[weakly]
+    - name: "test_classical_demo[classical_ct_headneck_affine]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_classical_demo[classical_ct_headneck_affine]
+    - name: "test_classical_demo[classical_mr_prostate_nonrigid]"
+      if: type = cron OR commit_message =~ /test-demo/
+      script: pytest test/demo/ -k test_classical_demo[classical_mr_prostate_nonrigid]

CHANGELOG.md

@@ -17,7 +17,7 @@ straightforward as possible.
 - Added max_epochs argument for training to overwrite configuration.
 - Added log_root argument for training and prediction to customize the log file
   location.
-- Added tests for all demos.
+- Added integration tests for all demos.
 - Added environment.yml file for Conda environment creation.
 - Added Dockerfile.
 - Added documentation about using UCL cluster with DeepReg.

demos/grouped_mask_prostate_longitudinal/demo_predict.py

@@ -1,9 +1,31 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "grouped_mask_prostate_longitudinal"
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -21,7 +43,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/weights-epoch500.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_unpaired_grouped.yaml")
 
 predict(
     gpu="0",

demos/grouped_mr_heart/demo_predict.py

@@ -1,9 +1,30 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "grouped_mr_heart"
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -21,7 +42,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/weights-epoch500.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_unpaired_grouped.yaml")
 
 predict(
     gpu="0",

demos/paired_ct_lung/demo_predict.py

@@ -1,9 +1,29 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "paired_ct_lung"
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
 
 print(
     "\n\n\n\n\n"
@@ -22,7 +42,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/learn2reg_t2_paired_train_logs/save/weights-epoch500.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_paired.yaml")
 
 predict(
     gpu="0",

demos/paired_mrus_brain/demo_predict.py

@@ -1,9 +1,30 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "paired_mrus_brain"
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -21,7 +42,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/learn2reg_t1_paired_train_logs/save/weights-epoch800.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_paired.yaml")
 
 predict(
     gpu="0",

demos/paired_mrus_prostate/demo_predict.py

@@ -1,9 +1,30 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "paired_mrus_prostate"
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -21,7 +42,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/weights-epoch500.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_paired.yaml")
 
 predict(
     gpu="0",

demos/unpaired_ct_abdomen/demo_predict.py

@@ -13,6 +13,19 @@
     type=str,
     required=True,
 )
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
 args = parser.parse_args()
 method = args.method
 assert method in [
@@ -39,7 +52,9 @@
 log_root = f"demos/{name}"
 log_dir = f"logs_predict/{method}/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/{method}/weights-epoch{ckpt_index}.ckpt"
-config_path = f"{log_root}/{name}_{method}.yaml"
+config_path = [f"{log_root}/{name}_{method}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_unpaired_grouped.yaml")
 
 predict(
     gpu="0",

demos/unpaired_ct_lung/demo_predict.py

@@ -1,9 +1,31 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "unpaired_ct_lung"
 
+
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -21,7 +43,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/learn2reg_t2_unpaired_train_logs/save/weights-epoch1500.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_unpaired_grouped.yaml")
 
 predict(
     gpu="0",

demos/unpaired_mr_brain/demo_predict.py

@@ -1,10 +1,31 @@
+import argparse
 from datetime import datetime
 
 from deepreg.predict import predict
 
 name = "unpaired_mr_brain"
 
 
+# parser is used to simplify testing, by default it is not used
+# please run the script with --no-test flag to ensure non-testing mode
+# for instance:
+# python script.py --no-test
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--test",
+    help="Execute the script for test purpose",
+    dest="test",
+    action="store_true",
+)
+parser.add_argument(
+    "--no-test",
+    help="Execute the script for non-test purpose",
+    dest="test",
+    action="store_false",
+)
+parser.set_defaults(test=False)
+args = parser.parse_args()
+
 print(
     "\n\n\n\n\n"
     "=========================================================\n"
@@ -22,7 +43,9 @@
 log_root = f"demos/{name}"
 log_dir = "logs_predict/" + datetime.now().strftime("%Y%m%d-%H%M%S")
 ckpt_path = f"{log_root}/dataset/pretrained/learn2reg_t4_unpaired_weakly_train_logs2/save/weights-epoch200.ckpt"
-config_path = f"{log_root}/{name}.yaml"
+config_path = [f"{log_root}/{name}.yaml"]
+if args.test:
+    config_path.append("config/test/demo_unpaired_grouped.yaml")
 
 predict(
     gpu="0",