diff --git a/.gitignore b/.gitignore index 57b6e0f..dd625ec 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ out_tmpdir*/ runs/ *.simg slurm-*.out -steps/*.cwl diff --git a/Makefile b/Makefile index cbc90bd..be02829 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ all: run SHELL=bash RUN := $(PWD)/runs/run_$(shell date +%F-%H-%M-%S) -SINGULARITY_PREFIX=$(shell echo "singularity, exec, $(PWD)/prefactor.simg, " | sed -e 's/[\/&]/\\&/g') ARCHIVE=ftp://ftp.astron.nl/outgoing/EOSC/datasets/ TINY=L591513_SB000_uv_delta_t_4.MS PULSAR=GBT_Lband_PSR.fil @@ -44,7 +43,7 @@ data/$(SMALL)/: data/$(SMALL_ARCHIVE) small: data/$(SMALL)/ echo "data/$(SMALL)/ is downloaded" -run-udocker: .virtualenv/bin/udocker steps/ndppp_prep_cal.cwl .virtualenv/bin/cwltool +run-udocker: .virtualenv/bin/udocker .virtualenv/bin/cwltool mkdir -p $(RUN) .virtualenv/bin/cwltool \ --user-space-docker-cmd `pwd`/.virtualenv/bin/udocker \ @@ -53,7 +52,7 @@ run-udocker: .virtualenv/bin/udocker steps/ndppp_prep_cal.cwl .virtualenv/bin/cw prefactor.cwl \ jobs/job_20sb.yaml > >(tee $(RUN)/output) 2> >(tee $(RUN)/log >&2) -run: .virtualenv/bin/cwltool steps/ndppp_prep_cal.cwl +run: .virtualenv/bin/cwltool mkdir -p $(RUN) .virtualenv/bin/cwltool \ --leave-tmpdir \ @@ -62,7 +61,7 @@ run: .virtualenv/bin/cwltool steps/ndppp_prep_cal.cwl prefactor.cwl \ jobs/job_2sb.yaml > >(tee $(RUN)/output) 2> >(tee $(RUN)/log >&2) -toil: data/$(SMALL)/ .virtualenv/bin/cwltoil steps/ndppp_prep_cal.cwl +toil: data/$(SMALL)/ .virtualenv/bin/cwltoil mkdir -p $(RUN)/results .virtualenv/bin/toil-cwl-runner \ --logFile $(RUN)/log \ @@ -73,12 +72,11 @@ toil: data/$(SMALL)/ .virtualenv/bin/cwltoil steps/ndppp_prep_cal.cwl prefactor.cwl \ jobs/job_20sb.yaml | tee $(RUN)/output -slurm: data/$(SMALL) .virtualenv/bin/cwltoil singularity +slurm: data/$(SMALL) .virtualenv/bin/cwltoil mkdir -p $(RUN)/results .virtualenv/bin/toil-cwl-runner \ --batchSystem=slurm \ --preserve-environment PATH \ - --no-container \ --logFile $(RUN)/log \ --outdir $(RUN)/results \ --jobStore file://$(RUN)/job_store \ @@ -93,7 +91,6 @@ mesos: data/$(SMALL) --batchSystem=mesos \ --mesosMaster=145.100.59.50:5050 \ --preserve-environment PATH \ - --no-container \ --logFile $(RUN)/log \ --outdir $(RUN)/results \ --jobStore file://$(RUN)/job_store \ @@ -107,11 +104,3 @@ prefactor.simg: singularity build prefactor.simg docker://kernsuite/prefactor singularity: prefactor.simg - for i in `ls steps/*.in`; do sed 's/CMD_PREFIX/$(SINGULARITY_PREFIX)/g' $$i> $${i:0:-3}; done - -no-singularity: - for i in `ls steps/*.in`; do sed 's/CMD_PREFIX//g' $$i> $${i:0:-3}; done - -steps/ndppp_prep_cal.cwl: - $(error "Run '$ make singularity' or '$ make no-singularity' first") - diff --git a/README.md b/README.md index ab5fee6..0410212 100644 --- a/README.md +++ b/README.md @@ -20,20 +20,6 @@ optional: * [Singularity](http://singularity.lbl.gov/) -# preperations - -You first need to preprocess the CWL files since the standard -doesn't support Singularity yet. If you want to use singularity run: - -```bash -$ make singularity -``` - -otherwise run: -```bash -$ make no-singularity -``` - # usage To run the pipeline with the example dataset just run: diff --git a/steps/ampl.cwl.in b/steps/ampl.cwl similarity index 92% rename from steps/ampl.cwl.in rename to steps/ampl.cwl index c51ad9f..4ab615b 100644 --- a/steps/ampl.cwl.in +++ b/steps/ampl.cwl @@ -1,6 +1,6 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython, /usr/lib/prefactor/scripts/amplitudes_losoto_3.py] +baseCommand: [python, /usr/lib/prefactor/scripts/amplitudes_losoto_3.py] label: "Smooth amplitudes" diff --git a/steps/calib_cal.cwl.in b/steps/calib_cal.cwl similarity index 97% rename from steps/calib_cal.cwl.in rename to steps/calib_cal.cwl index 899509e..f004370 100644 --- a/steps/calib_cal.cwl.in +++ b/steps/calib_cal.cwl @@ -29,7 +29,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIX/usr/bin/calibrate-stand-alone] +baseCommand: [/usr/bin/calibrate-stand-alone] label: "BBS calibrate" diff --git a/steps/createmap_cal.cwl.in b/steps/createmap_cal.cwl similarity index 98% rename from steps/createmap_cal.cwl.in rename to steps/createmap_cal.cwl index a0d4bc5..06a8328 100644 --- a/steps/createmap_cal.cwl.in +++ b/steps/createmap_cal.cwl @@ -29,7 +29,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython, +baseCommand: [python, /usr/lib/python2.7/dist-packages/lofarpipe/recipes/plugins/PipelineStep_createMapfile.py, createmap_cal.mapfile] diff --git a/steps/examine-npys.cwl.in b/steps/examine-npys.cwl similarity index 89% rename from steps/examine-npys.cwl.in rename to steps/examine-npys.cwl index 6ebff68..0e5ae61 100644 --- a/steps/examine-npys.cwl.in +++ b/steps/examine-npys.cwl @@ -1,6 +1,6 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython, /usr/lib/prefactor/scripts/examine_npys.py] +baseCommand: [python, /usr/lib/prefactor/scripts/examine_npys.py] hints: DockerRequirement: diff --git a/steps/fitclock.cwl.in b/steps/fitclock.cwl similarity index 93% rename from steps/fitclock.cwl.in rename to steps/fitclock.cwl index dab13a1..0b92e9a 100644 --- a/steps/fitclock.cwl.in +++ b/steps/fitclock.cwl @@ -1,6 +1,6 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython, /usr/lib/prefactor/scripts/fit_clocktec_initialguess_losoto.py] +baseCommand: [python, /usr/lib/prefactor/scripts/fit_clocktec_initialguess_losoto.py] label: "Separate clock/TEC" diff --git a/steps/h5imp_cal.cwl.in b/steps/h5imp_cal.cwl similarity index 93% rename from steps/h5imp_cal.cwl.in rename to steps/h5imp_cal.cwl index 4a9faaf..daa0fb1 100644 --- a/steps/h5imp_cal.cwl.in +++ b/steps/h5imp_cal.cwl @@ -1,6 +1,6 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython, /usr/lib/prefactor/scripts/losotoImporter.py, losoto.h5] +baseCommand: [python, /usr/lib/prefactor/scripts/losotoImporter.py, losoto.h5] label: "Convert solutions to H5parm" diff --git a/steps/ndppp_prep_cal.cwl.in b/steps/ndppp_prep_cal.cwl similarity index 98% rename from steps/ndppp_prep_cal.cwl.in rename to steps/ndppp_prep_cal.cwl index bf9cb05..0c1f910 100644 --- a/steps/ndppp_prep_cal.cwl.in +++ b/steps/ndppp_prep_cal.cwl @@ -20,7 +20,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXNDPPP, msout=calibrated.MS] +baseCommand: [NDPPP, msout=calibrated.MS] label: "DPPP flag and average" diff --git a/steps/phase.cwl.in b/steps/phase.cwl similarity index 98% rename from steps/phase.cwl.in rename to steps/phase.cwl index e35c8fb..303a3f3 100644 --- a/steps/phase.cwl.in +++ b/steps/phase.cwl @@ -6,7 +6,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython] +baseCommand: [python] label: "Find global phase offset" diff --git a/steps/plot_cal_phases.cwl.in b/steps/plot_cal_phases.cwl similarity index 98% rename from steps/plot_cal_phases.cwl.in rename to steps/plot_cal_phases.cwl index 66985c3..63b4b9e 100644 --- a/steps/plot_cal_phases.cwl.in +++ b/steps/plot_cal_phases.cwl @@ -20,7 +20,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXlosoto] +baseCommand: [losoto] label: "Plot raw calibration results" diff --git a/steps/plots.cwl.in b/steps/plots.cwl similarity index 98% rename from steps/plots.cwl.in rename to steps/plots.cwl index a201699..a649b0c 100644 --- a/steps/plots.cwl.in +++ b/steps/plots.cwl @@ -9,7 +9,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython] +baseCommand: [python] label: "Plot clock, TEC, amplitude" diff --git a/steps/sky_cal.cwl.in b/steps/sky_cal.cwl similarity index 97% rename from steps/sky_cal.cwl.in rename to steps/sky_cal.cwl index edb13e7..d565139 100644 --- a/steps/sky_cal.cwl.in +++ b/steps/sky_cal.cwl @@ -1,7 +1,7 @@ cwlVersion: v1.0 class: CommandLineTool -baseCommand: [CMD_PREFIXpython] +baseCommand: [python] label: "Find calibrator skymodel"