Skip to content

Commit

Permalink
Merge pull request #71 from e-merlin/fringefit
Browse files Browse the repository at this point in the history 
Fringefit branch
  • Loading branch information
@jmoldon
jmoldon authored Oct 30, 2017
2 parents 960f917 + 7aadce1 commit 2c2bef9
Show file tree
Hide file tree
Showing 11 changed files with 1,357 additions and 576 deletions.
33 changes: 20 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@

## TL;DR ##
This is the CASA pipeline for e-MERLIN data, it is designed to be fully parallelised. It is, however, very early stages and will progress rapidly. It currently does work but does not do calibration yet. Feel free to change/modify.

## Dependencies ##
- aoflagger v2.7+ (v2.9+ recommended) (see https://sourceforge.net/projects/aoflagger/)
- CASA v4.6+ (see https://casa.nrao.edu/), tested on CASA v4.6.0
- CASA v5.0+ (see https://casa.nrao.edu/)
- aoflagger v2.9+ (see https://sourceforge.net/projects/aoflagger/). v2.7+ accepted but not recommended, will be deprecated.
- python2.7

## Download ##
Expand All @@ -14,15 +13,23 @@ If you have git installed, you can get the pipeline using:
If you don't have git, you can download and unzip the files from [here](https://github.com/e-merlin/CASA_eMERLIN_pipeline/archive/master.zip).

## Usage ##
The pipeline uses a GUI interface as an input and now has a headless version. To run, do the following:
* Using GUI:
`mpicasa -n <num_cores casa> -c /path/to/pipeline/eMERLIN_CASA_pipeline.py -g`
* With an input file (find sameple in pipeline. Define data location and steps to follow)
`mpicasa -n <num_cores> casa -c /path/to/pipeline/eMERLIN_CASA_pipeline.py -i <input file>`

N.b. replace num_cores with the number of cores you would like to use.
To run not in parallel remove `mpicasa -n num_cores`

IMPORTANT: The pipeline will look in your data directory for a ms or mms with the prefix defined in inputs and copy it to your current directory if it cannot find it in the current directory.
To run the pipeline simply do:
`casa -c /path/to/pipeline/eMERLIN_CASA_pipeline.py -i <input file>`

To run the parallelized version using MPI in CASA you can use:
`mpicasa -n <num_cores> casa -c /path/to/pipeline/eMERLIN_CASA_pipeline.py -i <input file>`

To execute the pipeline from within CASA:
~~~~
> run_in_casa = True
> pipeline_path = '/path/to/pipeline_path/' # You need to define this variable explicitly
> execfile(pipeline_path + 'eMERLIN_CASA_pipeline.py')
> inputs, msinfo = run_pipeline(inputs_path=<input file>)
~~~~

## Additional information ##

- [Summary of procedures, pdf](https://github.com/e-merlin/CASA_eMERLIN_pipeline/wiki/files/summary.pdf)
- [Wiki pages](https://github.com/e-merlin/CASA_eMERLIN_pipeline/wiki)


34 changes: 13 additions & 21 deletions aoflagger_strategies/default/0319+415.rfis
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a strategy configuration file for the AOFlagger RFI
detector by André Offringa (offringa@gmail.com).
Created by AOFlagger 2.7.1 (2015-07-01)
Created by AOFlagger 2.9.0 (2016-12-20)
-->
<rfi-strategy format-version="3.7" reader-version-required="3.4">
<action type="Strategy">
Expand Down Expand Up @@ -30,18 +30,21 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<restore-from-amplitude>0</restore-from-amplitude>
<children>
<action type="IterationBlock">
<iteration-count>2</iteration-count>
<sensitivity-start>4</sensitivity-start>
<iteration-count>5</iteration-count>
<sensitivity-start>8</sensitivity-start>
<children>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>0.8</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
<action type="CombineFlagResults">
<children>
<action type="FrequencySelectionAction">
<threshold>3</threshold>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
<threshold>3</threshold>
</action>
</children>
</action>
Expand All @@ -56,7 +59,7 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<children>
<action type="HighPassFilterAction">
<horizontal-kernel-sigma-sq>2.5</horizontal-kernel-sigma-sq>
<vertical-kernel-sigma-sq>1</vertical-kernel-sigma-sq>
<vertical-kernel-sigma-sq>5</vertical-kernel-sigma-sq>
<window-width>21</window-width>
<window-height>31</window-height>
<mode>1</mode>
Expand All @@ -65,13 +68,10 @@ Created by AOFlagger 2.7.1 (2015-07-01)
</action>
</children>
</action>
<action type="CalibratePassbandAction">
<steps>48</steps>
</action>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>0.8</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
</children>
</action>
Expand All @@ -93,15 +93,7 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<minimum-good-time-ratio>0.2</minimum-good-time-ratio>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
</action>
<action type="BaselineSelectionAction">
<preparation-step>1</preparation-step>
<flag-bad-baselines>0</flag-bad-baselines>
<threshold>8</threshold>
<smoothing-sigma>0.6</smoothing-sigma>
<abs-threshold>0.4</abs-threshold>
<make-plot>0</make-plot>
<threshold>4</threshold>
</action>
</children>
</action>
Expand Down
34 changes: 13 additions & 21 deletions aoflagger_strategies/default/1331+305.rfis
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a strategy configuration file for the AOFlagger RFI
detector by André Offringa (offringa@gmail.com).
Created by AOFlagger 2.7.1 (2015-07-01)
Created by AOFlagger 2.9.0 (2016-12-20)
-->
<rfi-strategy format-version="3.7" reader-version-required="3.4">
<action type="Strategy">
Expand Down Expand Up @@ -30,18 +30,21 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<restore-from-amplitude>0</restore-from-amplitude>
<children>
<action type="IterationBlock">
<iteration-count>2</iteration-count>
<sensitivity-start>4</sensitivity-start>
<iteration-count>5</iteration-count>
<sensitivity-start>8</sensitivity-start>
<children>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>0.8</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
<action type="CombineFlagResults">
<children>
<action type="FrequencySelectionAction">
<threshold>3</threshold>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
<threshold>3</threshold>
</action>
</children>
</action>
Expand All @@ -56,7 +59,7 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<children>
<action type="HighPassFilterAction">
<horizontal-kernel-sigma-sq>2.5</horizontal-kernel-sigma-sq>
<vertical-kernel-sigma-sq>1</vertical-kernel-sigma-sq>
<vertical-kernel-sigma-sq>5</vertical-kernel-sigma-sq>
<window-width>21</window-width>
<window-height>31</window-height>
<mode>1</mode>
Expand All @@ -65,13 +68,10 @@ Created by AOFlagger 2.7.1 (2015-07-01)
</action>
</children>
</action>
<action type="CalibratePassbandAction">
<steps>48</steps>
</action>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>0.8</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
</children>
</action>
Expand All @@ -93,15 +93,7 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<minimum-good-time-ratio>0.2</minimum-good-time-ratio>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
</action>
<action type="BaselineSelectionAction">
<preparation-step>1</preparation-step>
<flag-bad-baselines>0</flag-bad-baselines>
<threshold>8</threshold>
<smoothing-sigma>0.6</smoothing-sigma>
<abs-threshold>0.4</abs-threshold>
<make-plot>0</make-plot>
<threshold>4</threshold>
</action>
</children>
</action>
Expand Down
34 changes: 13 additions & 21 deletions aoflagger_strategies/default/1407+284.rfis
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a strategy configuration file for the AOFlagger RFI
detector by André Offringa (offringa@gmail.com).
Created by AOFlagger 2.7.1 (2015-07-01)
Created by AOFlagger 2.9.0 (2016-12-20)
-->
<rfi-strategy format-version="3.7" reader-version-required="3.4">
<action type="Strategy">
Expand Down Expand Up @@ -30,16 +30,19 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<restore-from-amplitude>0</restore-from-amplitude>
<children>
<action type="IterationBlock">
<iteration-count>2</iteration-count>
<sensitivity-start>4</sensitivity-start>
<iteration-count>5</iteration-count>
<sensitivity-start>8</sensitivity-start>
<children>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>1</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
<action type="CombineFlagResults">
<children>
<action type="FrequencySelectionAction">
<threshold>3</threshold>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
</action>
Expand All @@ -50,13 +53,13 @@ Created by AOFlagger 2.7.1 (2015-07-01)
</action>
<action type="ChangeResolutionAction">
<time-decrease-factor>3</time-decrease-factor>
<frequency-decrease-factor>5</frequency-decrease-factor>
<frequency-decrease-factor>3</frequency-decrease-factor>
<restore-revised>1</restore-revised>
<restore-masks>0</restore-masks>
<children>
<action type="HighPassFilterAction">
<horizontal-kernel-sigma-sq>2.5</horizontal-kernel-sigma-sq>
<vertical-kernel-sigma-sq>9</vertical-kernel-sigma-sq>
<vertical-kernel-sigma-sq>5</vertical-kernel-sigma-sq>
<window-width>21</window-width>
<window-height>31</window-height>
<mode>1</mode>
Expand All @@ -65,13 +68,10 @@ Created by AOFlagger 2.7.1 (2015-07-01)
</action>
</children>
</action>
<action type="CalibratePassbandAction">
<steps>48</steps>
</action>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>1</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
</children>
</action>
Expand All @@ -93,15 +93,7 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<minimum-good-time-ratio>0.2</minimum-good-time-ratio>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
</action>
<action type="BaselineSelectionAction">
<preparation-step>1</preparation-step>
<flag-bad-baselines>0</flag-bad-baselines>
<threshold>8</threshold>
<smoothing-sigma>0.6</smoothing-sigma>
<abs-threshold>0.4</abs-threshold>
<make-plot>0</make-plot>
<threshold>4</threshold>
</action>
</children>
</action>
Expand Down
36 changes: 14 additions & 22 deletions aoflagger_strategies/default/default_faint.rfis
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- This is a strategy configuration file for the AOFlagger RFI
detector by André Offringa (offringa@gmail.com).
Created by AOFlagger 2.7.1 (2015-07-01)
Created by AOFlagger 2.9.0 (2016-12-20)
-->
<rfi-strategy format-version="3.7" reader-version-required="3.4">
<action type="Strategy">
Expand Down Expand Up @@ -30,27 +30,30 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<restore-from-amplitude>0</restore-from-amplitude>
<children>
<action type="IterationBlock">
<iteration-count>2</iteration-count>
<sensitivity-start>4</sensitivity-start>
<iteration-count>5</iteration-count>
<sensitivity-start>8</sensitivity-start>
<children>
<action type="SumThresholdAction">
<base-sensitivity>1.4</base-sensitivity>
<base-sensitivity>1</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
<action type="CombineFlagResults">
<children>
<action type="TimeSelectionAction">
<action type="FrequencySelectionAction">
<threshold>3</threshold>
</action>
<action type="TimeSelectionAction">
<threshold>3.5</threshold>
</action>
</children>
</action>
<action type="SetImageAction">
<new-image>1</new-image>
</action>
<action type="ChangeResolutionAction">
<time-decrease-factor>3</time-decrease-factor>
<frequency-decrease-factor>10</frequency-decrease-factor>
<frequency-decrease-factor>3</frequency-decrease-factor>
<restore-revised>1</restore-revised>
<restore-masks>0</restore-masks>
<children>
Expand All @@ -65,13 +68,10 @@ Created by AOFlagger 2.7.1 (2015-07-01)
</action>
</children>
</action>
<action type="CalibratePassbandAction">
<steps>48</steps>
</action>
<action type="SumThresholdAction">
<base-sensitivity>1.2</base-sensitivity>
<base-sensitivity>1</base-sensitivity>
<time-direction-flagging>1</time-direction-flagging>
<frequency-direction-flagging>1</frequency-direction-flagging>
<frequency-direction-flagging>0</frequency-direction-flagging>
</action>
</children>
</action>
Expand All @@ -93,16 +93,8 @@ Created by AOFlagger 2.7.1 (2015-07-01)
<minimum-good-time-ratio>0.2</minimum-good-time-ratio>
</action>
<action type="TimeSelectionAction">
<threshold>3.0</threshold>
</action>
<action type="BaselineSelectionAction">
<preparation-step>1</preparation-step>
<flag-bad-baselines>0</flag-bad-baselines>
<threshold>8</threshold>
<smoothing-sigma>0.6</smoothing-sigma>
<abs-threshold>0.4</abs-threshold>
<make-plot>0</make-plot>
<threshold>4</threshold>
</action>
</children>
</action>
</rfi-strategy>
</rfi-strategy>
Loading

0 comments on commit 2c2bef9

Please sign in to comment.