Merge pull request #161 from eqcorrscan/develop

Merge for Release 0.2.6

.circleci/config.yml

@@ -0,0 +1,69 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+general:
+  branches:
+    ignore:
+      - gh-pages
+
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
+      - image: circleci/python:3.6.1
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+      - restore_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: install dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install numpy>=1.12
+            pip install -r requirements.txt
+      - save_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
+          paths:
+            - "venv"
+      - run:
+          name: install fftw
+          command: |
+            sudo apt-get update
+            sudo apt-get install libfftw3-dev
+      - run:
+          name: install EQcorrscan
+          command: |
+            . venv/bin/activate
+            python setup.py install
+            python setup.py build_ext --inplace
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            . venv/bin/activate
+            py.test -m "network" -n 2
+            py.test eqcorrscan/doc/tutorials/*.rst eqcorrscan/doc/submodules/*.rst --cov-append
+
+      - run:
+          name: Upload to codecov
+          command: |
+            . venv/bin/activate
+            ls -a
+            codecov
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports

.travis.yml

@@ -15,6 +15,7 @@ matrix:
       env: PYTHON_VERSION=3.6
 
     - os: osx
+      osx_image: xcode8
       env:
         - PYTHON_VERSION=3.5
 
@@ -24,8 +25,9 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       export OS="MacOSX";
       export py=$PYTHON_VERSION;
-      export CC=gcc-4.9;
-      export CXX=g++-4.9;
+      brew install gcc6;
+      export CC=gcc-6;
+      export CXX=g++-6;
       brew install fftw;
     else
       export OS="Linux";
@@ -57,7 +59,7 @@ install:
   - conda create -q -n test-environment python=$PYTHON_VERSION colorama numpy scipy matplotlib obspy flake8 mock coverage bottleneck pyproj
   - source activate test-environment
   - conda install h5py
-  - pip install pep8-naming pytest pytest-cov pytest-pep8 pytest-xdist codecov Cython
+  - pip install pep8-naming pytest pytest-cov pytest-pep8 pytest-xdist pytest-rerunfailures codecov Cython
   - pip freeze
   - conda list
   # done installing dependencies
@@ -66,7 +68,9 @@ install:
   - python setup.py build_ext --inplace
 
 script:
-   - py.test --runslow
+   - py.test --runslow -m "not serial and not network" -n 2
+   - py.test -m "serial and not network" --cov-append
+#   - py.test -m "network" --cov-append -n 2
 
 after_success:
   # Check how much code is actually tested and send this report to codecov

CHANGES.md

@@ -1,3 +1,38 @@
+## Current
+* Added the ability to change the correlation functions used in detection
+  methods through the parameter xcorr_func of match_filter, Template.detect
+  and Tribe.detect, or using the set_xcorr context manager in
+  the utils.correlate module. Supported options are:
+    * numpy
+    * fftw
+    * time-domain
+    * or passing a function that implements the xcorr interface.
+* Added the ability to change the concurrency strategy of xcorr functions
+  using the paramter concurrency of match_filter, Template.detect
+  and Tribe.detect. Supported options are:
+    * None - for single-threaded execution in a single process
+    * multithread - for multi-threaded execution
+    * multiprocess- for multiprocess execution
+    * concurrent - allows functions to describe their own preferred currency
+    methods, defaults to multithread
+* Change debug printing output, it should be a little quieter;
+* Speed-up time-domain using a threaded C-routine - separate from frequency
+  domain C-routines;
+* Expose useful parallel options for all correlation routines;
+* Expose cores argument for match-filter objects to allow limits to be placed
+  on how much of your machine is used;
+* Limit number of workers created during pre-processing to never be more than
+  the number of traces in the stream being processed;
+* Implement openMP parallelisation of cross-correlation sum routines - memory
+  consumption reduced by using shared memory, and by computing the
+  cross-correlation sums rather than individual channel cross-correlations.
+  This also leads to a speed-up.  This routine is the default concurrent
+  correlation routine;
+* Test examples in rst doc files to ensure they are up-to-date;
+* Tests that were prone to timeout issues have been migrated to run on circleci
+  to allow quick re-starting of fails not due to code errors
+
+
 ## 0.2.5
 * Fix bug with \_group_process that resulted in stalled processes.
 * Force NumPy version
@@ -17,19 +52,19 @@ template;
 option (previously only used for debugging in development);
 * Increase test coverage in lag_calc;
 * Speed-up tests for brightness;
-* Increase test coverage for match_filter including testing io of 
+* Increase test coverage for match_filter including testing io of
 detections;
 * Increase subspace test coverage for edge cases;
 * Speed-up catalog_to_dd_tests;
 * Lag-calc will pick S-picks on channels ending E, N, 1 and 2, change
 from only picking on E and N before; warning added to docs;
 * Add full tests for pre-processing;
-* Run tests in parallel on ci, speed-up tests dramatically; 
-* Rename singular-value decomposition functions (with depreciation 
+* Run tests in parallel on ci, speed-up tests dramatically;
+* Rename singular-value decomposition functions (with depreciation
 warnings);
 * Rename SVD_moments to lower-case and add depreciation warning;
 * Increase test coverage in utils.mag_calc;
-* Add Template, Tribe, Family, Party objects and rename DETECTION to 
+* Add Template, Tribe, Family, Party objects and rename DETECTION to
 Detection;
     * Template objects maintain meta-data associated with their creation
     to stream-line processing of data (e.g. reduce chance of using the
@@ -45,10 +80,10 @@ Detection;
     * The Party object is a container for many Family objects.
     * Family objects are containers for detections from the same
     Template.
-    * Family and Party objects have a lag_calc method which computes 
+    * Family and Party objects have a lag_calc method which computes
     the cross-correlation pick-refinements.
     * The upshot of this is that it is possible to, in one line,
-    generate a Tribe of templates, compute their matched-filter 
+    generate a Tribe of templates, compute their matched-filter
     detections, and generate cross-correlation pick refinements, which
     output Event objects, which can be written to a catalog:
         Tribe.construct(method, **kwargs).detect(st, **kwargs).lag_calc(**kwargs).write()
@@ -81,33 +116,33 @@ fewer traces than template;
 
 ## 0.1.6
 * Fix bug introduced in version 0.1.5 for match_filter where looping
-through multiple templates did not correctly match image and template 
+through multiple templates did not correctly match image and template
 data: 0.1.5 fix did not work;
 * Bug-fix in catalog_to_dd for events without magnitudes;
 * Amend match-filter to not edit the list of template names in place.
-Previously, if a template was not used (due to no matching continuous 
-data) then the name of the template was removed: this now copies the 
+Previously, if a template was not used (due to no matching continuous
+data) then the name of the template was removed: this now copies the
 list of template_names internally and does not change the external list.
 
 ## 0.1.5
 * Migrate coverage to codecov;
 * Fix bug introduced in version 0.1.5 for match_filter where looping
-through multiple templates did not correctly match image and template 
+through multiple templates did not correctly match image and template
 data.
 
 ## 0.1.4
 * Bug-fix in plot_repicked removed where data were not normalized
 properly;
-* Bug-fix in lag_calc where data were missing in the continuous data 
+* Bug-fix in lag_calc where data were missing in the continuous data
 fixed (this led to incorrect picks, **major bug!**);
 * Output cross-channel correlation sum in lag-calc output;
 * Add id to DETECTION objects, which is consistent with the events
 within DETECTION objects and catalog output, and used in lag_calc to
 allow linking of detections to catalog events;
-* Add lots of logging and error messages to lag-calc to ensure user 
+* Add lots of logging and error messages to lag-calc to ensure user
 understands limits;
 * Add error to day-proc to ensure user is aware of risks of padding;
-* Change utils.pre_processing.process to accept different length of 
+* Change utils.pre_processing.process to accept different length of
 data enforcement, not just full day (allow for overlap in processing,
 which might be useful for reducing day start and end effects);
 * Bug-fix in mag_calc.amp_pick_event, broke loop if data were missing;

CONTRIBUTORS.md

@@ -1,5 +1,7 @@
-Calum John Chamberlain
-Chet Hopp
-Emily Warren-Smith
-Konstantinos Michailos
-Shanna Chu
+* Calum John Chamberlain
+* Chet Hopp
+* Emily Warren-Smith
+* Konstantinos Michailos
+* Shanna Chu
+* Derrick Chambers
+* Chris Scott

MANIFEST.in

@@ -5,6 +5,7 @@ include README.md
 include LICENCE.txt
 include CHANGES.md
 include CONTRIBUTORS.md
+include requirements.txt
 # include eqcorrscan/test_data/tutorial_data.tgz
 # include eqcorrscan/tutorial.py
 recursive-include eqcorrscan/utils *.py
@@ -16,6 +17,7 @@ recursive-include eqcorrscan/core *.py
 recursive-include eqcorrscan/tests *.py
 include eqcorrscan/lib/libutils.def
 include eqcorrscan/lib/multi_corr.c
+include eqcorrscan/lib/time_corr.c
 
 # exclude rules
 # global-exclude *.pyc *.ms *.tgz

README.md

@@ -1,14 +1,26 @@
 # EQcorrscan
 ## A python package for the detection and analysis of repeating and near-repeating earthquakes.
 
-[![Join the chat at https://gitter.im/eqcorrscan/EQcorrscan](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/eqcorrscan/EQcorrscan?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![TravisCIStatus](https://travis-ci.org/eqcorrscan/EQcorrscan.svg?branch=master)](https://travis-ci.org/eqcorrscan/EQcorrscan)
-[![Build status](https://ci.appveyor.com/api/projects/status/b0924mp0uwwyap3d/branch/master?svg=true)](https://ci.appveyor.com/project/calum-chamberlain/eqcorrscan-jsycv/branch/master)
-[![codecov](https://codecov.io/gh/eqcorrscan/EQcorrscan/branch/master/graph/badge.svg)](https://codecov.io/gh/eqcorrscan/EQcorrscan)
-[![DOI](https://zenodo.org/badge/35918157.svg)](https://zenodo.org/badge/latestdoi/35918157)
-[![DocumentationStatus](http://readthedocs.org/projects/eqcorrscan/badge/?version=latest)](http://eqcorrscan.readthedocs.org/en/latest/?badge=latest)
-[![Dependency Status](https://dependencyci.com/github/eqcorrscan/EQcorrscan/badge)](https://dependencyci.com/github/eqcorrscan/EQcorrscan)
-[![Stories in Ready](https://badge.waffle.io/eqcorrscan/EQcorrscan.png?label=ready&title=Ready)](http://waffle.io/eqcorrscan/EQcorrscan)
+# Citation:
+We have a manuscript in review, if you make use of EQcorrscan please cite the folloing paper:
+
+Chamberlain, C. J., Hopp, C. J., Boese, C. M., Warren-Smith, E., Chambers, D., Chu, S. X., Michailos, K., Townend, J., EQcorrscan: Repeating and near-repeating earthquake detection and analysis in Python. Seismological Research Letters *in review*
+
+If you want to you should also cite the version number: [![DOI](https://zenodo.org/badge/35918157.svg)](https://zenodo.org/badge/latestdoi/35918157)
+
+# Test status
+Note that tests for travis and appveyor are run daily on master as cron jobs, and may reflect time-out issues.
+
+| Service tests | Badge |
+|---------------|-------|
+| OSX & Linux | [![TravisCIStatus](https://travis-ci.org/eqcorrscan/EQcorrscan.svg?branch=master)](https://travis-ci.org/eqcorrscan/EQcorrscan)
+| Windows | [![Build status](https://ci.appveyor.com/api/projects/status/b0924mp0uwwyap3d/branch/master?svg=true)](https://ci.appveyor.com/project/calum-chamberlain/eqcorrscan-jsycv/branch/master)
+| Code coverage | [![codecov](https://codecov.io/gh/eqcorrscan/EQcorrscan/branch/master/graph/badge.svg)](https://codecov.io/gh/eqcorrscan/EQcorrscan)
+| Documentation | [![DocumentationStatus](http://readthedocs.org/projects/eqcorrscan/badge/?version=latest)](http://eqcorrscan.readthedocs.org/en/latest/?badge=latest)
+| Dependency status | [![Dependency Status](https://dependencyci.com/github/eqcorrscan/EQcorrscan/badge)](https://dependencyci.com/github/eqcorrscan/EQcorrscan)
+| Network tests | [![CircleCI](https://circleci.com/gh/eqcorrscan/EQcorrscan/tree/master.svg?style=svg)](https://circleci.com/gh/eqcorrscan/EQcorrscan/tree/master)
+| Issues ready | [![Stories in Ready](https://badge.waffle.io/eqcorrscan/EQcorrscan.png?label=ready&title=Ready)](http://waffle.io/eqcorrscan/EQcorrscan)
+| Chat on gitter | [![Join the chat at https://gitter.im/eqcorrscan/EQcorrscan](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/eqcorrscan/EQcorrscan?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 # Installation
 Installation has been tested on both OSX and Linux (Ubuntu), and
@@ -17,6 +29,9 @@ Note that, although we support Windows, EQcorrscan is optimized for
 linux style distributions, and the developers are not extensive Windows
 users.
 
+*OSX with gcc-4.9 from homebrew doesn't appear to compile properly, all other gcc versions
+ seem to work*
+
 Instructions for installing EQcorrscan and the required dependency, fftw
 are linked from the
 [docs](http://eqcorrscan.readthedocs.io/en/latest/intro.html#installation)
@@ -79,9 +94,7 @@ LGPL GNU License, Copyright EQcorrscan developers 2015, 2016, 2017.
 # Contributing
 
 Please fork this project and work on it there then create a pull request to
-merge back to this main repository.  If you are working on a bug-fix then
-use the *develop* branch, otherwise, create a feature branch and work
-on your addition there.
+merge back to this main repository.  Please create a branch from *develop*.
 
 When you make changes please run the tests in the test directory to ensure
 everything merges with minimum effort.  If there is not yet a test to cope

appveyor.yml

@@ -54,7 +54,7 @@ install:
 
   # Install other pip dependancies
   - "pip install -U future"
-  - "pip install pytest pytest-pep8 pytest-cov pytest-xdist codecov"
+  - "pip install pytest pytest-pep8 pytest-cov pytest-xdist pytest-rerunfailures codecov"
   # list package versions
   - "conda list"
   # Copy the windows fftw header file
@@ -66,11 +66,13 @@ build: false
 test_script:
   - "%CMD_IN_ENV% python setup.py build_ext --inplace"
   - "%CMD_IN_ENV% python setup.py install"
-  - "%CMD_IN_ENV% py.test -n4"
+  - "%CMD_IN_ENV% py.test -n4 -m \"not serial and not network\""
+  - "%CMD_IN_ENV% py.test -m \"serial and not network\" --cov-append"
+#  - "%CMD_IN_ENV% py.test -n4 -m \"network\" --cov-append"
 
 after_test:
   #  - "coverage combine"
-  - "powershell copy-item .coverage ..\\.coverage.empty"
-  - "cd .."
-  - "coverage combine"
+ #  - "powershell copy-item .coverage ..\\.coverage.empty"
+ #  - "cd .."
+ #  - "coverage combine"
   - "codecov"