Merge pull request #134 from eqcorrscan/develop

Prep for patch 0.2.5

.travis.yml

@@ -14,15 +14,9 @@ matrix:
     - os: linux
       env: PYTHON_VERSION=3.6
 
-    - os: osx
-      env:
-        - PYTHON_VERSION=2.7
     - os: osx
       env:
         - PYTHON_VERSION=3.5
-    - os: osx
-      env:
-        - PYTHON_VERSION=3.6
 
 sudo: false
 

CHANGES.md

@@ -1,6 +1,13 @@
-## Current
-* Fix bug with \_group_process
+## 0.2.5
+* Fix bug with \_group_process that resulted in stalled processes.
 * Force NumPy version
+* Support indexing of Tribe and Party objects by template-name.
+* Add tests for lag-calc issue with preparing data
+* Change internals of *eqcorrscan.core.lag_calc._prepare_data* to use a
+  dictionary for delays, and to work correctly! Issues arose from not checking
+  for masked data properly and not checking length properly.
+* Fix bug in match_filter.match_filter when checking for equal length traces,
+  length count was one sample too short.
 
 ## 0.2.4
 * Increase test coverage (edge-cases) in template_gen;

codecov.yml

@@ -36,4 +36,6 @@ parsers:
       macro: false
       method: false
   javascript:
-    enable_partials: false
+    enable_partials: false
+ignore:
+  eqcorrscan-jsyvc

eqcorrscan/__init__.py

@@ -14,7 +14,7 @@
 __all__ = ['core', 'utils', 'tutorials']
 
 
-__version__ = '0.2.4'
+__version__ = '0.2.5'
 
 
 # Cope with changes to name-space to remove most of the camel-case

eqcorrscan/core/lag_calc.py

@@ -173,8 +173,7 @@ def _channel_loop(detection, template, min_cc, detection_id, interpolate, i,
                 print('Template is %i long' % len(tr.data))
                 continue
             try:
-                shift, cc_max = _xcorr_interp(ccc=ccc,
-                                              dt=image[0].stats.delta)
+                shift, cc_max = _xcorr_interp(ccc=ccc, dt=image[0].stats.delta)
             except IndexError:
                 print('Could not interpolate ccc, not smooth')
                 ccc = normxcorr2(tr.data, image[0].data)
@@ -192,8 +191,8 @@ def _channel_loop(detection, template, min_cc, detection_id, interpolate, i,
                 print('Template is %i long' % len(tr.data))
                 continue
             cc_max = np.amax(ccc)
-            picktime = image[0].stats.starttime + (np.argmax(ccc) *
-                                                   image[0].stats.delta)
+            picktime = image[0].stats.starttime + (
+                np.argmax(ccc) * image[0].stats.delta)
         if debug > 3:
             print('Maximum cross-corr=%s' % cc_max)
         checksum += cc_max
@@ -235,9 +234,10 @@ def _channel_loop(detection, template, min_cc, detection_id, interpolate, i,
     ccc_str = ("detect_val=%s" % cccsum)
     event.comments.append(Comment(text=ccc_str))
     if used_chans == detect_chans:
-        if pre_lag_ccsum is not None and checksum - pre_lag_ccsum < -0.05:
+        if pre_lag_ccsum is not None and\
+           checksum - pre_lag_ccsum < -(0.30 * pre_lag_ccsum):
             msg = ('lag-calc has decreased cccsum from %f to %f - '
-                   'report this error' % (pre_lag_ccsum, checksum))
+                   % (pre_lag_ccsum, checksum))
             # warnings.warn(msg)
             raise LagCalcError(msg)
     else:
@@ -288,6 +288,8 @@ def _day_loop(detection_streams, template, min_cc, detections,
         this template.
     :rtype: :class:`obspy.core.event.Catalog`
     """
+    if len(detection_streams) == 0:
+        return Catalog()
     if not cores:
         num_cores = cpu_count()
     else:
@@ -323,14 +325,13 @@ def _day_loop(detection_streams, template, min_cc, detections,
                 pre_lag_ccsum=detections[i].detect_val,
                 detect_chans=detections[i].no_chans,
                 horizontal_chans=horizontal_chans,
-                vertical_chans=vertical_chans,
-                debug=debug))
+                vertical_chans=vertical_chans, debug=debug))
     temp_catalog = Catalog()
     temp_catalog.events = [event_tup[1] for event_tup in events_list]
     return temp_catalog
 
 
-def _prepare_data(detect_data, detections, zipped_templates, delays,
+def _prepare_data(detect_data, detections, template, delays,
                   shift_len, plot):
     """
     Prepare data for lag_calc - reduce memory here.
@@ -341,12 +342,11 @@ def _prepare_data(detect_data, detections, zipped_templates, delays,
     :param detections:
         List of :class:`eqcorrscan.core.match_filter.Detection` to get
         data for.
-    :type zipped_templates: zip
-    :param zipped_templates: Zipped list of (template_name, template)
+    :type template: tuple
+    :param template: tuple of (template_name, template)
     :type delays: list
     :param delays:
-        List of lists of the delays for each template in the form:
-        [(template_name, [(station, channel, delay)])]
+        Dictionary of delay times in seconds keyed by sta.channel.
     :type shift_len: float
     :param shift_len: Shift length in seconds allowed for picking.
     :type plot: bool
@@ -358,61 +358,49 @@ def _prepare_data(detect_data, detections, zipped_templates, delays,
     """
     detect_streams = []
     for detection in detections:
+        if detection.template_name != template[0]:
+            continue
         # Stream to be saved for new detection
         detect_stream = []
         max_delay = 0
-        template_st = [t for t in zipped_templates
-                       if str(t[0]) == str(detection.template_name)]
-        if len(template_st) > 0:
-            template_st = template_st[0]
-        else:
-            warnings.warn('No template with name: %s' %
-                          detection.template_name)
-            for t in zipped_templates:
-                print(t)
-            continue
         for tr in detect_data:
-            tr_copy = tr.copy()
-            # Right now, copying each trace hundreds of times...
-            template = template_st[1].select(station=tr.stats.station,
-                                             channel=tr.stats.channel)
-            if template:
+            template_tr = template[1].select(
+                station=tr.stats.station, channel=tr.stats.channel)
+            if len(template_tr) >= 1:
                 # Save template trace length in seconds
-                template_len = len(template[0]) / \
-                    template[0].stats.sampling_rate
+                template_len = (
+                    len(template_tr[0]) / template_tr[0].stats.sampling_rate)
             else:
                 continue
                 # If there is no template-data match then skip the rest
                 # of the trace loop.
             # Grab the delays for the desired template: [(sta, chan, delay)]
-            delay = []
-            for d in delays:
-                if d[0] == detection.template_name:
-                    delay.append(d)
-            delay = delay[0][1]
             # Now grab the delay for the desired trace for this template
-            delay = [d for d in delay if d[0] == tr.stats.station and
-                     d[1] == tr.stats.channel][0][2]
+            delay = delays[tr.stats.station + '.' + tr.stats.channel]
             if delay > max_delay:
                 max_delay = delay
-            detect_stream.append(tr_copy.trim(starttime=detection.detect_time -
-                                              shift_len + delay,
-                                              endtime=detection.detect_time +
-                                              delay + shift_len +
-                                              template_len))
-            del tr_copy
+            detect_stream.append(tr.slice(
+                starttime=detection.detect_time - shift_len + delay,
+                endtime=detection.detect_time + delay + shift_len +
+                template_len).copy())
         for tr in detect_stream:
             if len(tr.data) == 0:
                 msg = ('No data in %s.%s for detection at time %s' %
                        (tr.stats.station, tr.stats.channel,
                         detection.detect_time))
                 warnings.warn(msg)
                 detect_stream.remove(tr)
-            elif tr.stats.endtime - tr.stats.starttime < template_len:
+            elif tr.stats.endtime - tr.stats.starttime < (
+                        2 * shift_len) + template_len:
                 msg = ("Insufficient data for %s.%s will not use."
                        % (tr.stats.station, tr.stats.channel))
                 warnings.warn(msg)
                 detect_stream.remove(tr)
+            elif np.ma.is_masked(tr.data):
+                msg = ("Masked data found for %s.%s, will not use."
+                       % (tr.stats.station, tr.stats.channel))
+                warnings.warn(msg)
+                detect_stream.remove(tr)
         # Check for duplicate traces
         stachans = [(tr.stats.station, tr.stats.channel)
                     for tr in detect_stream]
@@ -423,9 +411,10 @@ def _prepare_data(detect_data, detections, zipped_templates, delays,
                        % (key[0], key[1]))
                 raise LagCalcError(msg)
         if plot:
-            background = detect_data.copy().trim(
+            background = detect_data.slice(
                 starttime=detection.detect_time - (shift_len + 5),
-                endtime=detection.detect_time + shift_len + max_delay + 7)
+                endtime=detection.detect_time +
+                shift_len + max_delay + 7).copy()
             for tr in background:
                 if len(tr.data) == 0:
                     background.remove(tr)
@@ -434,6 +423,8 @@ def _prepare_data(detect_data, detections, zipped_templates, delays,
                 times=[detection.detect_time - shift_len],
                 title='Detection Extracted')
         if not len(detect_stream) == 0:
+            detect_stream = Stream(detect_stream).split()
+            # Make sure there are no masks left over.
             # Create tuple of (template name, data stream)
             detect_streams.append((detection.template_name,
                                    Stream(detect_stream)))
@@ -566,16 +557,17 @@ def lag_calc(detections, detect_data, template_names, templates,
     detect_stachans = [(tr.stats.station, tr.stats.channel)
                        for tr in detect_data]
     for template in zipped_templates:
-        temp_delays = []
+        temp_delays = {}
         # Remove channels not present in continuous data
         _template = template[1].copy()
         for tr in _template:
             if (tr.stats.station, tr.stats.channel) not in detect_stachans:
                 _template.remove(tr)
         for tr in _template:
-            temp_delays.append((tr.stats.station, tr.stats.channel,
-                                tr.stats.starttime - _template.
-                                sort(['starttime'])[0].stats.starttime))
+            temp_delays.update(
+                {tr.stats.station + '.' + tr.stats.channel:
+                 tr.stats.starttime -
+                 _template.sort(['starttime'])[0].stats.starttime})
         delays.append((template[0], temp_delays))
         del _template
     # Segregate detections by template, then feed to day_loop
@@ -584,22 +576,21 @@ def lag_calc(detections, detect_data, template_names, templates,
         print('Running lag-calc for template %s' % template[0])
         template_detections = [detection for detection in detections
                                if detection.template_name == template[0]]
+        t_delays = [d for d in delays if d[0] == template[0]][0][1]
         if debug > 2:
             print('There are %i detections' % len(template_detections))
-        detect_streams = _prepare_data(detect_data=detect_data,
-                                       detections=template_detections,
-                                       zipped_templates=zipped_templates,
-                                       delays=delays, shift_len=shift_len,
-                                       plot=prep_plot)
+        detect_streams = _prepare_data(
+            detect_data=detect_data, detections=template_detections,
+            template=template, delays=t_delays, shift_len=shift_len,
+            plot=prep_plot)
         detect_streams = [detect_stream[1] for detect_stream in detect_streams]
         if len(template_detections) > 0:
-            template_cat = _day_loop(detection_streams=detect_streams,
-                                     template=template[1], min_cc=min_cc,
-                                     detections=template_detections,
-                                     horizontal_chans=horizontal_chans,
-                                     vertical_chans=vertical_chans,
-                                     interpolate=interpolate, cores=cores,
-                                     parallel=parallel, debug=debug)
+            template_cat = _day_loop(
+                detection_streams=detect_streams, template=template[1],
+                min_cc=min_cc, detections=template_detections,
+                horizontal_chans=horizontal_chans,
+                vertical_chans=vertical_chans, interpolate=interpolate,
+                cores=cores, parallel=parallel, debug=debug)
             initial_cat += template_cat
             if plot:
                 for i, event in enumerate(template_cat):