begin sbe parser (#20)

- add simple sbe39 parser w/wo pressure - initial test start
NOAA-PMEL · Mar 26, 2021 · d8eda3b · d8eda3b
1 parent 8238754
commit d8eda3b
Show file tree

Hide file tree

Showing 8 changed files with 574 additions and 5 deletions.
diff --git a/examples/16bsm2a_wpak_0000m.nc b/examples/16bsm2a_wpak_0000m.nc
diff --git a/examples/mtrduino_process.py b/examples/mtrduino_process.py
@@ -0,0 +1,26 @@
+"""
+MTR 5K (Arduino) Example
+========================
+
+Demonstrate how to use the routines to process downloaded mtr data.
+
+Only for the 5k Arduino Series
+"""
+import yaml
+
+import EcoFOCIpy.io.mtr_parser as mtr_parser
+import EcoFOCIpy.io.ncCFsave as ncCFsave
+import EcoFOCIpy.metaconfig.load_config as load_config
+
+###############################################################
+# edit to point to wpak raw datafile (arg parse?)
+datafile = '../staticdata/'
+instrument = ''
+mooring_meta_file = '../staticdata/mooring_example.yaml'
+inst_meta_file = '../staticdata/instr_metaconfig/mtr.yaml'
+inst_shortname = 'mtr'
+###############################################################
+
+#init and load data
+mtr = mtr_parser.mtrduino()
+mtr_data = mtr.parse(filename=datafile,datetime_index=True) 
diff --git a/examples/sbe39_process.py b/examples/sbe39_process.py
@@ -0,0 +1,29 @@
+"""
+SBE39 Example
+=============
+
+Demonstrate how to use the routines to process downloaded sbe39 cnv data
+
+* with pressure and without pressure
+
+"""
+import yaml
+
+import EcoFOCIpy.io.sbe_parser as sbe_parser
+import EcoFOCIpy.io.ncCFsave as ncCFsave
+import EcoFOCIpy.metaconfig.load_config as load_config
+
+###############################################################
+# edit to point to wpak raw datafile (arg parse?)
+datafile = '../staticdata/sbe39_wopress.cnv'
+instrument = 'SBE-39 1777'
+mooring_meta_file = '../staticdata/mooring_example.yaml'
+inst_meta_file = '../staticdata/instr_metaconfig/sbe39_cf.yaml'
+inst_shortname = 's39'
+###############################################################
+
+#init and load data
+sbe39_wop = sbe_parser.sbe39()
+(sbe39_wop_header,sbe39_wop_data) = sbe39_wop.parse(filename=datafile,
+                                                    return_header=True,
+                                                    datetime_index=True) 
diff --git a/src/EcoFOCIpy/io/mtr_parser.py b/src/EcoFOCIpy/io/mtr_parser.py
@@ -2,7 +2,44 @@
 
 These include:
 
-* Version 3/4 (old version)
-* Version 5 (MTRduino)
+* Version 3/4 (old version) [ ]
+* Version 5 (MTRduino) [x]
 
 """
+import pandas as pd
+
+class mtrduino(object):
+    r""" MicroTemperature Recorders (MTR) - 5k / MTRduino generation
+    Collection of static methods to define MTR processing and conversion
+
+    It is assumed the data passed here is preliminarily processed and has calibration
+    functions already applied
+
+    ToDO:  Allow raw data to be passed"""
+
+    @staticmethod
+    def parse(filename=None, datetime_index=True, **kwargs):
+        r"""
+        Basic Method to open and read mtrduino raw converted csv files
+            
+            kwargs:
+                round_10min_interval - force small deviations to 10min intervals.  Actually rounds all so it
+                    is user responisbility to make sure values are representative first.
+
+        """
+        assert filename != None , 'Must provide a datafile'
+
+        rawdata = pd.read_csv(
+            filename, delimiter=",", parse_dates=["date_time"]
+        )
+
+        if datetime_index:
+            rawdata_df = rawdata_df.set_index(pd.DatetimeIndex(rawdata_df['date_time'])).drop(['date_time'],axis=1)
+
+        #previous algorithms rounded to 10min
+        ## all time manipulation should either be outside the instrument methods
+        ##  or optional via a method call... maybe not a flag like here??
+        if kwargs["round_10min_interval"]:
+            rawdata.index = rawdata.index.round("10min", inplace=True)
+
+        return rawdata_df
diff --git a/src/EcoFOCIpy/io/sbe_parser.py b/src/EcoFOCIpy/io/sbe_parser.py
@@ -6,4 +6,60 @@
 Moored SBE (cnv files):
 * 16,19,26,37,39,56
 
-"""
+"""
+import pandas as pd
+import sys
+
+
+class sbe39(object):
+    r""" Seabird 39 Temperature (with optional pressure)
+        
+        Basic Method to open files.  Specific actions can be passes as kwargs for instruments
+
+        With or without header
+
+        With or without pressure - assume 4 cols = w/press, 3 cols = w/o press
+
+    """
+
+    @staticmethod
+    def parse(filename=None, return_header=True, datetime_index=True):
+        r"""
+        Basic Method to open and read sbe39 csv files
+            
+            kwargs
+            truncate_seconds : boolean (truncates down to nearest minute)
+
+        """
+        assert filename != None , 'Must provide a datafile'
+
+        header = []
+
+        with open(filename) as fobj:
+            for k, line in enumerate(fobj.readlines()):
+                header = header + [line]
+                if "*END*" in line:
+                    headercount=k+4
+                    break
+
+
+        rawdata_df = pd.read_csv(filename, 
+                        delimiter=",", 
+                        parse_dates=True, 
+                        header=None, 
+                        skiprows=headercount)
+
+        if len(rawdata_df.columns) == 4:
+            rawdata_df.columns = ['temperature','pressure','date','time']
+        elif len(rawdata_df.columns) == 3:
+            rawdata_df.columns = ['temperature','date','time']
+        else:
+            sys.exit('Unknown number of columns in raw data')
+
+        rawdata_df["date_time"] = pd.to_datetime(rawdata_df["date"] + " " + rawdata_df["time"], format="%d %b %Y %H:%M:%S")
+
+        if datetime_index:
+            rawdata_df = rawdata_df.set_index(pd.DatetimeIndex(rawdata_df['date_time'])).drop(['date_time','DATE','TIME'],axis=1)        
+
+        return (rawdata_df,header)
+
diff --git a/staticdata/instr_metaconfig/sbe39_cf.yaml b/staticdata/instr_metaconfig/sbe39_cf.yaml
@@ -43,13 +43,13 @@ timeseries_id:
   long_name: "timeseries id"
   standard_name: ''
 #variables
-temp:
+temperature:
   epic_key: T_20
   generic_name: temp
   long_name: "Sea temperature in-situ ITS-90 scale"
   standard_name: sea_water_temperature
   units: degree_C
-pres: 
+pressure: 
   epic_key: P_1
   generic_name: pres
   units: decibar

diff --git a/staticdata/sbe39_wopress.cnv b/staticdata/sbe39_wopress.cnv
@@ -0,0 +1,204 @@
+* Sea-Bird SBE39 Data File:
+* FileName = C:\Users\strausz.PMEL\Desktop\2016\bering_downloads\16bsm2a_sbe39_1777_21m.asc
+* Software Version 1.59
+* Temperature SN = 
+* Conductivity SN = 
+* System UpLoad Time = Oct 11 2016 21:04:04
+** real time:  205610
+** inst time: 205631
+* ds
+* SBE 39 V 2.2   SERIAL NO. 1777    11 Oct 2016  21:03:12
+* battery voltage = 8.6
+* not logging: received stop command
+* sample interval = 600 seconds
+* samplenumber = 23023, free = 576163
+* serial sync mode disabled
+* real-time output disabled
+* SBE 39 configuration = temperature only
+* binary upload does not include time
+* temperature = 21.23 deg C
+
+
+* S>
+* SBE39 V 2.2   01777
+* temperature:  02-dec-15
+*     TA0 = 7.411042e-05
+*     TA1 = 2.721709e-04
+*     TA2 = -2.270258e-06
+*     TA3 = 1.482924e-07
+
+
+* S> 
+*END*
+start time =  05 May 2016  00:00:00
+sample interval = 600 seconds
+start sample number = 1
+  8.2290, 05 May 2016, 00:00:00
+  8.1297, 05 May 2016, 00:10:00
+  8.1061, 05 May 2016, 00:20:00
+  8.2852, 05 May 2016, 00:30:00
+  8.4766, 05 May 2016, 00:40:00
+  9.0763, 05 May 2016, 00:50:00
+  8.9149, 05 May 2016, 01:00:00
+  9.0934, 05 May 2016, 01:10:00
+ 11.9529, 05 May 2016, 01:20:00
+ 12.4361, 05 May 2016, 01:30:00
+ 14.1294, 05 May 2016, 01:40:00
+ 12.8875, 05 May 2016, 01:50:00
+ 12.5928, 05 May 2016, 02:00:00
+ 12.9684, 05 May 2016, 02:10:00
+ 12.2063, 05 May 2016, 02:20:00
+ 12.3812, 05 May 2016, 02:30:00
+  7.6283, 05 May 2016, 02:40:00
+  7.3519, 05 May 2016, 02:50:00
+  6.7852, 05 May 2016, 03:00:00
+  7.0439, 05 May 2016, 03:10:00
+  6.8104, 05 May 2016, 03:20:00
+  6.6317, 05 May 2016, 03:30:00
+  7.6724, 05 May 2016, 03:40:00
+  8.3639, 05 May 2016, 03:50:00
+  5.3658, 05 May 2016, 04:00:00
+  5.3264, 05 May 2016, 04:10:00
+  5.3426, 05 May 2016, 04:20:00
+  5.3460, 05 May 2016, 04:30:00
+  5.4469, 05 May 2016, 04:40:00
+  5.2268, 05 May 2016, 04:50:00
+  5.0634, 05 May 2016, 05:00:00
+  5.0644, 05 May 2016, 05:10:00
+  5.1364, 05 May 2016, 05:20:00
+  5.2876, 05 May 2016, 05:30:00
+  5.0066, 05 May 2016, 05:40:00
+  5.0930, 05 May 2016, 05:50:00
+  5.0735, 05 May 2016, 06:00:00
+  5.1180, 05 May 2016, 06:10:00
+  5.0682, 05 May 2016, 06:20:00
+  5.1336, 05 May 2016, 06:30:00
+  5.1346, 05 May 2016, 06:40:00
+  5.1201, 05 May 2016, 06:50:00
+  5.1270, 05 May 2016, 07:00:00
+  5.1138, 05 May 2016, 07:10:00
+  5.0896, 05 May 2016, 07:20:00
+  5.0745, 05 May 2016, 07:30:00
+  5.1172, 05 May 2016, 07:40:00
+  5.1034, 05 May 2016, 07:50:00
+  5.1142, 05 May 2016, 08:00:00
+  5.1161, 05 May 2016, 08:10:00
+  5.1209, 05 May 2016, 08:20:00
+  5.1309, 05 May 2016, 08:30:00
+  5.1290, 05 May 2016, 08:40:00
+  5.1366, 05 May 2016, 08:50:00
+  5.1373, 05 May 2016, 09:00:00
+  5.1230, 05 May 2016, 09:10:00
+  5.0782, 05 May 2016, 09:20:00
+  5.0161, 05 May 2016, 09:30:00
+  4.9348, 05 May 2016, 09:40:00
+  4.8258, 05 May 2016, 09:50:00
+  4.8074, 05 May 2016, 10:00:00
+  4.8105, 05 May 2016, 10:10:00
+  4.8094, 05 May 2016, 10:20:00
+  4.8646, 05 May 2016, 10:30:00
+  4.9037, 05 May 2016, 10:40:00
+  4.9425, 05 May 2016, 10:50:00
+  5.0708, 05 May 2016, 11:00:00
+  5.0418, 05 May 2016, 11:10:00
+  5.0295, 05 May 2016, 11:20:00
+  5.0380, 05 May 2016, 11:30:00
+  5.1976, 05 May 2016, 11:40:00
+  5.2284, 05 May 2016, 11:50:00
+  5.1848, 05 May 2016, 12:00:00
+  5.1916, 05 May 2016, 12:10:00
+  5.1456, 05 May 2016, 12:20:00
+  5.1802, 05 May 2016, 12:30:00
+  5.2093, 05 May 2016, 12:40:00
+  5.1927, 05 May 2016, 12:50:00
+  5.1023, 05 May 2016, 13:00:00
+  5.1061, 05 May 2016, 13:10:00
+  5.1130, 05 May 2016, 13:20:00
+  5.0168, 05 May 2016, 13:30:00
+  4.9461, 05 May 2016, 13:40:00
+  4.7668, 05 May 2016, 13:50:00
+  4.8515, 05 May 2016, 14:00:00
+  4.7696, 05 May 2016, 14:10:00
+  4.7733, 05 May 2016, 14:20:00
+  4.7788, 05 May 2016, 14:30:00
+  4.7932, 05 May 2016, 14:40:00
+  4.8262, 05 May 2016, 14:50:00
+  4.8414, 05 May 2016, 15:00:00
+  4.9209, 05 May 2016, 15:10:00
+  5.1120, 05 May 2016, 15:20:00
+  5.2432, 05 May 2016, 15:30:00
+  5.2809, 05 May 2016, 15:40:00
+  5.2804, 05 May 2016, 15:50:00
+  5.3435, 05 May 2016, 16:00:00
+  5.3117, 05 May 2016, 16:10:00
+  5.3559, 05 May 2016, 16:20:00
+  5.3697, 05 May 2016, 16:30:00
+  5.3467, 05 May 2016, 16:40:00
+  5.3232, 05 May 2016, 16:50:00
+  5.3261, 05 May 2016, 17:00:00
+  5.3612, 05 May 2016, 17:10:00
+  5.3945, 05 May 2016, 17:20:00
+  5.3797, 05 May 2016, 17:30:00
+  5.4021, 05 May 2016, 17:40:00
+  5.3904, 05 May 2016, 17:50:00
+  5.3910, 05 May 2016, 18:00:00
+  5.3881, 05 May 2016, 18:10:00
+  5.3327, 05 May 2016, 18:20:00
+  5.3717, 05 May 2016, 18:30:00
+  5.3322, 05 May 2016, 18:40:00
+  5.3522, 05 May 2016, 18:50:00
+  5.3075, 05 May 2016, 19:00:00
+  5.3045, 05 May 2016, 19:10:00
+  5.2856, 05 May 2016, 19:20:00
+  5.3156, 05 May 2016, 19:30:00
+  5.3033, 05 May 2016, 19:40:00
+  5.3147, 05 May 2016, 19:50:00
+  5.3569, 05 May 2016, 20:00:00
+  5.1978, 05 May 2016, 20:10:00
+  5.2093, 05 May 2016, 20:20:00
+  5.3312, 05 May 2016, 20:30:00
+  5.4224, 05 May 2016, 20:40:00
+  5.4355, 05 May 2016, 20:50:00
+  5.4800, 05 May 2016, 21:00:00
+  5.4778, 05 May 2016, 21:10:00
+  5.4599, 05 May 2016, 21:20:00
+  5.4757, 05 May 2016, 21:30:00
+  5.4571, 05 May 2016, 21:40:00
+  5.5096, 05 May 2016, 21:50:00
+  5.4864, 05 May 2016, 22:00:00
+  5.4631, 05 May 2016, 22:10:00
+  5.4860, 05 May 2016, 22:20:00
+  5.4740, 05 May 2016, 22:30:00
+  5.5130, 05 May 2016, 22:40:00
+  5.4630, 05 May 2016, 22:50:00
+  5.4787, 05 May 2016, 23:00:00
+  5.4865, 05 May 2016, 23:10:00
+  5.4439, 05 May 2016, 23:20:00
+  5.4256, 05 May 2016, 23:30:00
+  5.4053, 05 May 2016, 23:40:00
+  5.3565, 05 May 2016, 23:50:00
+  5.3449, 06 May 2016, 00:00:00
+  5.2386, 06 May 2016, 00:10:00
+  5.2379, 06 May 2016, 00:20:00
+  5.1892, 06 May 2016, 00:30:00
+  5.2889, 06 May 2016, 00:40:00
+  5.4054, 06 May 2016, 00:50:00
+  5.4813, 06 May 2016, 01:00:00
+  5.4264, 06 May 2016, 01:10:00
+  5.2134, 06 May 2016, 01:20:00
+  5.3266, 06 May 2016, 01:30:00
+  5.4210, 06 May 2016, 01:40:00
+  5.5686, 06 May 2016, 01:50:00
+  5.5257, 06 May 2016, 02:00:00
+  5.3876, 06 May 2016, 02:10:00
+  5.3468, 06 May 2016, 02:20:00
+  5.5345, 06 May 2016, 02:30:00
+  5.1555, 06 May 2016, 02:40:00
+  5.5858, 06 May 2016, 02:50:00
+  5.3153, 06 May 2016, 03:00:00
+  5.4245, 06 May 2016, 03:10:00
+  5.4083, 06 May 2016, 03:20:00
+  5.5367, 06 May 2016, 03:30:00
+  5.3333, 06 May 2016, 03:40:00
+  5.2527, 06 May 2016, 03:50:00
+  5.2081, 06 May 2016, 04:00:00