updated tests and routes (#10)

README.md

@@ -15,11 +15,11 @@ Installation and Launch
 ------------------------
 - Docker Container:
 The epidata-community docker image is available as a stand-alone package with all code and required components. To build and start a docker container, simply execute the command shown below (replacing 'epidata123' with a custom token):  
-    - docker run -p 443:443 -it -e token=epidata123 epidataio/epidata-community:0.10.0
+    - docker run -p 443:443 -it -e token=epidata123 epidataio/epidata-community:0.11.0
 
     Below are other useful docker commands:
     - Pull the docker image: 
-        - docker pull epidataio/epidata-community:0.10.0
+        - docker pull epidataio/epidata-community:0.11.0
     - List all docker containers:
         - docker ps -a
     - Stop a docker container:

ipython/epidata/analytics.py

@@ -129,7 +129,15 @@ def substitute(df, meas_names, method="rolling", size=3):
     result : pandas DataFrame
         A DataFrame containing full copies of the substituted and non-substituted rows within df. Some substitution methods may add a new column to the result to indicate that substitution was perfomed.
     """
-    df["meas_value"] = df["meas_value"].apply(lambda x: x if (not isinstance(x, basestring) and not (x is None or np.isnan(x))) or (isinstance(x, basestring) and x != "") else np.nan)
+    df["meas_value"] = df["meas_value"].apply(
+        lambda x: x if (
+            not isinstance(
+                x,
+                basestring) and not (
+                x is None or np.isnan(x))) or (
+                    isinstance(
+                        x,
+                        basestring) and x != "") else np.nan)
     for meas_name in meas_names:
 
         if (method == "rolling"):

ipython/epidata/automated_test.py

@@ -47,41 +47,65 @@ def to_row(x):
                 output["meas_value_datatype"] = 'string'
 
         if "meas_value" in dict and isinstance(dict['meas_value'], float):
-            if output["meas_datatype"] == "long" and long(float(dict['meas_value'])) == float(dict['meas_value']):
+            if output["meas_datatype"] == "long" and long(
+                    float(dict['meas_value'])) == float(dict['meas_value']):
                 output["meas_value_l"] = float(dict["meas_value"])
                 output["meas_value_datatype"] = 'long'
             else:
                 output["meas_value_d"] = float(dict["meas_value"])
                 output["meas_value_datatype"] = 'double'
 
-        if "meas_value" in dict and ( isinstance(dict['meas_value'], int) or isinstance(dict['meas_value'], long)):
+        if "meas_value" in dict and (
+            isinstance(
+                dict['meas_value'],
+                int) or isinstance(
+                dict['meas_value'],
+                long)):
             output["meas_value_l"] = float(dict["meas_value"])
             output["meas_value_datatype"] = 'long'
 
         # Set meas_lower_limit
         output["meas_lower_limit_d"] = np.nan
         output["meas_lower_limit_l"] = np.nan
 
-        if "meas_lower_limit" in dict and isinstance(dict['meas_lower_limit'], float):
-            if long(float(dict["meas_lower_limit"])) == float(dict["meas_lower_limit"]):
+        if "meas_lower_limit" in dict and isinstance(
+                dict['meas_lower_limit'], float):
+            if long(
+                    float(
+                        dict["meas_lower_limit"])) == float(
+                    dict["meas_lower_limit"]):
                 output["meas_lower_limit_l"] = float(dict["meas_lower_limit"])
             else:
                 output["meas_lower_limit_d"] = float(dict["meas_lower_limit"])
 
-        if "meas_lower_limit" in dict and (isinstance(dict['meas_lower_limit'], int) or isinstance(dict['meas_lower_limit'], long)):
+        if "meas_lower_limit" in dict and (
+            isinstance(
+                dict['meas_lower_limit'],
+                int) or isinstance(
+                dict['meas_lower_limit'],
+                long)):
             output["meas_lower_limit_l"] = float(dict["meas_lower_limit"])
 
         # Set meas_upper_limit
         output["meas_upper_limit_d"] = np.nan
         output["meas_upper_limit_l"] = np.nan
 
-        if "meas_upper_limit" in dict and isinstance(dict['meas_upper_limit'], float):
-            if long(float(dict["meas_upper_limit"])) == float(dict["meas_upper_limit"]):
+        if "meas_upper_limit" in dict and isinstance(
+                dict['meas_upper_limit'], float):
+            if long(
+                    float(
+                        dict["meas_upper_limit"])) == float(
+                    dict["meas_upper_limit"]):
                 output["meas_upper_limit_l"] = float(dict["meas_upper_limit"])
             else:
                 output["meas_upper_limit_d"] = float(dict["meas_upper_limit"])
 
-        if "meas_upper_limit" in dict and (isinstance(dict['meas_upper_limit'], int) or isinstance(dict['meas_upper_limit'], long)):
+        if "meas_upper_limit" in dict and (
+            isinstance(
+                dict['meas_upper_limit'],
+                int) or isinstance(
+                dict['meas_upper_limit'],
+                long)):
             output["meas_upper_limit_l"] = float(dict["meas_upper_limit"])
 
         return Row(**output)

ipython/epidata/sensor_measurement.py

@@ -44,41 +44,65 @@ def to_row(x):
                 output["meas_value_datatype"] = 'string'
 
         if "meas_value" in dict and isinstance(dict['meas_value'], float):
-            if output["meas_datatype"] == "long" and long(float(dict['meas_value'])) == float(dict['meas_value']):
+            if output["meas_datatype"] == "long" and long(
+                    float(dict['meas_value'])) == float(dict['meas_value']):
                 output["meas_value_l"] = float(dict["meas_value"])
                 output["meas_value_datatype"] = 'long'
             else:
                 output["meas_value_d"] = float(dict["meas_value"])
                 output["meas_value_datatype"] = 'double'
 
-        if "meas_value" in dict and ( isinstance(dict['meas_value'], int) or isinstance(dict['meas_value'], long)):
+        if "meas_value" in dict and (
+            isinstance(
+                dict['meas_value'],
+                int) or isinstance(
+                dict['meas_value'],
+                long)):
             output["meas_value_l"] = float(dict["meas_value"])
             output["meas_value_datatype"] = 'long'
 
         # Set meas_lower_limit
         output["meas_lower_limit_d"] = np.nan
         output["meas_lower_limit_l"] = np.nan
 
-        if "meas_lower_limit" in dict and isinstance(dict['meas_lower_limit'], float):
-            if long(float(dict["meas_lower_limit"])) == float(dict["meas_lower_limit"]):
+        if "meas_lower_limit" in dict and isinstance(
+                dict['meas_lower_limit'], float):
+            if long(
+                    float(
+                        dict["meas_lower_limit"])) == float(
+                    dict["meas_lower_limit"]):
                 output["meas_lower_limit_l"] = float(dict["meas_lower_limit"])
             else:
                 output["meas_lower_limit_d"] = float(dict["meas_lower_limit"])
 
-        if "meas_lower_limit" in dict and (isinstance(dict['meas_lower_limit'], int) or isinstance(dict['meas_lower_limit'], long)):
+        if "meas_lower_limit" in dict and (
+            isinstance(
+                dict['meas_lower_limit'],
+                int) or isinstance(
+                dict['meas_lower_limit'],
+                long)):
             output["meas_lower_limit_l"] = float(dict["meas_lower_limit"])
 
         # Set meas_upper_limit
         output["meas_upper_limit_d"] = np.nan
         output["meas_upper_limit_l"] = np.nan
 
-        if "meas_upper_limit" in dict and isinstance(dict['meas_upper_limit'], float):
-            if long(float(dict["meas_upper_limit"])) == float(dict["meas_upper_limit"]):
+        if "meas_upper_limit" in dict and isinstance(
+                dict['meas_upper_limit'], float):
+            if long(
+                    float(
+                        dict["meas_upper_limit"])) == float(
+                    dict["meas_upper_limit"]):
                 output["meas_upper_limit_l"] = float(dict["meas_upper_limit"])
             else:
                 output["meas_upper_limit_d"] = float(dict["meas_upper_limit"])
 
-        if "meas_upper_limit" in dict and (isinstance(dict['meas_upper_limit'], int) or isinstance(dict['meas_upper_limit'], long)):
+        if "meas_upper_limit" in dict and (
+            isinstance(
+                dict['meas_upper_limit'],
+                int) or isinstance(
+                dict['meas_upper_limit'],
+                long)):
             output["meas_upper_limit_l"] = float(dict["meas_upper_limit"])
 
         return Row(**output)

ipython/epidata/streaming.py

@@ -8,7 +8,7 @@
 from kafka import KafkaClient
 import json
 
-from epidata._private.utils import ConvertUtils
+from utils import ConvertUtils
 from sensor_measurement import SensorMeasurement
 from automated_test import AutomatedTest
 
@@ -79,7 +79,6 @@ def process(time, rdd):
                             output_df = ConvertUtils.convert_meas_value(
                                 output_df, op.destination())
 
-
                             # convert it back to spark data frame
                             spark_output_df = self._sql_ctx.createDataFrame(
                                 output_df, self._get_schema(op.destination()))

ipython/epidata/_private/utils.py → ipython/epidata/utils.py

@@ -39,8 +39,9 @@ def convert_meas_value(df, destination):
                     ConvertUtils.try_set_cell_with_float_value_if_not_use_string(
                         df, index, 'meas_value_d', row, 'meas_value', 'meas_value_s')
 
-
-                if row['meas_datatype'] == "long" and ConvertUtils.is_long_number(row['meas_upper_limit']) and ConvertUtils.is_long_number(row['meas_lower_limit']):
+                if row['meas_datatype'] == "long" and ConvertUtils.is_long_number(
+                        row['meas_upper_limit']) and ConvertUtils.is_long_number(
+                        row['meas_lower_limit']):
                     ConvertUtils.try_set_cell_with_long_value(
                         df, index, 'meas_upper_limit_l', row, 'meas_upper_limit')
                     ConvertUtils.try_set_cell_with_long_value(
@@ -173,9 +174,9 @@ def convert_to_pandas_dataframe_model(rdd_df, clean_up=True):
                     row['meas_value_datatype'])
 
                 if row['meas_value_datatype'] == "string" and row['meas_value_str'] != "":
-                    df.set_value(index,'meas_upper_limit',np.nan)
-                    df.set_value(index,'meas_lower_limit',np.nan)
-                    df.set_value(index,'meas_unit',"")
+                    df.set_value(index, 'meas_upper_limit', np.nan)
+                    df.set_value(index, 'meas_lower_limit', np.nan)
+                    df.set_value(index, 'meas_unit', "")
 
         df = df.drop('meas_value_str', 1)
         df = df.drop('meas_value_d', 1)

ipython/home/tutorials/sensor_data_ingest.py

@@ -32,12 +32,10 @@
 
 if USE_KAFKA:
     CREATE_MEASUREMENT_URL = 'https://' + HOST + '/kafka/measurements'
-    CREATE_MEASUREMENT_ROUTE = '/kafka/measurements'
-    CREATE_MEASUREMENT_LIST_ROUTE = '/kafka/measurements_list'
+    CREATE_MEASUREMENT_LIST_ROUTE = '/kafka/measurements'
 else:
     CREATE_MEASUREMENT_URL = 'https://' + HOST + '/measurements'
-    CREATE_MEASUREMENT_ROUTE = '/measurements'
-    CREATE_MEASUREMENT_LIST_ROUTE = '/measurements_list'
+    CREATE_MEASUREMENT_LIST_ROUTE = '/measurements'
 
 iteration = 0
 post_iteration = 0
@@ -64,7 +62,8 @@ def add_time(time_string, delta):
 # EDIT THIS SECTION #
 #####################
 
-# Replace quoted string with API Token or GitHub Personal Access Token (REQUIRED)
+# Replace quoted string with API Token or GitHub Personal Access Token
+# (REQUIRED)
 ACCESS_TOKEN = 'API Token'
 
 # Modify default values (OPTIONAL)

ipython/home/tutorials/sensor_data_query.py

@@ -14,7 +14,8 @@
 import struct
 import time
 from time import sleep
-import urllib, urllib2
+import urllib
+import urllib2
 
 
 ##################################
@@ -35,11 +36,16 @@
 
 def get_time(time_string):
     date_object = datetime.strptime(time_string, '%m/%d/%Y %H:%M:%S.%f')
-    return long(time.mktime(date_object.timetuple()) * 1e3 + date_object.microsecond / 1e3)
+    return long(time.mktime(date_object.timetuple())
+                * 1e3 + date_object.microsecond / 1e3)
+
 
 def add_time(time_string, delta):
-    date_object = datetime.strptime(time_string, '%m/%d/%Y %H:%M:%S.%f') + timedelta(seconds=delta)
-    return long(time.mktime(date_object.timetuple()) * 1e3 + date_object.microsecond / 1e3)
+    date_object = datetime.strptime(
+        time_string, '%m/%d/%Y %H:%M:%S.%f') + timedelta(seconds=delta)
+    return long(time.mktime(date_object.timetuple())
+                * 1e3 + date_object.microsecond / 1e3)
+
 
 current_time_string = datetime.now().strftime("%m/%d/%Y %H:%M:%S.%f")
 current_time = get_time(current_time_string)
@@ -49,11 +55,12 @@ def add_time(time_string, delta):
 # EDIT THIS SECTION #
 #####################
 
-# Replace quoted string with API Token or GitHub Personal Access Token (REQUIRED)
+# Replace quoted string with API Token or GitHub Personal Access Token
+# (REQUIRED)
 ACCESS_TOKEN = 'API Token'
 
 # Modify default values (OPTIONAL)
-COMPANY ='EpiData'
+COMPANY = 'EpiData'
 SITE = 'San_Jose'
 STATION = 'WSN-1'
 SENSOR = "Temperature_Probe"
@@ -117,20 +124,26 @@ def add_time(time_string, delta):
 iteration = 0
 
 while (True):
-    
+
     try:
-        
+
         # Create instances that connect to the server
         conn = httplib.HTTPSConnection(HOST)
 
         # Specify measurement query parameters
         begin_time = get_time("8/1/2017 00:00:00.000")
         end_time = get_time("9/1/2017 00:00:00.000")
-
-        parameters = {'company': COMPANY, 'site': SITE, 'station': STATION, 'sensor': SENSOR, 'beginTime': begin_time, 'endTime': end_time}
+
+        parameters = {
+            'company': COMPANY,
+            'site': SITE,
+            'station': STATION,
+            'sensor': SENSOR,
+            'beginTime': begin_time,
+            'endTime': end_time}
 
         # Construct url with parameters
-        url = QUERY_MEASUREMENTS_ORIGINAL_URL+urllib.urlencode(parameters)
+        url = QUERY_MEASUREMENTS_ORIGINAL_URL + urllib.urlencode(parameters)
         print url
         json_header = {'Cookie': session_cookie, 'Accept': 'text/plain'}
 
@@ -140,8 +153,9 @@ def add_time(time_string, delta):
         response_status = get_response.status
         response_text = get_response.read()
         print response_status, response_text
-
-        # Check that the response's HTTP response code is 200 (OK) and read the response.
+
+        # Check that the response's HTTP response code is 200 (OK) and read the
+        # response.
         assert response_status == 200
         response_json = json.loads(response_text)
         print response_json
@@ -163,4 +177,3 @@ def add_time(time_string, delta):
 ################################
 # End of Data Query Script #
 ################################
-

ipython/home/tutorials/sensor_data_with_outliers.py

@@ -32,12 +32,10 @@
 
 if USE_KAFKA:
     CREATE_MEASUREMENT_URL = 'https://' + HOST + '/kafka/measurements'
-    CREATE_MEASUREMENT_ROUTE = '/kafka/measurements'
-    CREATE_MEASUREMENT_LIST_ROUTE = '/kafka/measurements_list'
+    CREATE_MEASUREMENT_LIST_ROUTE = '/kafka/measurements'
 else:
     CREATE_MEASUREMENT_URL = 'https://' + HOST + '/measurements'
-    CREATE_MEASUREMENT_ROUTE = '/measurements'
-    CREATE_MEASUREMENT_LIST_ROUTE = '/measurements_list'
+    CREATE_MEASUREMENT_LIST_ROUTE = '/measurements'
 
 iteration = 0
 post_iteration = 0
@@ -64,7 +62,8 @@ def add_time(time_string, delta):
 # EDIT THIS SECTION #
 #####################
 
-# Replace quoted string with API Token or GitHub Personal Access Token (REQUIRED)
+# Replace quoted string with API Token or GitHub Personal Access Token
+# (REQUIRED)
 ACCESS_TOKEN = 'API TOKEN'
 
 # Modify default values (OPTIONAL)