fix: take last win_size datapoints

Signed-off-by: kbatra <i.kushalbatra@gmail.com>
numaproj · Aug 10, 2023 · 953b7e9 · 953b7e9
1 parent 04c23f6
commit 953b7e9
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 68 deletions.
diff --git a/udf/anomaly-detection/src/udf/preprocess.py b/udf/anomaly-detection/src/udf/preprocess.py
@@ -21,15 +21,14 @@
 
 class Preprocess:
     @classmethod
-    def get_df(cls, data_payload: dict, features: List[str]) -> (pd.DataFrame, List[int]):
+    def get_df(cls, data_payload: dict, features: List[str], win_size: int) -> (pd.DataFrame, List[int]):
         start_time = int(data_payload["start_time"])
         end_time = int(data_payload["end_time"])
-
-        df = pd.DataFrame(data_payload["data"], columns=["timestamp", *features])
-        df.index = df.timestamp.astype(int)
-        timestamps = np.arange(start_time+6e4, end_time+6e4, 6e4, dtype=int)
+        df = pd.DataFrame(data_payload["data"], columns=["timestamp", *features]).astype(float).fillna(0)
+        timestamps = np.arange(start_time, end_time+6e4, 6e4, dtype='int')[-win_size:]
         df = df.reindex(timestamps, fill_value=0)
-        return df, timestamps
+        df.reset_index(drop=True, inplace=True)
+        return df[features], timestamps
 
     def preprocess(self, keys: List[str], payload: StreamPayload) -> (np.ndarray, Status):
         preprocess_cfgs = ConfigManager.get_preprocess_config(config_id=keys[0])
@@ -104,7 +103,7 @@ def run(self, keys: List[str], datum: Datum) -> Messages:
 
         # Load config
         stream_conf = ConfigManager.get_stream_config(config_id=data_payload["config_id"])
-        raw_df, timestamps = self.get_df(data_payload, stream_conf.metrics)
+        raw_df, timestamps = self.get_df(data_payload, stream_conf.metrics, stream_conf.window_size)
 
         # Prepare payload for forwarding
         payload = StreamPayload(

diff --git a/udf/anomaly-detection/tests/resources/data/stream.json b/udf/anomaly-detection/tests/resources/data/stream.json
@@ -1,61 +1 @@
-{
-    "uuid": "1234567890",
-    "config_id": "druid-config",
-    "start_time": "1684472580000",
-    "end_time": "1684473300000",
-    "data": [
-        {
-            "timestamp": "1684472580000",
-            "error_rate": "56.97",
-            "error_count": "10",
-            "failed": "6",
-            "degraded": "4"
-        },
-        {
-            "timestamp": "1684472700000",
-            "error_rate": "70.46",
-            "error_count": "15",
-            "failed": "10",
-            "degraded": "5"
-        },
-        {
-            "timestamp": "1684472820000",
-            "error_rate": "65.02",
-            "error_count": "12",
-            "failed": "8",
-            "degraded": "4"
-        },
-        {
-            "timestamp": "1684472940000",
-            "error_rate": "16.75",
-            "error_count": "3",
-            "failed": "1",
-            "degraded": "2"
-        },
-        {
-            "timestamp": "1684473060000",
-            "error_rate": "86.91",
-            "error_count": "18",
-            "failed": "10",
-            "degraded": "8"
-        },
-        {
-            "timestamp": "1684473180000",
-            "error_rate": "13.49",
-            "error_count": "1",
-            "failed": "0",
-            "degraded": "1"
-        },
-        {
-            "timestamp": "1684473240000",
-            "error_rate": "62.74",
-            "error_count": "11",
-            "failed": "8",
-            "degraded": "3"
-        }
-    ],
-    "metadata": {
-        "source":  "kafka",
-        "label": "value"
-    }
-}
+{"uuid":"dd7dfb43-532b-49a3-906e-f78f82ad9c4b","config_id":"druid-config","data":[{"degraded":14,"degraded_rate":0.003626943005181347,"error_rate":0.009067357512953367,"failed":21,"failed_rate":0.005440414507772021,"success":3825,"timestamp":1691622660000},{"degraded":10,"degraded_rate":0.002484472049689441,"error_rate":0.009937888198757764,"failed":30,"failed_rate":0.007453416149068323,"success":3985,"timestamp":1691622720000},{"degraded":9,"degraded_rate":0.0025906735751295338,"error_rate":0.005469199769717904,"failed":10,"failed_rate":0.0028785261945883708,"success":3455,"timestamp":1691622780000},{"degraded":13,"degraded_rate":0.003798947983635301,"error_rate":0.007597895967270602,"failed":13,"failed_rate":0.003798947983635301,"success":3396,"timestamp":1691622840000},{"degraded":11,"degraded_rate":0.0030328094844223876,"error_rate":0.005789909015715467,"failed":10,"failed_rate":0.0027570995312930797,"success":3606,"timestamp":1691622900000},{"degraded":17,"degraded_rate":0.004964953271028037,"error_rate":0.009345794392523364,"failed":15,"failed_rate":0.004380841121495327,"success":3392,"timestamp":1691622960000},{"degraded":10,"degraded_rate":0.002774694783573807,"error_rate":0.006104328523862375,"failed":12,"failed_rate":0.003329633740288568,"success":3582,"timestamp":1691623020000},{"degraded":12,"degraded_rate":0.003770028275212064,"error_rate":0.00942507068803016,"failed":18,"failed_rate":0.005655042412818096,"success":3153,"timestamp":1691623080000},{"degraded":14,"degraded_rate":0.003469640644361834,"error_rate":0.006195786864931847,"failed":11,"failed_rate":0.0027261462205700124,"success":4010,"timestamp":1691623140000},{"degraded":4,"degraded_rate":0.0010887316276537834,"error_rate":0.005715841045182362,"failed":17,"failed_rate":0.004627109417528579,"success":3653,"timestamp":1691623200000},{"degraded":13,"degraded_rate":0.0035278154681139757,"error_rate":0.009497964721845319,"failed":22,"failed_rate":0.005970149253731343,"success":3650,"timestamp":1691623260000},{"degraded":7,"degraded_rate":0.0019358407079646017,"error_rate":0.00663716814159292,"failed":17,"failed_rate":0.004701327433628319,"success":3592,"timestamp":1691623320000},{"degraded":18,"degraded_rate":0.004288777698355968,"error_rate":0.00976888253514415,"failed":23,"failed_rate":0.005480104836788182,"success":4156,"timestamp":1691623380000},{"degraded":15,"degraded_rate":0.00487012987012987,"error_rate":0.00974025974025974,"failed":15,"failed_rate":0.00487012987012987,"success":3050,"timestamp":1691623440000},{"degraded":9,"degraded_rate":0.0024563318777292577,"error_rate":0.006823144104803494,"failed":16,"failed_rate":0.004366812227074236,"success":3639,"timestamp":1691623500000},{"degraded":10,"degraded_rate":0.0031959092361776927,"error_rate":0.006391818472355385,"failed":10,"failed_rate":0.0031959092361776927,"success":3109,"timestamp":1691623560000},{"degraded":12,"degraded_rate":0.003827751196172249,"error_rate":0.004784688995215311,"failed":3,"failed_rate":0.0009569377990430622,"success":3120,"timestamp":1691623620000},{"degraded":21,"degraded_rate":0.007123473541383989,"error_rate":0.009158751696065129,"failed":6,"failed_rate":0.0020352781546811396,"success":2921,"timestamp":1691623680000},{"degraded":7,"degraded_rate":0.002027222704894295,"error_rate":0.0034752389226759338,"failed":5,"failed_rate":0.0014480162177816392,"success":3441,"timestamp":1691623740000},{"degraded":8,"degraded_rate":0.0022179096201829776,"error_rate":0.0049902966454117,"failed":10,"failed_rate":0.0027723870252287217,"success":3589,"timestamp":1691623800000}],"start_time":1691622660000,"end_time":1691623860000,"metadata":{"tags":{"asset_alias":"Intuit.smallbusiness.quickbooksmobile.qbmios","asset_id":"362557362191815079","env":"prd"}}}