Merge branch 'main' into dependabot/github_actions/aquasecurity/trivy…

…-action-0.17.0
bcgov · Feb 7, 2024 · ed1d077 · ed1d077
2 parents def65f1 + fb1fa61
commit ed1d077
Show file tree

Hide file tree

Showing 20 changed files with 636 additions and 234 deletions.
diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml
@@ -53,7 +53,7 @@ jobs:
             build_context: ./frontend
     steps:
       - uses: actions/checkout@v4
-      - uses: bcgov-nr/action-builder-ghcr@v2.0.1
+      - uses: bcgov-nr/action-builder-ghcr@v2.0.2
         with:
           package: ${{ matrix.package }}
           build_file: ${{ matrix.build_file }}

diff --git a/.gitignore b/.gitignore
@@ -109,3 +109,6 @@ test-report.xml
 
 # Docker/Podman volumes
 .volumes
+
+# test files
+*.xlsx
diff --git a/HydRA Test Curves_20240115/RC_LCR2122.csv b/HydRA Test Curves_20240115/RC_LCR2122.csv
@@ -0,0 +1,20 @@
+datetime,discharge,stage,uncertainty,comments
+2021-11-24 10:27,0.9085,0.492,8,"23 panels, max 9.2%"
+2021-12-07 11:15,1.751439049,0.501,3.11,
+2021-12-16 10:59,1.1704,0.457,8,"22 panels, max 7.4%"
+2022-01-14 11:05,4.891958877,0.73,2.67,
+2022-02-07 11:38,1.127406583,0.459,3.27,
+2022-03-23 10:16,2.319489087,0.55,2.65,
+2022-03-30 14:45,0.843988357,0.515,2.94,
+2022-04-22 13:56,0.569,0.397,,"20 panels, max 13.8%"
+2022-05-03 17:29,0.465,0.377,,"20 panels, max 13.6%"
+2022-05-18 14:37,0.981925337,0.437,3.23,
+2022-07-19 9:53,0.131,0.277,,"20 panels, max 14.3%"
+2022-07-19 10:09,0.1472,0.278,3,
+2022-08-04 9:54,0.152,0.283,,"20 panels, max 11.9%"
+2022-08-04 12:30,0.183566024,0.286,3.17,
+2022-08-31 11:51,0.111,0.263,,"20 panels, max 9.2%"
+2022-09-09 10:02,0.101913626,0.261,2.59,
+2022-10-20 8:40,0.0813,0.243,2.9,
+2022-12-12 10:55,1.672120912,0.4875,3.13,
+2022-12-12 11:02,1.631,0.485,2.7,
diff --git a/HydRA Test Curves_20240115/RC_Lizard.csv b/HydRA Test Curves_20240115/RC_Lizard.csv
@@ -0,0 +1,6 @@
+datetime,discharge,stage,uncertainty,comments
+2021-08-19 11:45,0.364,0.302,0.05,"old control, do not use MMT"
+2022-07-20 8:54,0.612,0.225,0.05,suggested PZF -0.150
+2022-07-27 15:27,0.482,0.193,0.05,
+2022-09-07 11:00,0.383,0.125,0.05,
+2022-10-18 11:33,0.201,0.097,0.05,
diff --git a/HydRA Test Curves_20240115/RC_cook1.csv b/HydRA Test Curves_20240115/RC_cook1.csv
@@ -0,0 +1,20 @@
+datetime,discharge,stage,uncertainty,comments
+2023-07-25 10:50:08,0.024789663,9.0635,0.099,approx PZF 9.0
+2023-05-08 13:23:17,0.468793188,9.193,0.031,
+2023-04-18 12:45:02,1.0337748,9.263,0.042,
+2023-02-06 12:00:00,1.080884219,9.27,0.032,
+2022-08-18 10:15:00,0.028,9.037,0.049,
+2022-07-19 10:11:16,0.093839378,9.065,0.03,
+2022-07-19 10:45:00,0.088,9.065,0.033,
+2022-04-13 13:00:00,0.557,9.177,0.03,
+2022-03-09 11:00:00,0.406728145,9.16,0.03,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
diff --git a/HydRA Test Curves_20240115/RC_cook2.csv b/HydRA Test Curves_20240115/RC_cook2.csv
@@ -0,0 +1,10 @@
+datetime,discharge,stage,stage_raw,uncertainty,comments
+2023-07-25 10:50,0.024789663,0.3085,9.0635,0.099,approx PZF .245
+2023-05-08 13:23,0.468793188,0.438,9.193,0.031,
+2023-04-18 12:45,1.0337748,0.508,9.263,0.042,
+2023-02-06 12:00,1.080884219,0.515,9.27,0.032,
+2022-08-18 10:15,0.028,0.282,9.037,0.049,
+2022-07-19 10:11,0.093839378,0.31,9.065,0.03,
+2022-07-19 10:45,0.088,0.31,9.065,0.033,
+2022-04-13 13:00,0.557,0.422,9.177,0.03,
+2022-03-09 11:00,0.406728145,0.405,9.16,0.03,
diff --git a/HydRA Test Curves_20240115/RC_grand.csv b/HydRA Test Curves_20240115/RC_grand.csv
@@ -0,0 +1,24 @@
+datetime,stage,discharge,Uncertainty,Comments
+2023-07-26 12:57:30,0.186,0.002772154,,"PZF 0.018 , BP @ 0.395M"
+2023-05-26 13:02:30,0.198,0.005309356,,minor debris
+2023-04-19 13:51:15,0.2965,0.039120759,6.8,section
+2023-03-03 15:42:40,0.442,0.161655436,3.7,submerged
+2023-02-12 15:15:00,0.368,0.066844777,3.683071598,section
+2022-11-04 13:00:00,0.24,0.016708148,8,
+2022-09-19 16:00:00,0.188,0.00292457,8,
+2022-08-17 10:00:00,0.19,0.003312321,5,
+2022-07-18 15:00:00,0.21,0.004,,
+2021-08-17 09:00:00,0.285,0.002114971,,debris
+2019-09-19 13:45:00,0.23,0.01125648,10,
+2019-07-29 11:15:00,0.196,0.003352529,10,
+2019-06-10 10:40:00,0.209,0.00646972,8,
+2018-06-05 12:00:00,0.218,0.007823333,8,
+2017-11-27 13:30:00,0.54,0.4602,3.6,
+2017-07-12 13:30:00,0.205,0.004951,8,
+2017-06-13 16:00:00,0.215,0.007517333,8,
+2014-12-19 11:30:00,0.498,0.2883,,submerged
+2014-02-17 13:00:00,0.658,1.063,8,submerged
+2013-10-03 12:00:00,0.318,0.0388,5,
+2013-06-25 09:30:00,0.247,0.0162,8,
+2012-10-11 15:00:00,0.203,0.00382,5,
+2012-09-12 13:00:00,0.198,0.002749,8,
diff --git a/HydRA Test Curves_20240115/RC_random.csv b/HydRA Test Curves_20240115/RC_random.csv
@@ -0,0 +1,22 @@
+date,time,DateTime,discharge,stage,uncertainty,comments
+2022-03-25,12:00:00,2022-03-25 12:00,2.796,0.759,0.004,
+2021-06-25,0:48:23,2021-06-25 0:48,2.827,0.774,0.008,
+2020-11-04,14:45:54,2020-11-04 14:45,3.816,2.792,0.060,
+2020-09-23,6:43:37,2020-09-23 6:43,3.530,1.838,0.095,
+2020-08-22,3:12:13,2020-08-22 3:12,4.587,5.577,0.041,
+2020-03-24,11:10:33,2020-03-24 11:10,4.328,3.894,0.001,
+2019-05-30,3:34:14,2019-05-30 3:34,3.805,2.830,0.098,
+2018-08-31,12:10:31,2018-08-31 12:10,3.588,1.945,0.022,
+2017-09-15,2:11:41,2017-09-15 2:11,3.808,2.590,0.023,
+2017-06-13,5:36:18,2017-06-13 5:36,4.358,4.321,0.004,
+2016-06-18,14:06:14,2016-06-18 14:06,5.582,8.625,0.029,
+2016-02-14,8:54:33,2016-02-14 8:54,5.537,9.343,0.021,
+2015-06-18,1:21:39,2015-06-18 1:21,5.468,8.690,0.063,
+2015-02-16,13:05:51,2015-02-16 13:05,2.377,0.349,0.047,
+2014-09-30,0:54:20,2014-09-30 0:54,5.054,9.185,0.080,
+2014-03-14,22:27:11,2014-03-14 22:27,5.047,9.827,0.001,
+2013-03-26,21:35:10,2013-03-26 21:35,4.690,6.033,0.064,
+2013-03-04,0:46:13,2013-03-04 0:46,4.009,3.816,0.046,
+2012-05-21,21:36:24,2012-05-21 21:36,5.416,9.181,0.018,
+,,,,,,
+,,,,,,
diff --git a/HydRA Test Curves_20240115/RC_shelly.csv b/HydRA Test Curves_20240115/RC_shelly.csv
@@ -0,0 +1,11 @@
+datetime,discharge,stage,uncertainty,comments
+2021-11-22 10:00,0.13,0.33,0.05,"21 panels, some snow on ground"
+2021-04-06 14:00,0.01,0.24,0.05,19 panels
+2021-03-08 9:00,0.05,0.28,0.03,"35 panels, recent high flow event in December"
+2021-02-24 13:00,0.03,0.27,0.05,"21 panels, rain overnight"
+2021-01-28 14:00,0.12,0.32,0.04,"21 panels, good mmt"
+2021-01-11 10:00,0.18,0.39,0.04,"21 panels, recent high flows after consistent rain. High turbidity"
+2020-12-30 11:30,0.36,0.45,0.03,"24 panels, recent high flows and heavy rains"
+2020-11-24 13:30,0.03,0.26,0.04,21 panels. Rain overnight
+2021-12-13 8:00,0.29,0.4,0.04,20 panels
+2020-01-12 17:00,0.67,0.55,0.03,"23 panels, recent rain on snow event, creek rising during mmt"
diff --git a/frontend/rctool/functions/fit_linear_model.py b/frontend/rctool/functions/fit_linear_model.py
@@ -6,81 +6,121 @@
 
 
 def fit_linear_model(df, offset, label, weighted=None, intersect_points=None, *args):
-
     # input dataframe with columns H, Q, and U
     df_data = df.copy()
-    df_data.drop(['datetime', 'comments', 'toggle_point'], axis=1, inplace=True)
-    df_data = df_data.rename(columns={'discharge': 'Q', 'stage':'H', 'uncertainty':'U'})
-    df_data.sort_values('H', inplace=True)
+    df_data.drop(["datetime", "comments", "toggle_point"], axis=1, inplace=True)
+    df_data = df_data.rename(
+        columns={"discharge": "Q", "stage": "H", "uncertainty": "U"}
+    )
+    df_data.sort_values("H", inplace=True)
 
     # Apply offset
-    df_data['H0'] = df_data['H'] - offset
+    df_data["H0"] = df_data["H"] - offset
 
     # init model
     plm = models.PowerLawModel()
     # set initial condition for model parameters
     # params = plm.guess(df_data['Q'], x=df_data['H0'])
-    
+
     # JOHNS CODE FOR GUESSING OFFSET
     params = Parameters()
-    params.add('amplitude', value=0.1, min=0.001, max=100)
-    params.add('exponent', value=2, min=0.5, max=5)
-
-
+    params.add("amplitude", value=0.1, min=0.001, max=100)
+    params.add("exponent", value=2, min=0.5, max=5)
 
     # add constraint so model passes through intersect points
     if intersect_points:
         for idx, point in enumerate(intersect_points):
-            constraint_name = 'amplitude'
-            constraint_expression = '{0:g}*{1:g}**(-(exponent))'.format(point[1], point[0] - offset)
+            constraint_name = "amplitude"
+            constraint_expression = "{0:g}*{1:g}**(-(exponent))".format(
+                point[1], point[0] - offset
+            )
             params.add(constraint_name, expr=constraint_expression)
-
-    result = plm.fit(df_data['Q'], params, x=df_data['H0'])
-
 
-    unw_const = round(result.best_values['amplitude'], 4)
-    unw_exp = round(result.best_values['exponent'], 4)
+    result = plm.fit(df_data["Q"], params, x=df_data["H0"])
+
+    unw_const = result.best_values["amplitude"]
+    unw_exp = result.best_values["exponent"]
     unw_best = result.best_fit
-    unw_residual = list( 100 * (np.array(unw_best) - np.array(df_data['Q']) ) / np.array(df_data['Q']) )
-    unw_seg_nodes = [[round(np.array(df_data['H'])[0], 3), round(unw_best[0], 3)], [round(np.array(df_data['H'])[-1], 3), round(unw_best[-1], 3)]]
+
+    unw_residual = list(
+        100 * (np.array(unw_best) - np.array(df_data["Q"])) / np.array(df_data["Q"])
+    )
+    unw_seg_nodes = [
+        [np.array(df_data["H"])[0], unw_best[0]],
+        [np.array(df_data["H"])[-1], unw_best[-1]],
+    ]
 
     # set uncertianty to %
-    df_data['U'] = df_data['U'].apply(lambda x: x / 100.)
+    df_data["U"] = df_data["U"].apply(lambda x: x / 100.0)
     # set weights as 1 - % unc
-    df_data['W'] = df_data['U'].apply(lambda x: 1 - x)
+    df_data["W"] = df_data["U"].apply(lambda x: 1 - x)
     # try weighting
-    result = plm.fit(df_data['Q'], params, x=df_data['H0'], weights=df_data['W'])
-    wgt_const = round(result.best_values['amplitude'], 4)
-    wgt_exp = round(result.best_values['exponent'], 4)
+    result = plm.fit(df_data["Q"], params, x=df_data["H0"], weights=df_data["W"])
+    wgt_const = result.best_values["amplitude"]
+    wgt_exp = result.best_values["exponent"]
     wgt_best = result.best_fit
     wgt_sigs = result.eval_uncertainty(sigma=2)
-    wgt_residual = list( 100 * (np.array(wgt_best) - np.array(df_data['Q']) ) / np.array(df_data['Q']) )
-    wgt_seg_nodes = [[round(np.array(df_data['H'])[0], 3), round(wgt_best[0], 3)], [round(np.array(df_data['H'])[-1], 3), round(wgt_best[-1], 3)]]
+    wgt_residual = list(
+        100 * (np.array(wgt_best) - np.array(df_data["Q"])) / np.array(df_data["Q"])
+    )
+    wgt_seg_nodes = [
+        [np.array(df_data["H"])[0], wgt_best[0]],
+        [np.array(df_data["H"])[-1], wgt_best[-1]],
+    ]
 
     # calculate statistical parameters to analyze goodness of fit
-    unw_mse = mean_squared_error(df_data['Q'], unw_best)
-    unw_rmse = round(math.sqrt(unw_mse), 3)
+    unw_mse = mean_squared_error(df_data["Q"], unw_best)
+    unw_rmse = math.sqrt(unw_mse)
 
-    wgt_mse = mean_squared_error(df_data['Q'], wgt_best)
-    wgt_rmse = round(math.sqrt(wgt_mse), 3)
+    wgt_mse = mean_squared_error(df_data["Q"], wgt_best)
+    wgt_rmse = math.sqrt(wgt_mse)
 
     # Process and ship output
     mdl_param = {
-        'unw': {'label': label ,'const': unw_const, 'exp': unw_exp, 'seg_bounds':unw_seg_nodes, 'offset': offset, 'rmse': unw_rmse},
-        'wgt': {'label': label, 'const': wgt_const,'exp': wgt_exp, 'seg_bounds':wgt_seg_nodes,'offset': offset, 'rmse': wgt_rmse}
+        "unw": {
+            "label": label,
+            "const": unw_const,
+            "exp": unw_exp,
+            "seg_bounds": unw_seg_nodes,
+            "offset": offset,
+            "rmse": unw_rmse,
+        },
+        "wgt": {
+            "label": label,
+            "const": wgt_const,
+            "exp": wgt_exp,
+            "seg_bounds": wgt_seg_nodes,
+            "offset": offset,
+            "rmse": wgt_rmse,
+        },
     }
 
     if weighted:
-        mdl_param = {'label': label, 'const': wgt_const,'exp': wgt_exp, 'seg_bounds':wgt_seg_nodes,'offset': offset, 'rmse': wgt_rmse}
-        wgt_data = [[a,b, c] for a, b, c in zip(df_data['H'].tolist(), wgt_best, wgt_residual)] 
-        mdl_data = [{'label': label, 'data': wgt_data}]
+        mdl_param = {
+            "label": label,
+            "const": wgt_const,
+            "exp": wgt_exp,
+            "seg_bounds": wgt_seg_nodes,
+            "offset": offset,
+            "rmse": wgt_rmse,
+        }
+        wgt_data = [
+            [a, b, c] for a, b, c in zip(df_data["H"].tolist(), wgt_best, wgt_residual)
+        ]
+        mdl_data = [{"label": label, "data": wgt_data}]
     else:
         # unweighted
-        mdl_param = {'label': label ,'const': unw_const, 'exp': unw_exp, 'seg_bounds':unw_seg_nodes, 'offset': offset, 'rmse': unw_rmse}
-        unw_data = [[a,b, c] for a, b, c in zip(df_data['H'].tolist(), unw_best, unw_residual)] 
-        mdl_data = [{'label': label, 'data': unw_data}]
+        mdl_param = {
+            "label": label,
+            "const": unw_const,
+            "exp": unw_exp,
+            "seg_bounds": unw_seg_nodes,
+            "offset": offset,
+            "rmse": unw_rmse,
+        }
+        unw_data = [
+            [a, b, c] for a, b, c in zip(df_data["H"].tolist(), unw_best, unw_residual)
+        ]
+        mdl_data = [{"label": label, "data": unw_data}]
 
     return mdl_data, mdl_param
-
-
-