Merge branch 'dev'

* dev:
  Bugfix in network-io reporter; Removed phase from measurements table;
  Extra newline

api/api.py

@@ -267,7 +267,7 @@ async def get_measurements_single(project_id: str):
 
     query = """
             SELECT measurements.detail_name, measurements.time, measurements.metric,
-                   measurements.value, measurements.unit, measurements.phase
+                   measurements.value, measurements.unit
             FROM measurements
             WHERE measurements.project_id = %s
             """

docker/structure.sql

@@ -33,12 +33,10 @@ CREATE TABLE measurements (
     value bigint NOT NULL,
     unit text NOT NULL,
     time bigint NOT NULL,
-    phase text DEFAULT null,
     created_at timestamp with time zone DEFAULT now()
 );
 
 CREATE UNIQUE INDEX measurements_get ON measurements(project_id ,metric ,detail_name ,time );
-CREATE INDEX measurements_phase_update ON measurements(project_id ,phase ,time );
 CREATE INDEX measurements_build_and_store_phase_stats ON measurements(project_id, metric, unit, detail_name);
 CREATE INDEX measurements_build_phases ON measurements(metric, unit, detail_name);
 

install_linux.sh

@@ -53,6 +53,7 @@ fi
 
 if [[ -z "$db_pw" ]] ; then
     read -sp "Please enter the new password to be set for the PostgreSQL DB: " db_pw
+    echo "" # force a newline, because print -sp will consume it
 fi
 
 if [[ $ask_tmpfs == true ]] ; then

migrations/2023_08_10_indices.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS measurements_phase_update;
+ALTER TABLE "measurements" DROP COLUMN "phase";

tools/phase_stats.py

@@ -36,17 +36,10 @@ def build_and_store_phase_stats(project_id):
     csv_buffer = StringIO()
 
     for idx, phase in enumerate(phases[0]):
-        query = """
-            UPDATE measurements
-            SET phase = %s
-            WHERE project_id = %s AND phase IS NULL and time > %s and time < %s
-            """
-        DB().query(query, (phase['name'], project_id, phase['start'], phase['end'], ))
-
         network_io_bytes_total = [] # reset; # we use array here and sum later, because checking for 0 alone not enough
 
         select_query = """
-            SELECT SUM(value), MAX(value), AVG(value), COUNT(value)
+            SELECT SUM(value), MAX(value), MIN(value), AVG(value), COUNT(value)
             FROM measurements
             WHERE project_id = %s AND metric = %s AND detail_name = %s AND time > %s and time < %s
         """
@@ -66,11 +59,11 @@ def build_and_store_phase_stats(project_id):
 
             value_sum = 0
             value_max = 0
+            value_min = 0
             value_avg = 0
             value_count = 0
 
-
-            value_sum, value_max, value_avg, value_count = results
+            value_sum, value_max, value_min, value_avg, value_count = results
 
             # no need to calculate if we have no results to work on
             # This can happen if the phase is too short
@@ -88,10 +81,10 @@ def build_and_store_phase_stats(project_id):
 
             elif metric == 'network_io_cgroup_container':
                 # These metrics are accumulating already. We only need the max here and deliver it as total
-                csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_max, 'TOTAL', None, unit))
+                csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_max-value_min, 'TOTAL', None, unit))
                 # No max here
                 # But we need to build the energy
-                network_io_bytes_total.append(value_max)
+                network_io_bytes_total.append(value_max-value_min)
 
             elif metric == 'energy_impact_powermetrics_vm':
                 csv_buffer.write(generate_csv_line(project_id, metric, detail_name, f"{idx:03}_{phase['name']}", value_avg, 'MEAN', value_max, unit))

tools/rebuild_phase_stats.py

@@ -0,0 +1,30 @@
+#pylint: disable=import-error,wrong-import-position
+
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib'))
+from phase_stats import build_and_store_phase_stats
+
+from db import DB
+
+if __name__ == '__main__':
+    print('This will remove ALL phase_stats and completely rebuild them. No data will get lost, but it will take some time. Continue? (y/N)')
+    answer = sys.stdin.readline()
+    if answer.strip().lower() == 'y':
+        print('Deleting old phase_stats ...')
+        DB().query('DELETE FROM phase_stats')
+        print('Fetching projects ...')
+        query = '''
+            SELECT id
+            FROM projects
+            WHERE
+                end_measurement IS NOT NULL AND phases IS NOT NULL
+        '''
+        projects = DB().fetch_all(query)
+
+        print(f"Fetched {len(projects)} projects. Commencing ...")
+        for idx, project_id in enumerate(projects):
+
+            print(f"Rebuilding phase_stats for project #{idx} {project_id[0]}")
+            build_and_store_phase_stats(project_id[0])
+        print('Done')