added monitor.dailystats.csv and improvements

README.md

@@ -88,8 +88,13 @@ python monitor.py cacheupdate
 or
 python monitor.py forceupdate
 ```
-- INPUTFILE: monitor.cfg (configuration of input to hyundai_kia_connect_api)
-- OUTPUTFILE: monitor.csv (appended) or monitor.VIN.csv (latter when multiple vehicles found)
+INPUTFILE: 
+- monitor.cfg (configuration of input to hyundai_kia_connect_api)
+
+OUTPUTFILES:
+- monitor.csv (appended when the last line is different) or monitor.VIN.csv (latter when multiple vehicles found)
+- monitor.dailystats.csv (appended with daily stats after last written daily stats date and not today) or monitor.dailystats.VIN.csv (latter when multiple vehicles found)
+- monitor.lastrun (rewritten with last run date/time of monitor.py) 
 
 Make sure to configure monitor.cfg once:
 ```
@@ -146,6 +151,17 @@ Following information from hyundai_kia_connect_api is added to the monitor.csv f
 - plugged
 - address (dependent on use_geocode configuration)
 
+Following information from hyundai_kia_connect_api is added to the monitor.dailystats.csv file (gathered by the car, so not computed by summary.py), with per day the following information:
+- date
+- distance
+- distance_unit
+- total_consumed Wh
+- regenerated_energy Wh
+- engine_consumption Wh
+- climate_consumption Wh
+- onboard_electronics_consumption Wh
+- battery_care_consumption Wh
+
 This information is used by the other tools:
 - summary.py
 - kml.py
@@ -413,13 +429,17 @@ python debug.py
 # Examples
 ## monitor.csv
 
-Here a csv file from 2022-09-17 till 2022-09-25 (about one week). I started with capturing once per hour. At 2022-09-20 I changed into once each half hour between 6:00 and 19:30, because I barely drive in the evening and still not too many captures per day. My crontab for this:
+Here a csv file from 2022-09-17 till 2022-09-25 (about one week). I started with capturing once per hour. At 2022-09-20 I changed into once each half hour between 6:00 and 19:30, because I barely drive in the evening and still not too many captures per day. 
+
+Example output file [monitor.csv](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/examples/monitor.csv)
+
+My crontab for this:
 
 ```
 */30 6-19 * * * ~/hyundai_kia_connect_monitor/run_monitor_once.sh >> ~/hyundai_kia_connect_monitor/run_monitor_once.log 2>&1
 ```
 
-Example output file [monitor.csv](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/examples/monitor.csv)
+Example output file [monitor.dailystats.csv](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/examples/monitor.dailystats.csv)
 
 ## python summary.py
 

examples/monitor.dailystats.csv

@@ -0,0 +1,27 @@
+datetime, distance, distance_unit, total_consumed, regenerated_energy, engine_consumption, climate_consumption, onboard_electronics_consumption, battery_care_consumption
+20221123, 41, km, 7221, 2060,  5565, 326, 1330, 0
+20221124, 18, km, 2789, 1335,  2234, 115, 440, 0
+20221126, 7, km, 1881, 634,  1121, 170, 590, 0
+20221127, 418, km, 73990, 3999,  68647, 1413, 3930, 0
+20221128, 14, km, 2996, 1126,  1897, 159, 940, 0
+20221129, 18, km, 3235, 1279,  2267, 248, 720, 0
+20221130, 4, km, 1008, 466,  615, 93, 300, 0
+20221201, 57, km, 9491, 3072,  7759, 352, 1380, 0
+20221202, 54, km, 10281, 3432,  7805, 666, 1810, 0
+20221203, 38, km, 6963, 2371,  5497, 556, 910, 0
+20221204, 154, km, 25672, 2536,  23245, 937, 1490, 0
+20221205, 14, km, 3205, 1106,  1985, 330, 890, 0
+20221206, 13, km, 2697, 896,  1677, 190, 830, 0
+20221207, 8, km, 1839, 404,  1121, 148, 570, 0
+20221208, 8, km, 1922, 637,  1031, 11, 880, 0
+20221209, 34, km, 6912, 1743,  5087, 525, 1300, 0
+20221210, 7, km, 1909, 663,  1207, 162, 540, 0
+20221211, 5, km, 1250, 446,  785, 115, 350, 0
+20221212, 45, km, 9087, 1776,  6945, 1112, 1030, 0
+20221213, 21, km, 4899, 1396,  2883, 946, 1070, 0
+20221214, 9, km, 2423, 982,  1349, 424, 650, 0
+20221216, 12, km, 4092, 815,  1795, 1057, 1240, 0
+20221220, 5, km, 1164, 339,  688, 116, 360, 0
+20221221, 16, km, 3623, 1082,  2266, 317, 1040, 0
+20221222, 16, km, 3209, 712,  2400, 139, 670, 0
+20221223, 154, km, 23817, 4307,  21661, 286, 1870, 0

monitor.py

@@ -31,6 +31,7 @@
 - visited places
 """
 import sys
+import os
 import configparser
 import traceback
 import time
@@ -112,6 +113,77 @@ def writeln(filename, string):
         file.write('\n')
 
 
+def get_last_line(filename):
+    """ get last line of filename """
+    with open(filename, "rb") as file:
+        try:
+            file.seek(-2, os.SEEK_END)
+            while file.read(1) != b'\n':
+                file.seek(-2, os.SEEK_CUR)
+        except OSError:
+            file.seek(0)
+        last_line = file.readline().decode().strip()
+        debug(f"{filename} last_line: [{last_line}]")
+        return last_line
+
+
+def get_last_date(last_line):
+    """ get last date of last_line """
+    last_date = '20000101'  # millenium
+    if last_line.startswith("20"):  # year starts with 20
+        last_date = last_line.split(',')[0].strip()
+    debug(f"{last_line} last_date: [{last_date}]")
+    return last_date
+
+
+def handle_daily_stats(vehicle, number_of_vehicles):
+    """ handle daily stats """
+    daily_stats = vehicle.daily_stats
+    if len(daily_stats) == 0:
+        debug("No daily stats")
+        return
+
+    filename = "monitor.dailystats.csv"
+    if number_of_vehicles > 1:
+        filename = "monitor.dailystats." + vehicle.VIN + ".csv"
+    dailystats_file = Path(filename)
+    write_header = False
+    # create header if file does not exists
+    if not dailystats_file.is_file():
+        dailystats_file.touch()
+        write_header = True
+    with dailystats_file.open("a", encoding="utf-8") as file:
+        if write_header:
+            file.write("date, distance, distance_unit, total_consumed, regenerated_energy, engine_consumption, climate_consumption, onboard_electronics_consumption, battery_care_consumption\n")  # noqa pylint:disable=line-too-long
+        today = datetime.now().strftime("%Y%m%d")
+        last_line = get_last_line(filename)
+        last_date = get_last_date(last_line)
+        i = len(daily_stats)
+        while i > 0:
+            i = i - 1
+            stat = daily_stats[i]
+            dailystats_date = stat.date.strftime("%Y%m%d")
+            if today != dailystats_date and dailystats_date >= last_date:
+                # only append not already written daily stats and not today
+                line = f"{dailystats_date}, {stat.distance}, {stat.distance_unit}, {stat.total_consumed}, {stat.regenerated_energy},  {stat.engine_consumption}, {stat.climate_consumption}, {stat.onboard_electronics_consumption}, {stat.battery_care_consumption}"  # noqa pylint:disable=line-too-long
+                if last_line != line:
+                    debug(f"Writing dailystats:\nline=[{line}]\nlast=[{last_line}]")  # noqa pylint:disable=line-too-long
+                    file.write(line)
+                    file.write("\n")
+                else:
+                    debug(f"Skipping dailystats: date=[{dailystats_date}]\nlast=[{last_line}]\nline=[{line}]")  # noqa pylint:disable=line-too-long
+            else:
+                debug(f"Skipping dailystats: [{dailystats_date}] [{last_line}]")  # noqa pylint:disable=line-too-long
+
+
+def write_last_run():
+    """ write last run """
+    filename = "monitor.lastrun"
+    lastrun_file = Path(filename)
+    with lastrun_file.open("w", encoding="utf-8") as file:
+        file.write(datetime.now().strftime("%Y%m%d %H:%M:%S\n"))
+
+
 def get_append_data():
     """ get_append_data """
     retries = 2
@@ -168,14 +240,28 @@ def get_append_data():
                     filename = "monitor.csv"
                     if number_of_vehicles > 1:
                         filename = "monitor." + vehicle.VIN + ".csv"
-                    writeln(filename, line)
+                    last_line = get_last_line(filename)
+                    last_date = get_last_date(last_line)
+                    current_date = line.split(",")[0].strip()
+                    debug(f"Current date:          [{current_date}]")
+                    if current_date == last_date:
+                        if line != last_line:
+                            debug(f"Writing1:\nline=[{line}]\nlast=[{last_line}]\ncurrent=[{current_date}]\nlast   =[{last_date}]")  # noqa pylint:disable=line-too-long
+                            writeln(filename, line)
+                        else:
+                            debug(f"Skipping1:\nline=[{line}]\nlast=[{last_line}]")  # noqa pylint:disable=line-too-long
+                    else:
+                        debug(f"Writing2:\nline=[{line}]\ncurrent=[{current_date}]\nlast   =[{last_date}]")   # noqa pylint:disable=line-too-long
+                        writeln(filename, line)
+                    handle_daily_stats(vehicle, number_of_vehicles)
 
             if 'None, None' in line:  # something gone wrong, retry
                 retries -= 1
                 log("Sleeping a minute")
                 time.sleep(60)
             else:
                 retries = 0  # successfully end while loop
+                write_last_run()
         except Exception as ex:  # pylint: disable=broad-except
             log('Exception: ' + str(ex))
             traceback.print_exc()

summary.py

@@ -3,6 +3,7 @@
 Simple Python3 script to make a summary of monitor.csv
 """
 import sys
+import os
 import configparser
 import traceback
 import time
@@ -86,6 +87,7 @@ def to_float(string):
 
 
 INPUT_CSV_FILE = Path("monitor.csv")
+INPUT_LASTRUN_FILE = Path("monitor.lastrun")
 OUTPUT_SPREADSHEET_NAME = "hyundai-kia-connect-monitor"
 LENCHECK = 1
 VIN = get_vin_arg()
@@ -359,14 +361,15 @@ def print_summary(prefix, current, values, split, factor):
         cost_str = cost_str.strip()
         prefix = prefix.replace("SHEET ", "")
         last_update_datetime = datetime.fromtimestamp(
-            INPUT_CSV_FILE.stat().st_mtime)
+            INPUT_LASTRUN_FILE.stat().st_mtime)
+        last_line = get_last_line(INPUT_CSV_FILE)
         day_info = last_update_datetime.strftime("%Y-%m-%d %H:%M %a")
         SHEET.batch_update([{
             'range': 'A1:B1',
             'values': [["Last update", f"{day_info}"]],
         }, {
             'range': 'A2:B2',
-            'values': [["Last entry", f"{prefix}"]],
+            'values': [["Last entry", f"{last_line}"]],
         }, {
             'range': 'A3:B3',
             'values': [["Last address", f"{location_str}"]],
@@ -738,6 +741,20 @@ def handle_line(linecount, split, prev_split, totals, last):
     return totals
 
 
+def get_last_line(filename):
+    """ get last line of filename """
+    with open(filename, "rb") as file:
+        try:
+            file.seek(-2, os.SEEK_END)
+            while file.read(1) != b'\n':
+                file.seek(-2, os.SEEK_CUR)
+        except OSError:
+            file.seek(0)
+        last_line = file.readline().decode().strip()
+        debug(f"{filename} last_line: [{last_line}]")
+        return last_line
+
+
 def summary():
     """ summary of monitor.csv file """
     with INPUT_CSV_FILE.open("r", encoding="utf-8") as inputfile: