From 476d53c30bbd25248f68f53c71aec51eedba03d3 Mon Sep 17 00:00:00 2001 From: Zuinige Rijder Date: Mon, 14 Oct 2024 16:41:08 +0200 Subject: [PATCH] Major update: introduction of running monitor.py infinitely --- README.md | 132 ++++++++++++++------- dailystats.py | 43 +++++-- debug.py | 1 + monitor.cfg | 3 + monitor.py | 255 +++++++++++++++++++++++++++++++--------- monitor_utils.py | 24 ++-- run_monitor_infinite.sh | 5 +- summary.py | 37 ++++-- 8 files changed, 368 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index 7e01b54..9050622 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,11 @@ - [debug.py](#debugpy) - [check\_monitor.py](#check_monitorpy) - [monitor\_utils.py](#monitor_utilspy) +- [logging\_config.ini](#logging_configini) - [Raspberry pi configuration](#raspberry-pi-configuration) + - [Running monitor.py infinitely and only running summary.py and dailystats.py when there is new cached server data received](#running-monitorpy-infinitely-and-only-running-summarypy-and-dailystatspy-when-there-is-new-cached-server-data-received) + - [Running monitor.py once](#running-monitorpy-once) + - [follow the last content of monitor.csv or run\_monitor\_infinite.log](#follow-the-last-content-of-monitorcsv-or-run_monitor_infinitelog) - [Examples](#examples) - [monitor.csv](#monitorcsv) - [python summary.py](#python-summarypy) @@ -72,7 +76,7 @@ Example screenshots showing the results in a Google Spreadsheet: - Summary ![alt text](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/examples/summary.py_GoogleSpreadsheet.png) -Run monitor.py e.g. once per hour (I use it on a Raspberry Pi and on Windows 10 with pure Python, but it will also run on Mac or a linux Operating System) and you can always check afterwards: +Run monitor.py e.g. once per hour or infinite (I use it on a Raspberry Pi and on Windows 10 with pure Python, but it will also run on Mac or a linux Operating System) and you can always check afterwards: - captured locations - odometer at specific day/hour - how much driven at a specific day @@ -122,14 +126,14 @@ I have installed the following packages (e.g. use python -m pip install "package pytz==2022.2.1 requests==2.28.1 -In hyundai_kia_connect_monitor summary.py also the following packages are used: +In hyundai_kia_connect_monitor summary.py and dailystats.py also the following packages are used: gspread==5.6.2 -If everything works, it's a matter of regularly collecting the information, for example by running the "python monitor.py" command once an hour. +If everything works, it's a matter of regularly collecting the information, for example by running the "python monitor.py" command once an hour or infinite. A server is of course best, I use a Raspberry Pi, but it can also regularly be done on a Windows 10 or Mac computer, provided the computer is on. -*Note: each time you run monitor.py it makes a snapshot of the latest server cache values. How more often you run it, the better charges and trips can be detected by summary.py.* +*Note: each time you run monitor.py it makes a snapshot of the latest server cache values. How more often you run it, the better charges and trips can be detected by summary.py. The easiest way is to run monitor.py infinite.* --- # configuration of gspread for "python summary.py sheetupdate" and "python dailystats.py sheetupdate" @@ -140,29 +144,29 @@ This [authentication configuration is described here](https://docs.gspread.org/e The summary.py and dailystats.py script uses access to the Google spreadsheets on behalf of a bot account using Service Account. Follow the steps in this link above, here is the summary of these steps: -- Enable API Access for a Project -- - Head to [Google Developers Console](https://console.developers.google.com/) and create a new project (or select the one you already have). -- - In the box labeled "Search for APIs and Services", search for "Google Drive API" and enable it. -- - In the box labeled "Search for APIs and Services", search for "Google Sheets API" and enable it -- For Bots: Using Service Account -- - Go to "APIs & Services > Credentials" and choose "Create credentials > Service account key". -- - Fill out the form -- - Click "Create" and "Done". -- - Press "Manage service accounts" above Service Accounts. -- - Press on : near recently created service account and select "Manage keys" and then click on "ADD KEY > Create new key". -- - Select JSON key type and press "Create". -- - You will automatically download a JSON file with credentials -- - Remember the path to the downloaded credentials json file. Also, in the next step you will need the value of client_email from this file. -- - Move the downloaded json file to ~/.config/gspread/service_account.json. Windows users should put this file to %APPDATA%\gspread\service_account.json. -- Setup a Google Spreasheet to be updated by sheetupdate -- - In Google Spreadsheet, create an empty Google Spreadsheet with the name: hyundai-kia-connect-monitor or monitor.VIN (latter if vin=VIN is given as parameter) -- - Go to your spreadsheet and share it with the client_email from the step above (inside service_account.json) -- - In Google Spreadsheet, create an empty Google Spreadsheet with the name: monitor.dailystats or monitor.dailystats.VIN (latter if vin=VIN is given as parameter). If you want nice diagrams, you can copy this [example Google spreadsheet](https://docs.google.com/spreadsheets/d/1WwdosLQ0ViTHct_kBSNddnd-H3IUc604_Tz-0dgYI9A/edit?usp=sharing) and change e.g. diagram titles into your own language. -- - Go to your spreadsheet and share it with the client_email from the step above (inside service_account.json) -- run "python summary.py sheetupdate" and if everything is correct, the hyundai-kia-connect-monitor or monitor.VIN spreadheet will be updated with a summary and the last 122 lines of standard output -- run "python dailystats.py sheetupdate" and if everything is correct, the monitor.dailystats or monitor.dailystats.VIN spreadheet will be updated with the last 122 lines of standard output -- configure to run "python summary.py sheetupdate" regularly, after having run "python monitor.py" -- configure to run "python dailystats.py sheetupdate" regularly, after having run "python summary.py sheetupdate" +1. Enable API Access for a Project + - Head to [Google Developers Console](https://console.developers.google.com/) and create a new project (or select the one you already have). + - In the box labeled "Search for APIs and Services", search for "Google Drive API" and enable it. + - In the box labeled "Search for APIs and Services", search for "Google Sheets API" and enable it +2. For Bots: Using Service Account + - Go to "APIs & Services > Credentials" and choose "Create credentials > Service account key". + - Fill out the form + - Click "Create" and "Done". + - Press "Manage service accounts" above Service Accounts. + - Press on : near recently created service account and select "Manage keys" and then click on "ADD KEY > Create new key". + - Select JSON key type and press "Create". + - You will automatically download a JSON file with credentials + - Remember the path to the downloaded credentials json file. Also, in the next step you will need the value of client_email from this file. + - Move the downloaded json file to ~/.config/gspread/service_account.json. Windows users should put this file to %APPDATA%\gspread\service_account.json. +3. Setup a Google Spreasheet to be updated by sheetupdate + - In Google Spreadsheet, create an empty Google Spreadsheet with the name: hyundai-kia-connect-monitor or monitor.VIN (latter if vin=VIN is given as parameter) + - Go to your spreadsheet and share it with the client_email from the step above (inside service_account.json) + - In Google Spreadsheet, create an empty Google Spreadsheet with the name: monitor.dailystats or monitor.dailystats.VIN (latter if vin=VIN is given as parameter). If you want nice diagrams, you can copy this [example Google spreadsheet](https://docs.google.com/spreadsheets/d/1WwdosLQ0ViTHct_kBSNddnd-H3IUc604_Tz-0dgYI9A/edit?usp=sharing) and change e.g. diagram titles into your own language. + - Go to your spreadsheet and share it with the client_email from the step above (inside service_account.json) +4. run "python summary.py sheetupdate" and if everything is correct, the hyundai-kia-connect-monitor or monitor.VIN spreadheet will be updated with a summary and the last 122 lines of standard output +5. run "python dailystats.py sheetupdate" and if everything is correct, the monitor.dailystats or monitor.dailystats.VIN spreadheet will be updated with the last 122 lines of standard output +6. configure to run "python summary.py sheetupdate" regularly, after having run "python monitor.py" +7. configure to run "python dailystats.py sheetupdate" regularly, after having run "python summary.py sheetupdate" --- # Translations @@ -211,6 +215,9 @@ odometer_metric = km include_regenerate_in_consumption = False consumption_efficiency_factor_dailystats = 1.0 consumption_efficiency_factor_summary = 1.0 +monitor_infinite = False +monitor_infinite_interval_minutes = 60 +monitor_execute_commands_when_something_written_or_error = ``` Explanation of the configuration items: @@ -221,15 +228,21 @@ Explanation of the configuration items: - pin: pincode of your bluelink account, required for CANADA, and potentially USA, otherwise pass a blank string - use_geocode: (default: True) find address with the longitude/latitude for each entry - use_geocode_email: (default: True) use email to avoid abuse of address lookup -- language: (default: en) the Bluelink App is reset to English for users who have set another language in the Bluelink App in Europe when using hyundai_kia_connect_api, you can configure another language as workaround. See Note 2 +- language: (default: en) the Bluelink App is reset to English for users who have set another language in the Bluelink App in Europe when using hyundai_kia_connect_api, you can configure another language as workaround. See Note 3 - odometer_metric, e.g. km or mi - include_regenerate_in_consumption, when set to True the regeneration is taken into account for the consumption calculation in daily stats. However, I think that the next 2 configuration items will better match the boardcomputer values. -- consumption_efficiency_factor_dailystats, see Note 1 -- consumption_efficiency_factor_summary, see Note 1 +- consumption_efficiency_factor_dailystats, see Note 2 +- consumption_efficiency_factor_summary, see Note 2 +- monitor_infinite, if set to True monitor.py keeps running using monitor_infinite_interval_minutes between getting cached server values +- monitor_infinite_interval_minutes, interval in minutes between getting cached server values +- monitor_execute_commands_when_something_written_or_error, when new cached server values are retrieved, the specified commands (separated by semicolon ;) are executed. See Note 1. + * example: monitor_execute_commands_when_something_written_or_error = python -u summary.py sheetupdate > summary.log;python -u dailystats.py sheetupdate > dailystats.log -*Note 1: I think that the consumption values ​​of the on-board computer are corrected with an efficiency number, e.g. 1 kWh of energy results in 0.9 kWh of real energy (losses when converting battery kWh by the car). So therefor I introduced an efficiency configuration factor in monitor.cfg, consumption_efficiency_factor_dailystats and consumption_efficiency_factor_summary. For example, when setting this to 0.9, 10% of the energy is lost during the conversion and is used in the consumption calculation. Default the values are 1.0, so no correction.* +*Note 1: in combination with infinite (monitor_infinite = True) summary.py and dailystats.py are only run when something is changed or error occurred (or once a day). You do not need to run summary.py and dailystats.py separately and it is only run when it is needed.* -*Note2: language is only implemented for Europe currently.* +*Note 2: I think that the consumption values ​​of the on-board computer are corrected with an efficiency number, e.g. 1 kWh of energy results in 0.9 kWh of real energy (losses when converting battery kWh by the car). So therefor I introduced an efficiency configuration factor in monitor.cfg, consumption_efficiency_factor_dailystats and consumption_efficiency_factor_summary. For example, when setting this to 0.9, 10% of the energy is lost during the conversion and is used in the consumption calculation. Default the values are 1.0, so no correction.* + +*Note 3: language is only implemented for Europe currently.* [For a list of language codes, see here.](https://www.science.co.il/language/Codes.php). Currently in Europe the Bluelink App shows the following languages: - "en" English @@ -308,8 +321,11 @@ Region Daily Limits Per Action Comments - KR ??? ``` +*Note that a Bluelink USA user has detected that there is a limit in the number of logins, not for the subsequent calls, therefore the option to run monitor.py infinite is a good choice. The monitor.py infinite does only login once per day and then the subsequent calls are done with the retrieved information. Unfortunately for Europe the total is restricted to about 200, so the number of logins does not matter. For the other regions I do not know the limit and behavior.* + So maybe you can capture more than once per hour, but you might run into the problem that you use too much API calls, especially when you also regularly use the Hyndai Bluelink or Kia UVO Connect app. You also can consider only to monitor between e.g. 6:00 and 22:00 (saves 1/3 of the calls). Dependent on your regular driving habit, choose the best option for you. Examples: +- run monitor.py infinite (monitor_infinite = True) with monitor_infinite_interval_minutes = 15 (means 96 requests per day and 1 login per day) - twice a day, e.g. 6.00 and 21:00, when you normally do not drive that late in the evening and charge in the night after 21:00 - each hour means 24 requests per day - each hour between 6:00 and 19:00 means 13 requests per day @@ -496,9 +512,9 @@ OUTPUT: The following information is written in the kml file: - document name: monitor + now in format "yyyymmdd hh:mm" - per placemark -- - name of place (index of Google Maps): datetime in format "yyyymmdd hh:mm" and optionally "C" when charging and "D" when in drive -- - description: SOC: nn% 12V: nn% ODO: odometer [(+distance since yyyymmdd hh:mm)] [drive] [charging] [plugged: n] -- - coordinate (longitude, latitude) + * name of place (index of Google Maps): datetime in format "yyyymmdd hh:mm" and optionally "C" when charging and "D" when in drive + * description: SOC: nn% 12V: nn% ODO: odometer [(+distance since yyyymmdd hh:mm)] [drive] [charging] [plugged: n] + * coordinate (longitude, latitude) Note: - the placemark lines are one-liners, so you can also search in monitor.kml @@ -538,16 +554,43 @@ Python script for testing: print when the odometer between two monitor.csv entri # monitor_utils.py Generic utility methods, used by the other python scripts. +--- +# logging_config.ini +Configuration of default logging and formatting of logging. + --- # Raspberry pi configuration -Example script [run_monitor_once.sh](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/run_monitor_once.sh) to run monitor.py on a linux based system. +Examples of running on Raspberry Pi or a linux based system. + +There are 2 different options to run monitor.py. Run infinitely or only once. The first one is more efficient, because then only summary.py and dailystats.py are run when there is new cached server data received. Also running infinitely does a login once per day, which is also more efficient and for Bluelink USA the rate limit is not restricted. + +## Running monitor.py infinitely and only running summary.py and dailystats.py when there is new cached server data received +Example script [run_monitor_infinite.sh](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/run_monitor_infinite.sh) to run monitor.py infinitely + +Steps: +1. create a directory hyundai_kia_connect_monitor in your home directory +2. copy hyundai_kia_connect_api as subdirectory of directory hyundai_kia_connect_monitor +3. copy run_monitor_infinite.sh, monitor.py, monitor.cfg, monitor.translations.csv, monitor_utils.py, summary.py, summary.cfg, dailystats.py and logging_config.ini +4. change inside monitor.cfg the appropriate hyundai_kia_connect settings, e.g. monitor_infinite = True and monitor_execute_commands_when_something_written_or_error = python -u summary.py sheetupdate > summary.log;python -u dailystats.py sheetupdate > dailystats.log +5. chmod + x run_monitor_infinite.sh + +Add to your crontab to run once per hour to restart after crashes or reboot (crontab -e) +``` +9 * * * * ~/hyundai_kia_connect_monitor/run_monitor_infinite.sh >> ~/hyundai_kia_connect_monitor/crontab_run_monitor_infinite.log 2>&1 +@reboot sleep 125 && ~/hyundai_kia_connect_monitor/run_monitor_infinite.sh >> ~/hyundai_kia_connect_monitor/crontab_run_monitor_infinite.log 2>&1 +``` + +*Note: there is a limit in the number of request per country, but 1 request per hour should not hamper using the Bluelink or UVO Connect App at the same time* + +## Running monitor.py once +Example script [run_monitor_once.sh](https://raw.githubusercontent.com/ZuinigeRijder/hyundai_kia_connect_monitor/main/run_monitor_once.sh) to run monitor.py once on a linux based system. Steps: -- create a directory hyundai_kia_connect_monitor in your home directory -- copy hyundai_kia_connect_api as subdirectory of directory hyundai_kia_connect_monitor -- copy run_monitor_once.sh, monitor.py and monitor.cfg in the hyundai_kia_connect_monitor directory -- change inside monitor.cfg the hyundai_kia_connect settings -- chmod + x run_monitor_once.sh +1. create a directory hyundai_kia_connect_monitor in your home directory +2. copy hyundai_kia_connect_api as subdirectory of directory hyundai_kia_connect_monitor +3. copy run_monitor_once.sh, monitor.py, monitor.cfg, monitor.translations.csv, monitor_utils.py and logging_config.ini in the hyundai_kia_connect_monitor directory +4. change inside monitor.cfg the appropriate hyundai_kia_connect settings, e.g. monitor_infinite = False +5. chmod + x run_monitor_once.sh Add the following line in your crontab -e to run it once per hour (crontab -e): ``` @@ -561,6 +604,13 @@ Add the following line in your crontab -e to run it every 15 minutes between 6 a *Note: there is a limit in the number of request per country, but 1 request per hour should not hamper using the Bluelink or UVO Connect App at the same time* + +## follow the last content of monitor.csv or run_monitor_infinite.log + +There is another python tool to follow the content of a file on a server and send it to a Google Sheet with the same filename. [See tail2GoogleSheet](https://github.com/ZuinigeRijder/tail2GoogleSheet?tab=readme-ov-file#example-crontab-to-run-on-raspberry-pi-or-another-linux-system). + +Another example is [tail_run_monitor_infinite.log.sh](https://raw.githubusercontent.com/ZuinigeRijder/tail2GoogleSheet/main/examples/tail_run_monitor_infinite.log.sh) which is following ~/hyundai_kia_connect_monitor/run_monitor_infinite.log + --- # Examples diff --git a/dailystats.py b/dailystats.py index 40cdd93..819d3eb 100644 --- a/dailystats.py +++ b/dailystats.py @@ -2,25 +2,29 @@ """ Simple Python3 script to make a dailystats overview """ +# pylint:disable=logging-fstring-interpolation,logging-not-lazy import configparser from dataclasses import dataclass +import logging +import logging.config +from os import path import re import sys import traceback from datetime import datetime from pathlib import Path from collections import deque +import typing from dateutil.relativedelta import relativedelta import gspread from monitor_utils import ( float_to_string_no_trailing_zero, get, get_filepath, - log, arg_has, get_vin_arg, safe_divide, - sleep, + sleep_a_minute, split_on_comma, split_output_to_sheet_float_list, to_int, @@ -33,17 +37,21 @@ split_output_to_sheet_list, ) +SCRIPT_DIRNAME = path.abspath(path.dirname(__file__)) +logging.config.fileConfig(f"{SCRIPT_DIRNAME}/logging_config.ini") +D = arg_has("debug") +if D: + logging.getLogger().setLevel(logging.DEBUG) + # Initializing a queue for about 30 days MAX_QUEUE_LEN = 122 PRINTED_OUTPUT_QUEUE: deque[str] = deque(maxlen=MAX_QUEUE_LEN) -D = arg_has("debug") - def dbg(line: str) -> bool: """print line if debugging""" if D: - print(line) + logging.debug(line) return D # just to make a lazy evaluation expression possible @@ -59,11 +67,12 @@ def dbg(line: str) -> bool: KEYWORD_ERROR = True if KEYWORD_ERROR or arg_has("help"): - print("Usage: python dailystats.py [sheetupdate] [vin=VIN]") + logging.info("Usage: python dailystats.py [sheetupdate] [vin=VIN]") exit() SHEETUPDATE = arg_has("sheetupdate") OUTPUT_SPREADSHEET_NAME = "monitor.dailystats" +SHEET: typing.Any = None DAILYSTATS_CSV_FILE = Path("monitor.dailystats.csv") TRIPINFO_CSV_FILE = Path("monitor.tripinfo.csv") @@ -389,7 +398,7 @@ def get_trip_for_datetime( trip_datetime = splitted_line[0] date_elements = trip_datetime.split(" ") if len(date_elements) < 2: - log(f"Warning: skipping unexpected line: {line}") + logging.warning(f"Warning: skipping unexpected line: {line}") reverse_read_next_summary_trip_line() continue trip_date = date_elements[0] @@ -856,6 +865,7 @@ def print_output_queue() -> None: # main program +RETRIES = -1 if SHEETUPDATE: RETRIES = 2 while RETRIES > 0: @@ -864,12 +874,14 @@ def print_output_queue() -> None: spreadsheet = gc.open(OUTPUT_SPREADSHEET_NAME) SHEET = spreadsheet.sheet1 SHEET.batch_clear(["A:G", "N:V"]) - RETRIES = 0 + RETRIES = -1 except Exception as ex: # pylint: disable=broad-except - log("Exception: " + str(ex)) + logging.error("Exception: " + str(ex)) traceback.print_exc() - RETRIES = sleep(RETRIES) + RETRIES = sleep_a_minute(RETRIES) +if RETRIES != -1: + exit(-1) reverse_print_dailystats(totals=True) # do the total dailystats summary first print_dailystats( @@ -886,13 +898,18 @@ def print_output_queue() -> None: summary_tripinfo() # then the total tripinfo reverse_print_dailystats(totals=False) # and then dailystats +RETRIES = -1 if SHEETUPDATE: RETRIES = 2 while RETRIES > 0: try: print_output_queue() - RETRIES = 0 + RETRIES = -1 except Exception as ex: # pylint: disable=broad-except - log("Exception: " + str(ex)) + logging.error("Exception: " + str(ex)) traceback.print_exc() - RETRIES = sleep(RETRIES) + RETRIES = sleep_a_minute(RETRIES) + +if RETRIES == -1: + exit(0) +exit(-1) diff --git a/debug.py b/debug.py index 22c2d6f..8f59e54 100644 --- a/debug.py +++ b/debug.py @@ -3,6 +3,7 @@ import configparser from datetime import datetime import logging +import logging.config from hyundai_kia_connect_api import VehicleManager, Vehicle from monitor_utils import get_filepath diff --git a/monitor.cfg b/monitor.cfg index 9812e78..608d222 100644 --- a/monitor.cfg +++ b/monitor.cfg @@ -11,3 +11,6 @@ odometer_metric = km include_regenerate_in_consumption = False consumption_efficiency_factor_dailystats = 1.0 consumption_efficiency_factor_summary = 1.0 +monitor_infinite = False +monitor_infinite_interval_minutes = 60 +monitor_execute_commands_when_something_written_or_error = diff --git a/monitor.py b/monitor.py index 3f511d6..e1b38cf 100644 --- a/monitor.py +++ b/monitor.py @@ -30,14 +30,19 @@ - charging pattern over time - visited places """ +# pylint:disable=logging-fstring-interpolation,logging-not-lazy +from os import path import re +import subprocess import sys import io import configparser import traceback import logging +import logging.config from pathlib import Path -from datetime import datetime +from datetime import datetime, timedelta +import typing from dateutil.relativedelta import relativedelta from hyundai_kia_connect_api import VehicleManager, Vehicle, exceptions from monitor_utils import ( @@ -49,11 +54,17 @@ get_safe_datetime, get_safe_float, km_to_mile, - log, - sleep, + same_day, + sleep_a_minute, + sleep_seconds, to_int, ) +SCRIPT_DIRNAME = path.abspath(path.dirname(__file__)) +logging.config.fileConfig(f"{SCRIPT_DIRNAME}/logging_config.ini") +D = arg_has("debug") +if D: + logging.getLogger().setLevel(logging.DEBUG) # keep forceupdate and cacheupdate as keyword, but do nothing with them KEYWORD_LIST = ["forceupdate", "cacheupdate", "debug"] @@ -67,11 +78,6 @@ print("Usage: python monitor.py") exit() -D = arg_has("debug") -if D: - logging.basicConfig(level=logging.DEBUG) - - # == read monitor in monitor.cfg =========================== parser = configparser.ConfigParser() parser.read(get_filepath("monitor.cfg")) @@ -86,18 +92,28 @@ USE_GEOCODE_EMAIL = monitor_settings["use_geocode_email"].lower() == "true" LANGUAGE = monitor_settings["language"] ODO_METRIC = get(monitor_settings, "odometer_metric", "km").lower() +MONITOR_INFINITE = get(monitor_settings, "monitor_infinite", "False").lower() == "true" +MONITOR_INFINITE_INTERVAL_MINUTES = to_int( + get(monitor_settings, "monitor_infinite_interval_minutes", "60") +) +MONITOR_EXECUTE_COMMANDS_WHEN_SOMETHING_WRITTEN_OR_ERROR = get( + monitor_settings, "monitor_execute_commands_when_something_written_or_error", "" +) + +MONITOR_SOMETHING_WRITTEN_OR_ERROR = False # == subroutines ============================================================= def dbg(line: str) -> bool: """print line if debugging""" if D: - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ": " + line) + logging.debug(line) return D # just to make a lazy evaluation expression possible def writeln(filename: str, string: str) -> None: """append line at monitor text file with end of line character""" + global MONITOR_SOMETHING_WRITTEN_OR_ERROR # pylint:disable=global-statement _ = D and dbg(string) monitor_csv_file = Path(filename) write_header = False @@ -112,6 +128,7 @@ def writeln(filename: str, string: str) -> None: ) file.write(string) file.write("\n") + MONITOR_SOMETHING_WRITTEN_OR_ERROR = True def to_miles_needed(vehicle: Vehicle) -> bool: @@ -121,6 +138,7 @@ def to_miles_needed(vehicle: Vehicle) -> bool: def handle_daily_stats(vehicle: Vehicle, number_of_vehicles: int) -> None: """handle daily stats""" + global MONITOR_SOMETHING_WRITTEN_OR_ERROR # pylint:disable=global-statement daily_stats = vehicle.daily_stats if len(daily_stats) == 0: _ = D and dbg("No daily stats") @@ -183,6 +201,7 @@ def handle_daily_stats(vehicle: Vehicle, number_of_vehicles: int) -> None: ) file.write(full_line) file.write("\n") + MONITOR_SOMETHING_WRITTEN_OR_ERROR = True last_line = line else: if D: @@ -220,6 +239,16 @@ def write_last_run( file.write(f"{today_daily_stats_line}\n") +def append_error_to_last_run(error_string: str) -> None: + """append error to last run""" + filename = "monitor.lastrun" + if MANAGER and MANAGER.vehicles and len(MANAGER.vehicles) > 1: + filename = "monitor." + MANAGER.vehicles[0].VIN + ".lastrun" + lastrun_file = Path(filename) + with lastrun_file.open("a", encoding="utf-8") as file: + file.write(f"{error_string}\n") + + def handle_day_trip_info( manager: VehicleManager, vehicle: Vehicle, @@ -229,6 +258,7 @@ def handle_day_trip_info( last_hhmmss: str, ) -> tuple[str, str]: """handle_day_trip_info""" + global MONITOR_SOMETHING_WRITTEN_OR_ERROR # pylint:disable=global-statement for day in month_trip_info.day_list: yyyymmdd = day.yyyymmdd if yyyymmdd >= last_date: @@ -255,6 +285,7 @@ def handle_day_trip_info( _ = D and dbg(f"Writing tripinfo line:[{line}]") file.write(line) file.write("\n") + MONITOR_SOMETHING_WRITTEN_OR_ERROR = True last_date = yyyymmdd last_hhmmss = hhmmss else: @@ -324,7 +355,7 @@ def handle_one_vehicle( try: handle_trip_info(manager, vehicle, number_of_vehicles) except Exception as ex: # pylint: disable=broad-except - log("Warning: handle_trip_info Exception: " + str(ex)) + logging.warning("Warning: handle_trip_info Exception: " + str(ex)) traceback.print_exc() geocode = "" @@ -369,17 +400,18 @@ def handle_one_vehicle( ) if newest_updated_at < previous_updated_at: utcoffset = newest_updated_at.utcoffset() - newest_updated_at_corrected = newest_updated_at + utcoffset - if newest_updated_at_corrected >= previous_updated_at: - log( - f"fixed newest_updated_at: old: {newest_updated_at} new: {newest_updated_at_corrected} previous: {previous_updated_at}" # noqa - ) - newest_updated_at = newest_updated_at_corrected + if utcoffset: + newest_updated_at_corrected = newest_updated_at + utcoffset + if newest_updated_at_corrected >= previous_updated_at: + _ = D and dbg( + f"fixed newest_updated_at: old: {newest_updated_at} new: {newest_updated_at_corrected} previous: {previous_updated_at}" # noqa + ) + newest_updated_at = newest_updated_at_corrected newest_updated_at = max(newest_updated_at, previous_updated_at) line = f"{newest_updated_at}, {location_longitude}, {location_latitude}, {vehicle.engine_is_running}, {vehicle.car_battery_percentage}, {float_to_string_no_trailing_zero(odometer)}, {vehicle.ev_battery_percentage}, {vehicle.ev_battery_is_charging}, {vehicle.ev_battery_is_plugged_in}, {geocode}, {ev_driving_range}" # noqa if "None, None" in line: # something gone wrong, retry - log(f"Skipping Unexpected line: {line}") + logging.warning(f"Skipping Unexpected line: {line}") return True # exit subroutine with error if line != last_line: @@ -407,7 +439,7 @@ def handle_one_vehicle( return False -def handle_exception(ex: Exception, retries: int, stacktrace=False) -> int: +def handle_exception(ex: Exception, retries: int, stacktrace=False) -> tuple[int, str]: """ If an error is found, an exception is raised. retCode known values: @@ -423,65 +455,172 @@ def handle_exception(ex: Exception, retries: int, stacktrace=False) -> int: - 9999: "Undefined Error - Response timeout" """ exception_str = str(ex) - log(f"Exception: {exception_str}") + error_string = f"Exception: {exception_str}" + logging.warning(error_string) if stacktrace and "Service Temporary Unavailable" not in exception_str: traceback.print_exc() - retries = sleep(retries) - return retries + retries = sleep_a_minute(retries) + return retries, error_string + + +def run_commands(): + """run_commands""" + commands = MONITOR_EXECUTE_COMMANDS_WHEN_SOMETHING_WRITTEN_OR_ERROR.split(";") + count = 0 + for command in commands: + count += 1 + command = command.strip() + if len(command) > 0: + _ = D and dbg(f"full command: {command}") + output_filename = f"command{count}.log" + open_mode = "w" + if ">>" in command: # append to file + open_mode = "a" + command = command.replace(">>", ">") + if ">" in command: # write to file + splitted = command.split(">") + command = splitted[0].strip() + output_filename = splitted[1].strip() + _ = D and dbg(f"command: {command}") + _ = D and dbg(f"output_filename: {output_filename}") + _ = D and dbg(f"open_mode: {open_mode}") + try: + with open(output_filename, open_mode, encoding="utf-8") as outfile: + process = subprocess.run( + command, + check=True, + shell=True, + stderr=subprocess.STDOUT, + stdout=outfile, + ) + if process.returncode != 0: + logging.error( + f"Error in running {command}: returncode {process.returncode}" + ) + except Exception as ex: # pylint: disable=broad-except + (_, error_string) = handle_exception(ex, 1, True) + logging.error(f"Error in running {command}: {error_string}") -def handle_vehicles() -> None: +# get MANAGER only once +MANAGER: typing.Union[VehicleManager, None] = None + + +def handle_vehicles(login: bool) -> bool: """handle vehicles""" - retries = 2 + global MANAGER, MONITOR_SOMETHING_WRITTEN_OR_ERROR # pylint:disable=global-statement # noqa + MONITOR_SOMETHING_WRITTEN_OR_ERROR = False + retries = 15 # retry for maximum of 15 minutes (15 x 60 seconds sleep) while retries > 0: + error_string = "" try: - # get information and add to comma separated file - manager = VehicleManager( - region=int(REGION), - brand=int(BRAND), - username=USERNAME, - password=PASSWORD, - pin=PIN, - geocode_api_enable=USE_GEOCODE, - geocode_api_use_email=USE_GEOCODE_EMAIL, - language=LANGUAGE, - ) - manager.check_and_refresh_token() - manager.update_all_vehicles_with_cached_state() # needed >= 2.0.0 - - number_of_vehicles = len(manager.vehicles) - error = False - for _, vehicle in manager.vehicles.items(): - error = handle_one_vehicle(manager, vehicle, number_of_vehicles) - if error: # something gone wrong, exit loop - break - - if error: # something gone wrong, retry - retries -= 1 - sleep(retries) + if login: + logging.info("Login using VehicleManager") + # get information and add to comma separated file + MANAGER = VehicleManager( + region=int(REGION), + brand=int(BRAND), + username=USERNAME, + password=PASSWORD, + pin=PIN, + geocode_api_enable=USE_GEOCODE, + geocode_api_use_email=USE_GEOCODE_EMAIL, + language=LANGUAGE, + ) + + if MANAGER: + MANAGER.check_and_refresh_token() + MANAGER.update_all_vehicles_with_cached_state() # needed >= 2.0.0 + + error = False + number_of_vehicles = len(MANAGER.vehicles) + for _, vehicle in MANAGER.vehicles.items(): + error = handle_one_vehicle(MANAGER, vehicle, number_of_vehicles) + if error: # something gone wrong, exit vehicles loop + error_string = "Error occurred in handle_one_vehicle()" + break + + if error or MANAGER is None: # something gone wrong, retry + retries = sleep_a_minute(retries) else: - retries = 0 # successfully end while loop + retries = -1 # successfully end while loop without error except exceptions.AuthenticationError as ex: - retries = handle_exception(ex, retries) + (retries, error_string) = handle_exception(ex, retries) except exceptions.RateLimitingError as ex: - retries = handle_exception(ex, retries) + (retries, error_string) = handle_exception(ex, retries) except exceptions.NoDataFound as ex: - retries = handle_exception(ex, retries) + (retries, error_string) = handle_exception(ex, retries) except exceptions.DuplicateRequestError as ex: - retries = handle_exception(ex, retries) + (retries, error_string) = handle_exception(ex, retries) except exceptions.RequestTimeoutError as ex: - retries = handle_exception(ex, retries) + (retries, error_string) = handle_exception(ex, retries) # Not yet available, so workaround for now in handle_exception # except exceptions.ServiceTemporaryUnavailable as ex: # retries = handle_exception(ex, retries) except exceptions.InvalidAPIResponseError as ex: - retries = handle_exception(ex, retries, True) + (retries, error_string) = handle_exception(ex, retries, True) except exceptions.APIError as ex: - retries = handle_exception(ex, retries, True) + (retries, error_string) = handle_exception(ex, retries, True) except exceptions.HyundaiKiaException as ex: - retries = handle_exception(ex, retries, True) + (retries, error_string) = handle_exception(ex, retries, True) + except Exception as ex: # pylint: disable=broad-except + (retries, error_string) = handle_exception(ex, retries, True) + + error = retries != -1 + if error: + logging.error(error_string) + MONITOR_SOMETHING_WRITTEN_OR_ERROR = True # indicate error + try: + append_error_to_last_run(error_string) except Exception as ex: # pylint: disable=broad-except - retries = handle_exception(ex, retries, True) + (retries, error_string) = handle_exception(ex, retries, True) + else: + MONITOR_SOMETHING_WRITTEN_OR_ERROR = False + + return error + + +def monitor(): + """monitor""" + global MONITOR_SOMETHING_WRITTEN_OR_ERROR # pylint:disable=global-statement + prev_time = datetime.now() - timedelta(days=1.0) # once per day login + not_done_once = False + error_count = 1 + while not_done_once or MONITOR_INFINITE: + not_done_once = True + current_time = datetime.now() + # login when 4x15 minutes subsequent errors or once at beginning of new day + login = (error_count % 4 == 0) or not same_day(prev_time, current_time) + MONITOR_SOMETHING_WRITTEN_OR_ERROR = login # run commands at least once per day + error = handle_vehicles(login) + if error: + logging.error(f"error count: {error_count}") + error_count += 1 + else: + error_count = 1 + if ( + MONITOR_SOMETHING_WRITTEN_OR_ERROR + and len(MONITOR_EXECUTE_COMMANDS_WHEN_SOMETHING_WRITTEN_OR_ERROR) > 0 + ): + logging.info( + "New data added or first run today or errors, running configured commands" # noqa + ) + run_commands() + + if MONITOR_INFINITE: + prev_time = current_time + + # calculate number of seconds to sleep + next_time = prev_time + timedelta(minutes=MONITOR_INFINITE_INTERVAL_MINUTES) + current_time = datetime.now() + delta = next_time - current_time + total_seconds = delta.total_seconds() + if total_seconds > 0: + sleep_seconds(total_seconds) + + if error_count > 96: # more than 24 hours subsequnt errors + logging.error("Too many subsequent errors occurred, exiting monitor.py") + sys.exit(-1) -handle_vehicles() # do the work +monitor() diff --git a/monitor_utils.py b/monitor_utils.py index b0cb4e8..94e3473 100644 --- a/monitor_utils.py +++ b/monitor_utils.py @@ -2,8 +2,11 @@ """ monitor utils """ +# pylint:disable=logging-fstring-interpolation import configparser import errno +import logging +import logging.config import sys import os from datetime import datetime, timezone @@ -12,11 +15,6 @@ from typing import Generator -def log(msg: str) -> None: - """log a message prefixed with a date/time format yyyymmdd hh:mm:ss""" - print(datetime.now().strftime("%Y%m%d %H:%M:%S") + ": " + msg) - - def arg_has(string: str) -> bool: """arguments has string""" for i in range(1, len(sys.argv)): @@ -37,12 +35,18 @@ def get_vin_arg() -> str: return "" -def sleep(retries: int) -> int: - """sleep when retries > 0""" +def sleep_seconds(seconds: int) -> None: + """sleep seconds""" + logging.debug(f"Sleeping {seconds} seconds") + time.sleep(seconds) + + +def sleep_a_minute(retries: int) -> int: + """sleep a minute when retries > 0""" if retries > 0: retries -= 1 if retries > 0: - log("Sleeping a minute") + logging.info("Sleeping a minute") time.sleep(60) return retries @@ -281,7 +285,9 @@ def read_translations() -> dict: linecount += 1 split = split_on_comma(line) if len(split) < 15: - log(f"Error: unexpected translation csvline {linecount}: {line}") + logging.error( + f"Error: unexpected translation csvline {linecount}: {line}" + ) continue key = split[0] translation = split[1] diff --git a/run_monitor_infinite.sh b/run_monitor_infinite.sh index c98de49..39fcff5 100644 --- a/run_monitor_infinite.sh +++ b/run_monitor_infinite.sh @@ -5,8 +5,9 @@ # Assumption is that monitor.cfg is configured to run infinite (monitor_infinite = True) and # monitor_execute_commands_when_something_written_or_error is configured to run summary.py and/or dailystats.py, e.g. # monitor_execute_commands_when_something_written_or_error = python -u summary.py sheetupdate > summary.log;python -u dailystats.py sheetupdate > dailystats.log -# Add to your crontab to run once per hour to restart after crashes or reboot -# 0 * * * * ~/hyundai_kia_connect_monitor/run_monitor_infinite.sh >> ~/hyundai_kia_connect_monitor/crontab_run_monitor_infinite.log 2>&1 +# Add to your crontab to run once per hour to restart after crashes or reboot (crontab -e) +# 9 * * * * ~/hyundai_kia_connect_monitor/run_monitor_infinite.sh >> ~/hyundai_kia_connect_monitor/crontab_run_monitor_infinite.log 2>&1 +# @reboot sleep 125 && ~/hyundai_kia_connect_monitor/run_monitor_infinite.sh >> ~/hyundai_kia_connect_monitor/crontab_run_monitor_infinite.log 2>&1 # --------------------------------------------------------------- script_name=$(basename -- "$0") cd ~/hyundai_kia_connect_monitor diff --git a/summary.py b/summary.py index 777e9c8..2c9a596 100644 --- a/summary.py +++ b/summary.py @@ -2,8 +2,12 @@ """ Simple Python3 script to make a summary of monitor.csv """ +# pylint:disable=logging-fstring-interpolation,logging-not-lazy from copy import deepcopy from io import TextIOWrapper +import logging +import logging.config +from os import path import sys import configparser import traceback @@ -11,16 +15,16 @@ from datetime import datetime from pathlib import Path from collections import deque +import typing import gspread from dateutil import parser from monitor_utils import ( get_filepath, - log, arg_has, get, get_vin_arg, safe_divide, - sleep, + sleep_a_minute, split_on_comma, to_int, to_float, @@ -36,13 +40,17 @@ float_to_string_no_trailing_zero, ) +SCRIPT_DIRNAME = path.abspath(path.dirname(__file__)) +logging.config.fileConfig(f"{SCRIPT_DIRNAME}/logging_config.ini") D = arg_has("debug") +if D: + logging.getLogger().setLevel(logging.DEBUG) def dbg(line: str) -> bool: """print line if debugging""" if D: - print(line) + logging.debug(line) return D # just to make a lazy evaluation expression possible @@ -152,6 +160,7 @@ def dbg(line: str) -> bool: LAST_OUTPUT_QUEUE_MAX_LEN = 122 LAST_OUTPUT_QUEUE: deque[str] = deque(maxlen=LAST_OUTPUT_QUEUE_MAX_LEN) +SHEET: typing.Any = None SHEET_ROW_A = "" SHEET_ROW_B = "" @@ -336,7 +345,9 @@ def sheet_append_first_rows(row_a: str, row_b: str) -> None: len_a = len(row_a_values) len_b = len(row_b_values) if len_a != len_b: - log(f"ERROR: sheet_append_first_rows, length A {len_a} != length B {len_b}") + logging.error( + f"ERROR: sheet_append_first_rows, length A {len_a} != length B {len_b}" + ) return # nothing to do array = [] @@ -459,7 +470,7 @@ def get_splitted_list_item(the_list: list[str], index: int) -> list[str]: if not MONITOR_CSV_FILENAME.is_file(): - log(f"ERROR: file does not exist: {MONITOR_CSV_FILENAME}") + logging.error(f"ERROR: file does not exist: {MONITOR_CSV_FILENAME}") sys.exit(-1) MONITOR_CSV_FILE: TextIOWrapper = MONITOR_CSV_FILENAME.open("r", encoding="utf-8") @@ -690,6 +701,9 @@ def print_summary( location_last_updated_at = get_splitted_list_item(lastrun_lines, 3) last_upd_dt = last_updated_at[1] location_last_upd_dt = location_last_updated_at[1] + if len(lastrun_lines) > 6: # error occurred, line 6 contains error_string + last_upd_dt = f"{last_upd_dt} ERROR: {lastrun_lines[6]}" + SHEET_ROW_A = f"{TR.last_run},{TR.vehicle_upd},{TR.gps_update},{TR.last_entry},{TR.last_address},{TR.odometer} {ODO_METRIC},{TR.driven} {ODO_METRIC},+kWh,-kWh,{ODO_METRIC}/kWh,kWh/100{ODO_METRIC},{TR.cost} {COST_CURRENCY},{TR.soc_perc},{TR.avg} {TR.soc_perc},{TR.min} {TR.soc_perc},{TR.max} {TR.soc_perc},{TR.volt12_perc},{TR.avg} {TR.volt12_perc},{TR.min} {TR.volt12_perc},{TR.max} {TR.volt12_perc},{TR.charges},{TR.trips},{TR.ev_range}" # noqa SHEET_ROW_B = f"{last_run_dt},{last_upd_dt},{location_last_upd_dt},{last_line},{location_str},{odo:.1f},{float_to_string_no_trailing_zero(delta_odo)},{float_to_string_no_trailing_zero(charged_kwh)},{float_to_string_no_trailing_zero(discharged_kwh)},{km_mi_per_kwh_str},{kwh_per_km_mi_str},{cost_str},{float_to_string_no_trailing_zero(t_soc_cur)},{float_to_string_no_trailing_zero(t_soc_avg)},{float_to_string_no_trailing_zero(t_soc_min)},{float_to_string_no_trailing_zero(t_soc_max)},{t_volt12_cur},{t_volt12_avg},{t_volt12_min},{t_volt12_max},{t_charges},{t_trips},{ev_range}" # noqa else: @@ -1027,7 +1041,7 @@ def handle_line( def summary(): """summary of monitor.csv file""" if not MONITOR_CSV_FILENAME.is_file(): - log(f"ERROR: file does not exist: {MONITOR_CSV_FILENAME}") + logging.error(f"ERROR: file does not exist: {MONITOR_CSV_FILENAME}") return prev_line = "" @@ -1088,6 +1102,7 @@ def summary(): DAY_CSV_FILE.close() TRIP_CSV_FILE.close() +RETRIES = -1 if SHEETUPDATE: RETRIES = 2 while RETRIES > 0: @@ -1098,8 +1113,12 @@ def summary(): SHEET.clear() sheet_append_first_rows(SHEET_ROW_A, SHEET_ROW_B) print_output_queue() - RETRIES = 0 + RETRIES = -1 except Exception as ex: # pylint: disable=broad-except - log("Exception: " + str(ex)) + logging.info("Exception: " + str(ex)) traceback.print_exc() - RETRIES = sleep(RETRIES) + RETRIES = sleep_a_minute(RETRIES) + +if RETRIES == -1: + exit(0) +exit(-1)