diff --git a/README.md b/README.md index 6334c15..e746f22 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,11 @@ from meteomoris import get_main_message Meteo.CHECK_INTERNET = True # Will check if there is internet Meteo.EXIT_ON_NO_INTERNET = True # Will exit if no internet +Meteo.ALREADY_CHECKED_INTERNET = False +Meteo.EXIT_ON_NO_INTERNET = True +Meteo.CHECK_INTERNET = True +Meteo.DEBUG = False # used during development +Meteo.today = "2023-11-10" # If you want to override cache data Meteo.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0', 'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -293,7 +298,6 @@ Meteo.headers = { 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1' } # Redefine default headers here -Meteo.DEBUG = True # during development print(get_main_message()) ``` # Installing @@ -335,10 +339,14 @@ Run # Changelog + +## 2.8.1 + +- Feat cache + ## 2.8.0 -- uvindex -- +- Uv index ## 2.7.8 diff --git a/meteomoris/__init__.py b/meteomoris/__init__.py index e64e8ff..a0a9aaf 100644 --- a/meteomoris/__init__.py +++ b/meteomoris/__init__.py @@ -19,4 +19,4 @@ get_latest = Meteo.get_latest get_uvindex = Meteo.get_uvindex -__version__ = "2.8.0" +__version__ = "2.8.1" diff --git a/meteomoris/meteo.py b/meteomoris/meteo.py index f1f39bb..e246fee 100644 --- a/meteomoris/meteo.py +++ b/meteomoris/meteo.py @@ -16,6 +16,10 @@ import calendar import http.client as httplib import re + import site + import os + import http.client as httplib + import json except Exception as e: pass @@ -26,27 +30,28 @@ # r = requests.get(url, allow_redirects=True) # open(path, 'wb').write(r.content) +def site_package_path(): + paths = site.getsitepackages() + for p in paths: + if p[len(p)-len('site-packages'):] == "site-packages": + return p + + raise Exception("Path could not be found") + +def cache_path(): + return os.path.join(site_package_path(), "meteomoris_cache.json") -def internet_present(): - console = Console() - with console.status("Checking internet ...", spinner="material"): - - conn = httplib.HTTPSConnection("8.8.8.8", timeout=5) - try: - conn.request("HEAD", "/") - return True - except Exception: - return False - finally: - conn.close() class Meteo: + ALREADY_CHECKED_INTERNET = False EXIT_ON_NO_INTERNET = True - CHECK_INTERNET = False + CHECK_INTERNET = True DEBUG = False + + today = str(datetime.date.today()) # Decoupled for tests try: # thanks pypa for broken tricks like this @@ -70,62 +75,135 @@ class Meteo: "Upgrade-Insecure-Requests": "1", } + # --- Utilities --- + @classmethod + def verify_cache_exists(cls): + if not os.path.exists(cache_path()): + with open(cache_path(), 'w+') as f: + json.dump({}, f) + + @classmethod + def get_cache_data(cls): + cls.verify_cache_exists() + with open(cache_path()) as f: + data = json.load(f) + return data + + @classmethod + def get_from_cache(cls, key): + ''' + keep cache day only per day for now + until we add functionalities to clear + the cache for all + ''' + cache_data = cls.get_cache_data() + if cls.today not in cache_data: + cache_data = {} # clear cache + cache_data[str(cls.today)] = {} + with open(cache_path(), 'w+') as f: + json.dump(cache_data, f) + return False + if key not in cache_data[cls.today]: + return False + return cache_data[cls.today][key] + + @classmethod + def add_to_cache(cls, key, data): + cache_data = cls.get_cache_data() + cache_data[cls.today][key] = data + + with open(cache_path(), 'w+') as f: + json.dump(cache_data, f) + + @classmethod + def internet_present(cls): + console = Console() + + with console.status("Checking internet ...", spinner="aesthetic"): + + conn = httplib.HTTPSConnection("8.8.8.8", timeout=5) + try: + conn.request("HEAD", "/") + return True + except Exception: + return False + finally: + cls.ALREADY_CHECKED_INTERNET = True + conn.close() + @classmethod def check_internet(cls): if cls.CHECK_INTERNET: - if not internet_present(): - print("No internet") - if cls.EXIT_ON_NO_INTERNET: - sys.exit() + if not cls.ALREADY_CHECKED_INTERNET: + if not cls.internet_present(): + print("No internet") + if cls.EXIT_ON_NO_INTERNET: + sys.exit() + # --- Data --- @classmethod def get_weekforecast(cls, day=None, print=False): print_ = print - cls.check_internet() - URL = "http://metservice.intnet.mu" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + return_data = None + try: + cache = cls.get_from_cache('weekforecast') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + return_data = cache + else: + cls.check_internet() - w_forecast = soup.find(attrs={"class": "daysforecast"}) + URL = "http://metservice.intnet.mu" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - week_forcecast = w_forecast.find_all(attrs={"class": "forecast"}) + w_forecast = soup.find(attrs={"class": "daysforecast"}) - week = [] + week_forcecast = w_forecast.find_all(attrs={"class": "forecast"}) - for i, _day in enumerate(week_forcecast): - fulldate = _day.find(attrs={"class": "fday"}) - fdate = fulldate.text.split(",") - date_day = fdate[0] - date_date = fdate[1].strip() + week = [] - return_dict = {} + for i, _day in enumerate(week_forcecast): + fulldate = _day.find(attrs={"class": "fday"}) + fdate = fulldate.text.split(",") + date_day = fdate[0] + date_date = fdate[1].strip() - return_dict["day"] = date_day - return_dict["date"] = date_date + return_dict = {} - condition = _day.find(attrs={"class": "fcondition"}) - return_dict["condition"] = condition.text + return_dict["day"] = date_day + return_dict["date"] = date_date - temp = _day.find_all(attrs={"class": "ftemp"}) - min_temp = temp[0].text - max_temp = temp[1].text - return_dict["min"] = min_temp - return_dict["max"] = max_temp + condition = _day.find(attrs={"class": "fcondition"}) + return_dict["condition"] = condition.text - fgrey = _day.find_all(attrs={"class": "fgrey"}) - wind = fgrey[0].text - sea_condition = fgrey[1].text - return_dict["wind"] = wind - return_dict["sea condition"] = sea_condition + temp = _day.find_all(attrs={"class": "ftemp"}) + min_temp = temp[0].text + max_temp = temp[1].text + return_dict["min"] = min_temp + return_dict["max"] = max_temp - prob = _day.find_all(attrs={"class": "fgrey1"}) - probability = prob[0].text - return_dict["probability"] = probability + fgrey = _day.find_all(attrs={"class": "fgrey"}) + wind = fgrey[0].text + sea_condition = fgrey[1].text + return_dict["wind"] = wind + return_dict["sea condition"] = sea_condition - week.append(return_dict) + prob = _day.find_all(attrs={"class": "fgrey1"}) + probability = prob[0].text + return_dict["probability"] = probability - return_data = {} + week.append(return_dict) + + return_data = week + + try: + cls.add_to_cache('weekforecast', return_data) + except: + pass if print_: console = Console() @@ -138,65 +216,71 @@ def get_weekforecast(cls, day=None, print=False): table.add_column("Sea condition", justify="left") table.add_column("Wind", justify="left", no_wrap=True) table.add_column("Probability", justify="left") - - - if day is None: - return_data = week - - if print_: + if day is None: table.title = 'Week forecast' for d in return_data: table.add_row(d['day'], d['date'], d['min'], d['max'], d['condition'], d['sea condition'], d['wind'], d['probability']) console.print(table) - return - else: - return_data = week[day] - if print_: + else: + return_data = return_data[day] table.title = 'Day forecast' d = return_data table.add_row(d['day'], d['date'], d['min'], d['max'], d['condition'], d['sea condition'], d['wind'], d['probability']) console.print(table) return - return return_data + if day is None: + return return_data + else: + return return_data[day] @classmethod def get_cityforecast(cls, print=False, day=None): print_ = print - cls.check_internet() - # for city of PL - URL = "http://metservice.intnet.mu" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + return_data = None + try: + cache = cls.get_from_cache('cityforecast') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + return_data = cache + else: + cls.check_internet() + # for city of PL + + URL = "http://metservice.intnet.mu" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - w_forecast = soup.find(attrs={"class": "city_forecast"}) - week_forcecast = w_forecast.find_all(attrs={"class": "block"}) + w_forecast = soup.find(attrs={"class": "city_forecast"}) + week_forcecast = w_forecast.find_all(attrs={"class": "block"}) - week = [] + week = [] - for i, _day in enumerate(week_forcecast): - date_day = _day.find(attrs={"class": "cday"}) - return_dict = {} - return_dict["day"] = date_day.text[:-1] + for i, _day in enumerate(week_forcecast): + date_day = _day.find(attrs={"class": "cday"}) + return_dict = {} + return_dict["day"] = date_day.text[:-1] - date_date = _day.find(attrs={"class": "cdate"}) - return_dict["date"] = date_date.text + date_date = _day.find(attrs={"class": "cdate"}) + return_dict["date"] = date_date.text - condition = _day.find(attrs={"class": "fcondition"}) - return_dict["condition"] = condition.text + condition = _day.find(attrs={"class": "fcondition"}) + return_dict["condition"] = condition.text - min_temp = _day.find(attrs={"class": "ctemp"}) - return_dict["min"] = min_temp.text + min_temp = _day.find(attrs={"class": "ctemp"}) + return_dict["min"] = min_temp.text - max_temp = _day.find(attrs={"class": "ctemp1"}) - return_dict["max"] = max_temp.text + max_temp = _day.find(attrs={"class": "ctemp1"}) + return_dict["max"] = max_temp.text - wind = _day.find(attrs={"class": "cwind1"}) - return_dict["wind"] = wind.text + wind = _day.find(attrs={"class": "cwind1"}) + return_dict["wind"] = wind.text - week.append(return_dict) + week.append(return_dict) if print_: @@ -230,7 +314,11 @@ def get_cityforecast(cls, print=False, day=None): table.add_row(d['day'], d['date'], d['min'], d['max'], d['condition'], d['wind']) console.print(table) return - + + try: + cls.add_to_cache('cityforecast', return_data) + except: + pass return return_data @@ -412,39 +500,51 @@ def test(cls): def get_sunrisemu(cls, print=False): print_ = print - cls.check_internet() + data = None + try: + cache = cls.get_from_cache('sunrisemu') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + data = cache + else: + cls.check_internet() - URL = "http://metservice.intnet.mu/sun-moon-and-tides-sunrise-sunset-mauritius.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + URL = "http://metservice.intnet.mu/sun-moon-and-tides-sunrise-sunset-mauritius.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - table = soup.find("table") - table_body = table.find("tbody") + table = soup.find("table") + table_body = table.find("tbody") - rows = table_body.find_all("tr") - data = dict() - for i, row in enumerate(rows): - cols = row.find_all("td") - - cols = [ele.text.strip() for ele in cols] + rows = table_body.find_all("tr") + data = dict() + for i, row in enumerate(rows): + cols = row.find_all("td") - if i == 0: - month1 = re.sub('\d+', '', cols[1].lower()).strip() - month2 = re.sub('\d+', '', cols[2].lower()).strip() - data = {month1: {}, month2: {}} + cols = [ele.text.strip() for ele in cols] - elif i > 1: - date = int(cols[0]) - m1_rise = cols[1] - m1_set = cols[2] - m2_rise = cols[3] - m2_set = cols[4] + if i == 0: + month1 = re.sub('\d+', '', cols[1].lower()).strip() + month2 = re.sub('\d+', '', cols[2].lower()).strip() + data = {month1: {}, month2: {}} - if m1_rise and m1_set: - data[month1][date] = {"rise": m1_rise, "set": m1_set} - if m2_rise and m2_set: - data[month2][date] = {"rise": m2_rise, "set": m2_set} + elif i > 1: + date = int(cols[0]) + m1_rise = cols[1] + m1_set = cols[2] + m2_rise = cols[3] + m2_set = cols[4] + if m1_rise and m1_set: + data[month1][date] = {"rise": m1_rise, "set": m1_set} + if m2_rise and m2_set: + data[month2][date] = {"rise": m2_rise, "set": m2_set} + try: + cls.add_to_cache('sunrisemu', data) + except: + pass def get_sun_info(data, month, date, point): try: return '{}'.format(data[month][date][point]) @@ -469,37 +569,50 @@ def get_sun_info(data, month, date, point): @classmethod def get_sunriserodr(cls, print=False): print_ = print - cls.check_internet() - URL = "http://metservice.intnet.mu/sun-moon-and-tides-sunrise-sunset-rodrigues.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + data = None + try: + cache = cls.get_from_cache('sunriserodr') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + data = cache + else: + cls.check_internet() + URL = "http://metservice.intnet.mu/sun-moon-and-tides-sunrise-sunset-rodrigues.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - table = soup.find("table") - table_body = table.find("tbody") + table = soup.find("table") + table_body = table.find("tbody") - rows = table_body.find_all("tr") - data = dict() - for i, row in enumerate(rows): - cols = row.find_all("td") + rows = table_body.find_all("tr") + data = dict() + for i, row in enumerate(rows): + cols = row.find_all("td") - cols = [ele.text.strip() for ele in cols] + cols = [ele.text.strip() for ele in cols] + + if i == 0: + month1 = cols[1].lower() + month2 = cols[2].lower() + data = {month1: {}, month2: {}} - if i == 0: - month1 = cols[1].lower() - month2 = cols[2].lower() - data = {month1: {}, month2: {}} - - elif i > 1: - date = int(cols[0]) - m1_rise = cols[1] - m1_set = cols[2] - m2_rise = cols[3] - m2_set = cols[4] - - if m1_rise and m1_set: - data[month1][date] = {"rise": m1_rise, "set": m1_set} - if m2_rise and m2_set: - data[month2][date] = {"rise": m2_rise, "set": m2_set} + elif i > 1: + date = int(cols[0]) + m1_rise = cols[1] + m1_set = cols[2] + m2_rise = cols[3] + m2_set = cols[4] + + if m1_rise and m1_set: + data[month1][date] = {"rise": m1_rise, "set": m1_set} + if m2_rise and m2_set: + data[month2][date] = {"rise": m2_rise, "set": m2_set} + try: + cls.add_to_cache('sunriserodr', data) + except: + pass def get_sun_info(data, month, date, point): try: return '{}'.format(data[month][date][point]) @@ -525,194 +638,211 @@ def get_sun_info(data, month, date, point): @classmethod def get_eclipses_raw(cls): - cls.check_internet() - URL = "http://metservice.intnet.mu/sun-moon-and-tides-info-eclipses.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + eclipses_data = None - tables = soup.find_all("table") - - len_tables = len(tables) - data = {'eclipses':[]} - year = None - eclipse_info = {} - all_tables = [] - equinoxes = [] - # cls.print(str(len(tables))) - for table_index, table in enumerate(tables): - current_list = table.text.strip().split('\n') - current_list = [e for e in current_list if bool(e.strip())] - if table_index == 0: - pass + try: + cache = cls.get_from_cache('eclipses_raw') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + eclipses_data = cache + else: + cls.check_internet() + URL = "http://metservice.intnet.mu/sun-moon-and-tides-info-eclipses.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - elif table_index == len(tables)-1: - equinoxes = current_list - else: - - all_tables += current_list - ''' - [ - { - 'end': {'date': 1, 'hour': 2, 'minute': 37, 'month': 'may'}, - 'info': 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', - 'start': {'date': 30, 'hour': 22, 'minute': 45, 'month': 'april'}, - 'status': 'partial', - 'type': 'sun' - }, - ... - { - 'end': {'date': 8, 'hour': 17, 'minute': 56, 'month': 'november'}, - 'info': 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', - 'start': {'date': 8, 'hour': 12, 'minute': 2, 'month': 'november'}, - 'status': 'total', - 'type': 'moon' - } - ] - ''' - eclipse_info = [] - - if cls.DEBUG: - cls.print(all_tables) - ''' - [ - 'Annular-Total eclipse of the Sun - April 20', - 'The eclipse begins on 20 April at 05h34 and ends on 20 April at 10h59.', - 'The penumbral part of the eclipse will be visible in Mauritius, Rodrigues, St. Brandon but will not be visible in - Agalega.', - 'Penumbral eclipse of the Moon - May 05', - 'The eclipse begins on 05 May at 19h12 and ends at 23h33.', - 'The eclipse will be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', - 'Annular eclipse of the Sun - October 14-15', - 'The eclipse begins on 14 October at 19h04 and ends on 15 October at 00h55.', - 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', - 'Partial eclipse of the Moon - October 28-29', - 'The eclipse begins on 28 October at 23h34 and ends on 29 October at 02h28.', - 'The eclipse will be visible in Mauritius, Rodrigues, St. Brandon and Agalega.' - ] - ''' - for table_index, row in enumerate(all_tables): - info = {} - if ( - ('eclipse of the' in row.casefold()) and - ('-' in row.casefold()) - ): - info['title'] = row.split(' - ')[0].strip() - info['info'] = all_tables[table_index+2] - if 'sun' in info['title'].casefold(): - info['type'] = 'sun' - elif 'moon' in info['title'].casefold(): - info['type'] = 'moon' - info['status'] = info['title'].casefold().split()[0].strip() - - # info['date'] = row.split(' - ')[1].strip() - - next_row = all_tables[table_index+1] - - next_row_words = next_row.split() - for i, word in enumerate(next_row_words): - if word == 'begins': - if next_row_words[i+1] == 'on': - info['start'] = {} - info['start']['date'] = int(next_row_words[i+2]) - info['start']['month'] = next_row_words[i+3].casefold() - info['start']['hour'] = int(next_row_words[i+5].split('h')[0].strip('.')) - info['start']['minute'] = int(next_row_words[i+5].split('h')[1].strip('.')) - if word == 'ends': - if next_row_words[i+1] == 'on': - info['end'] = {} - info['end']['date'] = int(next_row_words[i+2]) - info['end']['month'] = next_row_words[i+3].casefold() - info['end']['hour'] = int(next_row_words[i+5].split('h')[0].strip('.')) - info['end']['minute'] = int(next_row_words[i+5].split('h')[1].strip('.')) - elif next_row_words[i+1] == 'at': - info['end'] = {} - info['end']['date'] = int(info['start']['date']) - info['end']['month'] = info['start']['month'].casefold() - info['end']['hour'] = int(next_row_words[i+2].split('h')[0].strip('.')) - info['end']['minute'] = int(next_row_words[i+2].split('h')[1].strip('.')) - - eclipse_info.append(info) - - if cls.DEBUG: - cls.print(equinoxes) - ''' - [ - 'EQUINOXES and SOLSTICES - 2023', - 'Equinoxes\xa0\xa0\xa0 :\xa0 \xa0March 21 at 01h24 and September 23 at 10h50.', - 'Solstices\xa0\xa0\xa0 :\xa0 \xa0June 21 at 18h57 and December 22 at 07h27.' - ] - ''' - equinox = [e.strip().casefold().strip('.') for e in equinoxes[1].split()] - solstice = [e.strip().casefold().strip('.') for e in equinoxes[2].split()] - year = equinoxes[0].split(' - ')[1].strip().casefold() - year = int(year) - ''' + tables = soup.find_all("table") + + len_tables = len(tables) + data = {'eclipses':[]} + year = None + eclipse_info = {} + all_tables = [] + equinoxes = [] + # cls.print(str(len(tables))) + for table_index, table in enumerate(tables): + current_list = table.text.strip().split('\n') + current_list = [e for e in current_list if bool(e.strip())] + if table_index == 0: + pass + + elif table_index == len(tables)-1: + equinoxes = current_list + else: + + all_tables += current_list + ''' + [ + { + 'end': {'date': 1, 'hour': 2, 'minute': 37, 'month': 'may'}, + 'info': 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', + 'start': {'date': 30, 'hour': 22, 'minute': 45, 'month': 'april'}, + 'status': 'partial', + 'type': 'sun' + }, + ... + { + 'end': {'date': 8, 'hour': 17, 'minute': 56, 'month': 'november'}, + 'info': 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', + 'start': {'date': 8, 'hour': 12, 'minute': 2, 'month': 'november'}, + 'status': 'total', + 'type': 'moon' + } + ] + ''' + eclipse_info = [] + + if cls.DEBUG: + cls.print(all_tables) + ''' + [ + 'Annular-Total eclipse of the Sun - April 20', + 'The eclipse begins on 20 April at 05h34 and ends on 20 April at 10h59.', + 'The penumbral part of the eclipse will be visible in Mauritius, Rodrigues, St. Brandon but will not be visible in + Agalega.', + 'Penumbral eclipse of the Moon - May 05', + 'The eclipse begins on 05 May at 19h12 and ends at 23h33.', + 'The eclipse will be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', + 'Annular eclipse of the Sun - October 14-15', + 'The eclipse begins on 14 October at 19h04 and ends on 15 October at 00h55.', + 'The eclipse will not be visible in Mauritius, Rodrigues, St. Brandon and Agalega.', + 'Partial eclipse of the Moon - October 28-29', + 'The eclipse begins on 28 October at 23h34 and ends on 29 October at 02h28.', + 'The eclipse will be visible in Mauritius, Rodrigues, St. Brandon and Agalega.' + ] + ''' + for table_index, row in enumerate(all_tables): + info = {} + if ( + ('eclipse of the' in row.casefold()) and + ('-' in row.casefold()) + ): + info['title'] = row.split(' - ')[0].strip() + info['info'] = all_tables[table_index+2] + if 'sun' in info['title'].casefold(): + info['type'] = 'sun' + elif 'moon' in info['title'].casefold(): + info['type'] = 'moon' + info['status'] = info['title'].casefold().split()[0].strip() + + # info['date'] = row.split(' - ')[1].strip() + + next_row = all_tables[table_index+1] + + next_row_words = next_row.split() + for i, word in enumerate(next_row_words): + if word == 'begins': + if next_row_words[i+1] == 'on': + info['start'] = {} + info['start']['date'] = int(next_row_words[i+2]) + info['start']['month'] = next_row_words[i+3].casefold() + info['start']['hour'] = int(next_row_words[i+5].split('h')[0].strip('.')) + info['start']['minute'] = int(next_row_words[i+5].split('h')[1].strip('.')) + if word == 'ends': + if next_row_words[i+1] == 'on': + info['end'] = {} + info['end']['date'] = int(next_row_words[i+2]) + info['end']['month'] = next_row_words[i+3].casefold() + info['end']['hour'] = int(next_row_words[i+5].split('h')[0].strip('.')) + info['end']['minute'] = int(next_row_words[i+5].split('h')[1].strip('.')) + elif next_row_words[i+1] == 'at': + info['end'] = {} + info['end']['date'] = int(info['start']['date']) + info['end']['month'] = info['start']['month'].casefold() + info['end']['hour'] = int(next_row_words[i+2].split('h')[0].strip('.')) + info['end']['minute'] = int(next_row_words[i+2].split('h')[1].strip('.')) + + eclipse_info.append(info) + + if cls.DEBUG: + cls.print(equinoxes) + ''' + [ + 'EQUINOXES and SOLSTICES - 2023', + 'Equinoxes\xa0\xa0\xa0 :\xa0 \xa0March 21 at 01h24 and September 23 at 10h50.', + 'Solstices\xa0\xa0\xa0 :\xa0 \xa0June 21 at 18h57 and December 22 at 07h27.' + ] + ''' + equinox = [e.strip().casefold().strip('.') for e in equinoxes[1].split()] + solstice = [e.strip().casefold().strip('.') for e in equinoxes[2].split()] + year = equinoxes[0].split(' - ')[1].strip().casefold() + year = int(year) + ''' + + >>> get_equinoxes() + [ + { + 'day': 20, 'hour': 19, 'minute': 33, 'month': 'march', 'year': 2022 + }, + { + 'day': 23, 'hour': 5, 'minute': 3, 'month': 'september', 'year': 2022 + } + ] - >>> get_equinoxes() - [ - { - 'day': 20, 'hour': 19, 'minute': 33, 'month': 'march', 'year': 2022 - }, - { - 'day': 23, 'hour': 5, 'minute': 3, 'month': 'september', 'year': 2022 - } - ] - - >>> get_solstices() - [ - { - 'day': 21, 'hour': 13, 'minute': 13, 'month': 'june', 'year': 2022 - }, - { - 'day': 22, 'hour': 1, 'minute': 48, 'month': 'december', 'year': 2022 - } - ]''' - equinox_info = None - for i, e in enumerate(equinox): - if e == 'and': - equinox_info = [ - { - 'day': int(equinox[i-3]), - 'month': equinox[i-4], - 'year': year, - 'hour': int(equinox[i-1].split('h')[0]), - 'minute': int(equinox[i-1].split('h')[1]), - }, - { - 'day': int(equinox[i+2]), - 'month': equinox[i+1], - 'year': year, - 'hour': int(equinox[i+4].split('h')[0]), - 'minute': int(equinox[i+4].split('h')[1]), - }, - ] - - - solstice_info = None - for i, e in enumerate(equinox): - if e == 'and': - solstice_info = [ - { - 'day': int(solstice[i-3]), - 'month': solstice[i-4], - 'year': year, - 'hour': int(solstice[i-1].split('h')[0]), - 'minute': int(solstice[i-1].split('h')[1]), - }, - { - 'day': int(solstice[i+2]), - 'month': solstice[i+1], - 'year': year, - 'hour': int(solstice[i+4].split('h')[0]), - 'minute': int(solstice[i+4].split('h')[1]), - }, - ] + >>> get_solstices() + [ + { + 'day': 21, 'hour': 13, 'minute': 13, 'month': 'june', 'year': 2022 + }, + { + 'day': 22, 'hour': 1, 'minute': 48, 'month': 'december', 'year': 2022 + } + ]''' + equinox_info = None + for i, e in enumerate(equinox): + if e == 'and': + equinox_info = [ + { + 'day': int(equinox[i-3]), + 'month': equinox[i-4], + 'year': year, + 'hour': int(equinox[i-1].split('h')[0]), + 'minute': int(equinox[i-1].split('h')[1]), + }, + { + 'day': int(equinox[i+2]), + 'month': equinox[i+1], + 'year': year, + 'hour': int(equinox[i+4].split('h')[0]), + 'minute': int(equinox[i+4].split('h')[1]), + }, + ] + + + solstice_info = None + for i, e in enumerate(equinox): + if e == 'and': + solstice_info = [ + { + 'day': int(solstice[i-3]), + 'month': solstice[i-4], + 'year': year, + 'hour': int(solstice[i-1].split('h')[0]), + 'minute': int(solstice[i-1].split('h')[1]), + }, + { + 'day': int(solstice[i+2]), + 'month': solstice[i+1], + 'year': year, + 'hour': int(solstice[i+4].split('h')[0]), + 'minute': int(solstice[i+4].split('h')[1]), + }, + ] + + eclipses_data = { + 'eclipses': eclipse_info, + 'equinoxes': equinox_info, + 'solstices': solstice_info + } - return { - 'eclipses': eclipse_info, - 'equinoxes': equinox_info, - 'solstices': solstice_info - } + try: + cls.add_to_cache('eclipses_raw', eclipses_data) + except: + pass + + return eclipses_data @classmethod def get_eclipses(cls): @@ -731,6 +861,7 @@ def get_solstices(cls): @classmethod def print_today(cls, country='mu'): + cls.check_internet() ''' All info for today @@ -757,9 +888,9 @@ def print_today(cls, country='mu'): except: skip_moonphase = True if country == 'mu': - sun = cls.get_sunrisemu()[month][day] + sun = cls.get_sunrisemu()[month][str(day)] else: - sun = cls.get_sunriserodr()[month][day] + sun = cls.get_sunriserodr()[month][str(day)] # cls.print(forecast) @@ -898,8 +1029,7 @@ def print_today(cls, country='mu'): ### tides_all = cls.get_tides() - - tides = tides_all['months'][month][day] + tides = tides_all['months'][month][str(day)] tidetable = Table() tidetable.add_column("Tide", justify="left", style="slate_blue3", no_wrap=True) @@ -1010,206 +1140,264 @@ def add_row(grid, elements): def get_tides(cls, print=False): print_ = print - URL = "http://metservice.intnet.mu/sun-moon-and-tides-tides-mauritius.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") - tables = soup.find_all("table") - #cls.print(len(tables)) - if len(tables) == 1: - text_arr = tables[0].text.split() - text_arr_cleaned = [] - months = [] - years = [] - month_names = [m.casefold() for m in calendar.month_name if m] - for t in text_arr: - if t.casefold() in month_names: - months.append(t.casefold()) - text_arr_cleaned.append(t.casefold()) - elif re.findall('\d\d\d\d', t): - text_arr_cleaned.append(t.casefold()) - years.append(t) - elif ((t.casefold() not in ['date', '1st', '2nd', '(local)', '(height)', - 'tide', 'low', 'high', 'time', 'height', - '(cm)']) - ): - text_arr_cleaned.append(t) - - # cls.print(text_arr_cleaned) - # [ - # 'july', - # '2023', - # '1', - # '12:02', - # '54', - # '23:11', - # '57', - # '05:29', - # '6', - # '17:28', - # '28', - # '2', - # cls.print(months) - # cls.print(years) - tide_info = { - 'months': { + tide_info = None + + try: + cache = cls.get_from_cache('tides') + except Exception as e: + # TODO Add debug if permm error cache + cache = False + raise e + if cache: + tide_info = cache + else: + cls.check_internet() + URL = "http://metservice.intnet.mu/sun-moon-and-tides-tides-mauritius.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") + tables = soup.find_all("table") + #cls.print(len(tables)) + if len(tables) == 1: + text_arr = tables[0].text.split() + text_arr_cleaned = [] + months = [] + years = [] + month_names = [m.casefold() for m in calendar.month_name if m] + for t in text_arr: + if t.casefold() in month_names: + months.append(t.casefold()) + text_arr_cleaned.append(t.casefold()) + elif re.findall('\d\d\d\d', t): + text_arr_cleaned.append(t.casefold()) + years.append(t) + elif ((t.casefold() not in ['date', '1st', '2nd', '(local)', '(height)', + 'tide', 'low', 'high', 'time', 'height', + '(cm)']) + ): + text_arr_cleaned.append(t) + + # cls.print(text_arr_cleaned) + # [ + # 'july', + # '2023', + # '1', + # '12:02', + # '54', + # '23:11', + # '57', + # '05:29', + # '6', + # '17:28', + # '28', + # '2', + # cls.print(months) + # cls.print(years) + tide_info = { + 'months': { - }, - 'month_format': { - 'date': [ - '1st High Tide (Time (Local))', - '1st High Tide (Height (cm))', - '2nd High Tide (Time (Local))', - '2nd High Tide (Height (cm))', - '1st Low Tide (Time (Local))', - '1st Low Tide (Height (cm))', - '2nd Low Tide (Time (Local))', - '2nd Low Tide (Height (cm))', - ] - }, - 'meta':{ - 'months':[] + }, + 'month_format': { + 'date': [ + '1st High Tide (Time (Local))', + '1st High Tide (Height (cm))', + '2nd High Tide (Time (Local))', + '2nd High Tide (Height (cm))', + '1st Low Tide (Time (Local))', + '1st Low Tide (Height (cm))', + '2nd Low Tide (Time (Local))', + '2nd Low Tide (Height (cm))', + ] + }, + 'meta':{ + 'months':[] + } } - } - for i, month in enumerate(months): - tide_info['meta']['months'].append([months[i], years[i]]) - - tc_i = 0 - current_month = None - month_pad = 0 - while tc_i < len(text_arr_cleaned): - text = text_arr_cleaned[tc_i] - if text in month_names: - # current_month = f'{text_arr_cleaned[tc_i]}_{text_arr_cleaned[tc_i+1]}' - current_month = f'{text_arr_cleaned[tc_i]}' - tide_info['months'][current_month] = {} - month_pad += 1 - tc_i += 2 - continue - else: - if (tc_i - (2 * month_pad)) % 9 == 0: - date = int(text_arr_cleaned[tc_i]) - tide_info['months'][current_month][date] = text_arr_cleaned[tc_i+1:tc_i+9] - tc_i += 8 - tc_i += 1 - # cls.print(tide_info) + for i, month in enumerate(months): + tide_info['meta']['months'].append([months[i], years[i]]) + + tc_i = 0 + current_month = None + month_pad = 0 + while tc_i < len(text_arr_cleaned): + text = text_arr_cleaned[tc_i] + if text in month_names: + # current_month = f'{text_arr_cleaned[tc_i]}_{text_arr_cleaned[tc_i+1]}' + current_month = f'{text_arr_cleaned[tc_i]}' + tide_info['months'][current_month] = {} + month_pad += 1 + tc_i += 2 + continue + else: + if (tc_i - (2 * month_pad)) % 9 == 0: + date = int(text_arr_cleaned[tc_i]) + tide_info['months'][current_month][date] = text_arr_cleaned[tc_i+1:tc_i+9] + tc_i += 8 + tc_i += 1 + + try: + cls.add_to_cache('tides', tide_info) + except Exception as e: + raise e + cls.print(tide_info) return tide_info @classmethod def get_rainfall(cls, print=False): - print_ = print + print_ = print - URL = "http://metservice.intnet.mu/forecast-bulletin-english-mauritius.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + try: + cache = cls.get_from_cache('rainfall') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + rainfall_data = cache + else: + cls.check_internet() + + URL = "http://metservice.intnet.mu/forecast-bulletin-english-mauritius.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - content = soup.find("div", attrs={'class': 'left_content'}) + content = soup.find("div", attrs={'class': 'left_content'}) - content = [c for c in content.text.strip().split('\n') if c.strip()] + content = [c for c in content.text.strip().split('\n') if c.strip()] - info = None - data = {} - for line in content: - if 'highest rainfall' in line.casefold(): - info = line.split('during the')[1].strip()[:-len(' today:')] - if ( - ('mm' in line) and - (':' in line) - ): - region = line.strip().split(':')[0].strip() - rain = line.strip().split(':')[1].strip() - data[region] = rain + info = None + data = {} + for line in content: + if 'highest rainfall' in line.casefold(): + info = line.split('during the')[1].strip()[:-len(' today:')] + if ( + ('mm' in line) and + (':' in line) + ): + region = line.strip().split(':')[0].strip() + rain = line.strip().split(':')[1].strip() + data[region] = rain - rainfall_data = { - 'info': info, - 'rain': data - } + rainfall_data = { + 'info': info, + 'rain': data + } + try: + cls.add_to_cache('rainfall', rainfall_data) + except: + pass return rainfall_data @classmethod def get_latest(cls, print=False): - print_ = print - - URL = "http://metservice.intnet.mu/latest-weather-data.php" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + print_ = print + infos = None + try: + cache = cls.get_from_cache('latest') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + infos = cache + else: + cls.check_internet() + URL = "http://metservice.intnet.mu/latest-weather-data.php" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - weather_info = soup.find_all('div', attrs={'class': 'weatherinfo'}) - weather_info = [w.text.strip() for w in weather_info] - tables = soup.find_all("table", attrs={'class': 'tableau'}) + weather_info = soup.find_all('div', attrs={'class': 'weatherinfo'}) + weather_info = [w.text.strip() for w in weather_info] + tables = soup.find_all("table", attrs={'class': 'tableau'}) - infos = { - "rainfall24h": {}, - "rainfall3hrs": {}, - "wind": {}, - "humidity": {}, - "minmaxtemp": {} - } - for i, table in enumerate(tables): - title = weather_info[i].replace('\r', '').replace('\n', '') - if 'humidity' in title.casefold(): - key = 'humidity' - elif 'wind' in title.casefold(): - key = 'wind' - elif 'maximum and minimum' in title.casefold(): - key = 'minmaxtemp' - elif '3hrs' in title.casefold(): - key = 'rainfall3hrs' - else: - key = 'rainfall24h' - infos[key] = { - 'info': title, - 'data': {} + infos = { + "rainfall24h": {}, + "rainfall3hrs": {}, + "wind": {}, + "humidity": {}, + "minmaxtemp": {} } - trs = table.find_all('tr') - for tr in trs: - tds = tr.find_all('td') - for itd, td in enumerate(tds): - if td.text.strip().replace(' ', '').isalpha(): - try: - # infos.append(td.text.strip()) - # infos.append(tds[itd+1].text.strip()) - if key == 'minmaxtemp': - #cls.print(tds) - infos[key]['data'][td.text.strip()] = {} - infos[key]['data'][td.text.strip()]['min'] = tds[itd+1].text.strip() - infos[key]['data'][td.text.strip()]['max'] = tds[itd+2].text.strip() - - else: - infos[key]['data'][td.text.strip()] = tds[itd+1].text.strip() - - except Exception as e: - #cls.print(e) - pass + + for i, table in enumerate(tables): + title = weather_info[i].replace('\r', '').replace('\n', '') + if 'humidity' in title.casefold(): + key = 'humidity' + elif 'wind' in title.casefold(): + key = 'wind' + elif 'maximum and minimum' in title.casefold(): + key = 'minmaxtemp' + elif '3hrs' in title.casefold(): + key = 'rainfall3hrs' + else: + key = 'rainfall24h' + infos[key] = { + 'info': title, + 'data': {} + } + trs = table.find_all('tr') + for tr in trs: + tds = tr.find_all('td') + for itd, td in enumerate(tds): + if td.text.strip().replace(' ', '').isalpha(): + try: + # infos.append(td.text.strip()) + # infos.append(tds[itd+1].text.strip()) + if key == 'minmaxtemp': + #cls.print(tds) + infos[key]['data'][td.text.strip()] = {} + infos[key]['data'][td.text.strip()]['min'] = tds[itd+1].text.strip() + infos[key]['data'][td.text.strip()]['max'] = tds[itd+2].text.strip() + + else: + infos[key]['data'][td.text.strip()] = tds[itd+1].text.strip() + + except Exception as e: + #cls.print(e) + pass + try: + cls.add_to_cache('latest', infos) + except: + pass return infos @classmethod def get_uvindex(cls, print=False): print_ = print - cls.check_internet() - + regions = ["vacoas", "port-louis", "plaisance", "triolet", "camp-diable", "centre-de-flacq", "flic-en-flac", "tamarin", "rodrigues"] - data = {} + + try: + cache = cls.get_from_cache('uvindex') + except: + # TODO Add debug if permm error cache + cache = False + if cache: + data = cache + else: + cls.check_internet() + data = {} - for region in regions: - URL = f"https://en.tutiempo.net/ultraviolet-index/{region}.html" - r = requests.get(URL, headers=cls.headers) - soup = BeautifulSoup(r.content, "html.parser") + for region in regions: + URL = f"https://en.tutiempo.net/ultraviolet-index/{region}.html" + r = requests.get(URL, headers=cls.headers) + soup = BeautifulSoup(r.content, "html.parser") - uvindex = soup.find("div", attrs={'class': 'UvIndex'}) + uvindex = soup.find("div", attrs={'class': 'UvIndex'}) - today = uvindex.find("h4") + today = uvindex.find("h4") - uv_status = today.text - data[region] = uv_status + uv_status = today.text + data[region] = uv_status + try: + cls.add_to_cache('uvindex', data) + except: + pass if print_: uv_string = '' diff --git a/mmm.py b/mmm.py index 84ec05a..e3309f5 100644 --- a/mmm.py +++ b/mmm.py @@ -5,5 +5,4 @@ # Meteo.print(Meteo.get_sunrisemu()) #Meteo.print(Meteo.get_tides()) -Meteo.CHECK_INTERNET = True Meteo.print(Meteo.get_uvindex()) \ No newline at end of file