From 1cc13b638a9d609627ab5595bca07ac9c3d04c3d Mon Sep 17 00:00:00 2001 From: Tobias Kadelka Date: Wed, 20 Mar 2024 12:27:44 +0100 Subject: [PATCH] clean up code, variable names, comments, whitespace, help-text; add type hints --- .../exporter_file.py | 13 +++--- .../fzj_weather.py | 40 ++++++++----------- .../fzj_weather_crawler.py | 16 ++++---- prometheus_fzj_weather_exporter/main.py | 32 ++++++--------- 4 files changed, 44 insertions(+), 57 deletions(-) diff --git a/prometheus_fzj_weather_exporter/exporter_file.py b/prometheus_fzj_weather_exporter/exporter_file.py index 046ceba..c03f52d 100644 --- a/prometheus_fzj_weather_exporter/exporter_file.py +++ b/prometheus_fzj_weather_exporter/exporter_file.py @@ -13,15 +13,18 @@ class FZJWeatherExporter: - insec: bool + insecure: bool + url: str - def __init__(self, insec_bool) -> None: - self.insec = insec_bool + def __init__(self, + url: str, + insecure: bool) -> None: + self.url = url + self.insecure = insecure @REQUEST_TIME.time() def collect(self): - - weather = fzj_weather_crawler.fzj_weather_crawler(self.insec) + weather = fzj_weather_crawler.fzj_weather_crawler(self.url, self.insecure) g = GaugeMetricFamily( name='fzj_weather_air_temperature_celsius', diff --git a/prometheus_fzj_weather_exporter/fzj_weather.py b/prometheus_fzj_weather_exporter/fzj_weather.py index 4e95350..c820c7d 100755 --- a/prometheus_fzj_weather_exporter/fzj_weather.py +++ b/prometheus_fzj_weather_exporter/fzj_weather.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # This file is licensed under the ISC license. # Oskar Druska 2022 -# For further information look up LICENSE.txt +# For further information look up LICENSE # This script parses weather data from an FZJ inside website @@ -10,43 +10,35 @@ from bs4 import BeautifulSoup -# Python module to execute - -def get_weather_data(insec_bool): - url = "https://www.fz-juelich.de/de/gs/ueber-uns/meteo/aktuelle-wetterdaten/wetterdaten" - - # if insec_bool, then Request shall ignore the SSL certificate - r = requests.get(url, verify=(not insec_bool)) - +def get_weather_data(url: str, + insecure: bool) -> dict: + # if `insecure`, then Request shall ignore the SSL certificate + r = requests.get(url, verify=(not insecure)) if r.status_code != 200: - raise ConnectionError("Something's wrong with the Website:\n" + url + "\n" + str(r.status_code)) + raise ConnectionError(f"Something's wrong with the Website:\n{url}\n{r.status_code}") soup = BeautifulSoup(r.text, 'html.parser') - - weather_dict = make_weather_dict(url, soup) # {header: data} + weather_dict = make_weather_dict(url, soup) return weather_dict -def make_weather_dict(url, soup): - # Parses the table containing the needed information to get all table rows. - weather_tablerows = soup.table.find_all("tr") - - # Creates a dictionary with headers as keys and data as values - # (i.e. Luftdruck: 1016.6 hPa). - # `.replace(u'\xa0', u' ')` replaces parsing errors with whitespaces - # `re.sub('[^0-9 , .]', '', weather_td[1].get_text(strip=True)` strips - # all non-numeric characters from the string - +def make_weather_dict(url, soup) -> dict: + """Parses the table containing weather information from the webpage into a + dictionary with headers as keys and data as values (i.e. Luftdruck: 1016.6 hPa). + """ + weather_table = soup.table.find_all("tr") weather_data = { "source": url, "title": soup.title.get_text(strip=True), "date": soup.u.get_text(strip=True) } - for row in weather_tablerows: - weather_td = row.find_all("td") # td, table data + for row in weather_table: + weather_td = row.find_all("td") # td: table data + # `replace(u'\xa0', u' ')` replaces parsing errors with whitespaces + # `re.sub('[^0-9 , .]', ''` strips all non-numeric characters from the string weather_data[weather_td[0].get_text(strip=True).replace(u'\xa0', u' ')] \ = re.sub('[^0-9 , .]', '', weather_td[1].get_text(strip=True)) diff --git a/prometheus_fzj_weather_exporter/fzj_weather_crawler.py b/prometheus_fzj_weather_exporter/fzj_weather_crawler.py index 574f502..aa40d94 100755 --- a/prometheus_fzj_weather_exporter/fzj_weather_crawler.py +++ b/prometheus_fzj_weather_exporter/fzj_weather_crawler.py @@ -16,13 +16,13 @@ class Weather: wind_direction: int # degree -def fzj_weather_crawler(insec_bool): - """ scrapes data from the FZJ weather site via the fzj_weather.py script - and returns a dataclass object containing the information """ - - crawled_weather_data = fzj_weather.get_weather_data(insec_bool) - - weather_return = Weather( +def fzj_weather_crawler(url: str, + insecure: bool) -> Weather: + """Scrape data from the FZJ weather site via fzj_weather.py + and return a dataclass object containing the information. + """ + crawled_weather_data = fzj_weather.get_weather_data(url, insecure) + weather = Weather( temperature=float(crawled_weather_data['Lufttemperatur']), air_pressure=float(crawled_weather_data['Luftdruck (92 m ΓΌ.N.H.N.)']), humidity=int(crawled_weather_data['relative Feuchte']), @@ -30,4 +30,4 @@ def fzj_weather_crawler(insec_bool): wind_direction=int(crawled_weather_data['Windrichtung']) ) - return weather_return + return weather diff --git a/prometheus_fzj_weather_exporter/main.py b/prometheus_fzj_weather_exporter/main.py index a6001e0..9bc0f52 100644 --- a/prometheus_fzj_weather_exporter/main.py +++ b/prometheus_fzj_weather_exporter/main.py @@ -1,21 +1,11 @@ #!/usr/bin/env python3 # This file is licensed under the ISC license. # Oskar Druska 2022 -# For further information look up LICENSE.txt - -# exporter entry point - -# test usage: -# > python prometheus_fzj_weather_exporter/main.py --web.listen-address 127.0.0.1:9184 -# > curl 127.0.0.1:9184 (in different window) -# expected output (similar to): -# > # HELP fzj_weather_air_temperature temperature in celsius -# > # TYPE fzj_weather_air_temperature gauge -# > fzj_weather_air_temperature 14.0 -# (equivalent output for other data i.e. humidity) +# For further information look up LICENSE import sys import argparse +from argparse import RawTextHelpFormatter import time from prometheus_client import start_http_server, REGISTRY from prometheus_fzj_weather_exporter import exporter_file @@ -23,36 +13,38 @@ def main(): args = get_parsed_args() - + url = "https://www.fz-juelich.de/de/gs/ueber-uns/meteo/aktuelle-wetterdaten/wetterdaten" try: - REGISTRY.register(exporter_file.FZJWeatherExporter(args.insecure)) + REGISTRY.register(exporter_file.FZJWeatherExporter(url, args.insecure)) except ConnectionError as c: sys.exit(c.strerror) - if args.listenaddress is None: - start_http_server(port=9184, addr='127.0.0.1') - else: + # start the http server + if args.listenaddress: ip, port = args.listenaddress.split(":") if ip: start_http_server(port=int(port), addr=ip) else: # listen on all interfaces start_http_server(port=int(port)) + else: + start_http_server(port=9184, addr='127.0.0.1') - # keep the thing going indefinitely + # keep the exporter running indefinitely while True: time.sleep(1) def get_parsed_args(): parser = argparse.ArgumentParser( - description='Set up the Prometheus exporter (connection ports)') + description='Set up the Prometheus exporter (connection ports)', + formatter_class=RawTextHelpFormatter) group = parser.add_argument_group() group.add_argument( '-w', '--web.listen-address', type=str, dest='listenaddress', help='Address and port to expose metrics and web interface. Default: ":9184"\n' - 'To listen on all interfaces, omit the IP. ":"\n' + 'To listen on all interfaces, omit the IP: ":"\n' 'To listen on a specific IP:
:') group.add_argument( '-i', '--insecure',