Add support for timeout

docs/Configuration.md

@@ -48,6 +48,9 @@ They are grouped by services.
 ```yaml
 common:
   github:
+  docker:
+  pypi:
+  raw_html:
 ```
 
 ### GitHub
@@ -57,17 +60,55 @@ common:
   github:
     username: name
     password: password or pat
+    timeout: 10
     rate_limit_wait_max: 120
 ```
 
 These settings are applied by default on `github_release`, `github_tag` and `github_commit` watchers.
 
 * `username` : username to access the GitHub API
 * `password` : password or [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) to access GitHub API
+* `timeout` : timeout in seconds for each request
 * `rate_limit_wait_max` : maximum number of seconds allowed to wait if the rate limit is exceeded
 
 When authenticated, GitHub has a much high [rate limit](https://developer.github.com/v3/#rate-limiting).
 
+### Docker
+
+```yaml
+common:
+  docker:
+    timeout: 10
+```
+
+These settings are applied by default on `docker_registry` watchers.
+
+* `timeout` : timeout in seconds for each request
+
+### Pypi
+
+```yaml
+common:
+  pypi:
+    timeout: 10
+```
+
+These settings are applied by default on `pypi` watchers.
+
+* `timeout` : timeout in seconds for each request
+
+### Raw HTML
+
+```yaml
+common:
+  raw_html:
+    timeout: 10
+```
+
+These settings are applied by default on `raw_html` watchers.
+
+* `timeout` : timeout in seconds for each request
+
 ## Sources
 
 Multiple sources can be used at the same time.

example/config.yaml

@@ -10,9 +10,16 @@ core:
 
 common:
   github:
+    timeout: 10
     rate_limit_wait_max: 120
     #username: name
     #password: password or pat
+  docker:
+    timeout: 10
+  pypi:
+    timeout: 10
+  raw_html:
+    timeout: 10
 
 sources:
   - type: inline
@@ -36,20 +43,21 @@ sources:
           - 5\..*
         excludes:
           - .*[ab][1-9]$
-      - name: TestGogs
+      # Example only, use the github_commit type instead !
+      - name: Raw Release Watcher
         type: raw_html
-        page_url: https://try.gogs.io/gogs/gogs/commits/master
-        id: 5a52ee75e3
-        container_selector: "#commits-table"
-        item_selector: "tbody tr"
-        id_selector: "[class~=sha]"
+        page_url: https://github.com/Neraud/release-watcher/commits/main
+        id: 3d0f1c0
+        container_selector: ".js-navigation-container"
+        item_selector: "li"
+        id_selector: ".Button-withTooltip a .Button-label"
         id_attribute: ""
-        date_selector: "[class~=time-since]"
-        date_attribute: "title"
+        date_selector: "relative-time"
+        date_attribute: "datetime"
         reverse: False
-        basic_auth:
-          username: your_username
-          password: your_password
+        #basic_auth:
+        #  username: your_username
+        #  password: your_password
   - type: file
     path: github_watchers.yaml
 

release_watcher/config_loader.py

@@ -6,7 +6,8 @@
 from release_watcher.sources import source_manager
 from release_watcher.outputs import output_manager
 from release_watcher.config_models import \
-    ConfigException, GlobalConfig, LoggerConfig, CommonConfig, CoreConfig, GithubConfig
+    ConfigException, GlobalConfig, LoggerConfig, CommonConfig, CoreConfig, \
+    GithubConfig, DockerConfig, PypiConfig, RawHtmlConfig
 
 try:
     from yaml import CLoader as Loader  # pylint: disable=ungrouped-imports
@@ -71,6 +72,10 @@ def _parse_conf(conf: Dict) -> GlobalConfig:
     config.core = _parse_core_conf(conf)
     config.common = CommonConfig()
     config.common.github = _parse_github_conf(conf)
+    config.common.docker = _parse_docker_conf(conf)
+    config.common.pypi = _parse_pypi_conf(conf)
+    config.common.raw_html = _parse_raw_html_conf(conf)
+
     config.sources = _parse_sources_conf(conf)
     if not config.sources:
         raise ConfigException('No valid source configuration found')
@@ -118,26 +123,99 @@ def _parse_core_conf(conf: Dict) -> CoreConfig:
 def _parse_github_conf(conf: Dict) -> GithubConfig:
     logger.debug('Loading github configuration')
 
-    default_conf = {'rate_limit_wait_max': 120}
+    default_conf = {
+        'timeout': 10,
+        'rate_limit_wait_max': 120,
+    }
+
     common_conf = conf.get('common', {'github': default_conf})
-    githug_conf = common_conf.get('github', default_conf)
+    github_conf = common_conf.get('github', default_conf)
     github_config = GithubConfig()
 
-    github_config.username = githug_conf.get('username')
-    github_config.password = githug_conf.get('password')
+    github_config.username = github_conf.get('username')
+    github_config.password = github_conf.get('password')
+
+    timeout = github_conf.get('timeout', default_conf['timeout'])
+    if not isinstance(timeout, (int, float)):
+        logger.warning('timeout %s is not a number, falling back to %d',
+                       timeout, default_conf['timeout'])
+        timeout = default_conf['timeout']
+    github_config.timeout = timeout
 
-    rate_limit_wait_max = githug_conf.get('rate_limit_wait_max',
+    rate_limit_wait_max = github_conf.get('rate_limit_wait_max',
                                           default_conf['rate_limit_wait_max'])
     if not isinstance(rate_limit_wait_max, int):
         logger.warning('rate_limit_wait_max %s is not a number, falling back to %d',
                        rate_limit_wait_max, default_conf['rate_limit_wait_max'])
         rate_limit_wait_max = default_conf['rate_limit_wait_max']
-
     github_config.rate_limit_wait_max = rate_limit_wait_max
 
     return github_config
 
 
+def _parse_docker_conf(conf: Dict) -> DockerConfig:
+    logger.debug('Loading docker configuration')
+
+    default_conf = {
+        'timeout': 10,
+    }
+
+    common_conf = conf.get('common', {'docker': default_conf})
+    docker_conf = common_conf.get('docker', default_conf)
+    docker_config = DockerConfig()
+
+    timeout = docker_conf.get('timeout', default_conf['timeout'])
+    if not isinstance(timeout, (int, float)):
+        logger.warning('timeout %s is not a number, falling back to %d',
+                       timeout, default_conf['timeout'])
+        timeout = default_conf['timeout']
+    docker_config.timeout = timeout
+
+    return docker_config
+
+
+def _parse_pypi_conf(conf: Dict) -> PypiConfig:
+    logger.debug('Loading pypi configuration')
+
+    default_conf = {
+        'timeout': 10,
+    }
+
+    common_conf = conf.get('common', {'pypi': default_conf})
+    pypi_conf = common_conf.get('pypi', default_conf)
+    pypi_config = PypiConfig()
+
+    timeout = pypi_conf.get('timeout', default_conf['timeout'])
+    if not isinstance(timeout, (int, float)):
+        logger.warning('timeout %s is not a number, falling back to %d',
+                       timeout, default_conf['timeout'])
+        timeout = default_conf['timeout']
+    pypi_config.timeout = timeout
+
+    return pypi_config
+
+
+def _parse_raw_html_conf(conf: Dict) -> PypiConfig:
+    logger.debug('Loading raw_html configuration')
+
+    default_conf = {
+        'timeout': 10,
+    }
+
+    common_conf = conf.get('common', {'raw_html': default_conf})
+    raw_html_conf = common_conf.get('raw_html', default_conf)
+    raw_html_config = RawHtmlConfig()
+
+    timeout = raw_html_conf.get('timeout', default_conf['timeout'])
+    if not isinstance(timeout, (int, float)):
+        logger.warning('timeout %s is not a number, falling back to %d',
+                       timeout, default_conf['timeout'])
+        timeout = default_conf['timeout']
+    raw_html_config.timeout = timeout
+
+    return raw_html_config
+
+
 def _parse_sources_conf(conf: Dict) -> Sequence[source_manager.SourceConfig]:
     logger.debug('Loading sources configuration')
     sources_conf = []

release_watcher/config_models.py

@@ -42,13 +42,35 @@ class GithubConfig:
 
     username: str = None
     password: str = None
+    timeout: float = None
     rate_limit_wait_max: int = None
 
 
+class DockerConfig:
+    """Model representing the docker configuration"""
+
+    timeout: float = None
+
+
+class PypiConfig:
+    """Model representing the pypi configuration"""
+
+    timeout: float = None
+
+
+class RawHtmlConfig:
+    """Model representing the raw_html configuration"""
+
+    timeout: float = None
+
+
 class CommonConfig:
     """Model representing common watchers configuration"""
 
     github: GithubConfig = None
+    docker: DockerConfig = None
+    pypi: PypiConfig = None
+    raw_html: RawHtmlConfig = None
 
 
 class GlobalConfig:

release_watcher/watchers/base_github_watcher.py

@@ -17,6 +17,7 @@ class BaseGithubConfig(WatcherConfig):
     repo: str = None
     username: str = None
     password: str = None
+    timeout: float
     rate_limit_wait_max: int
 
     def __init__(self, watcher_type_name: str, name: str, repo: str):
@@ -42,7 +43,7 @@ def _do_call_api(self, github_url: str, headers: Dict) -> Sequence[Dict]:
         else:
             auth = None
 
-        response = requests.get(github_url, headers=headers, auth=auth)
+        response = requests.get(github_url, headers=headers, auth=auth, timeout=self.config.timeout)
 
         if response.status_code == 200:
             res = json.loads(response.content.decode('utf-8'))

release_watcher/watchers/docker_registry_watcher.py

@@ -23,15 +23,17 @@ class DockerRegistryWatcherConfig(WatcherConfig):
     tag: str = None
     includes: Sequence[str] = []
     excludes: Sequence[str] = []
+    timeout: float
 
     def __init__(self, name: str, repo: str, image: str, tag: str,
-                 includes: Sequence[str], excludes: Sequence[str]):
+                 includes: Sequence[str], excludes: Sequence[str], timeout: float):
         super().__init__(WATCHER_TYPE_NAME, name)
         self.repo = repo
         self.image = image
         self.tag = tag
         self.includes = includes
         self.excludes = excludes
+        self.timeout = timeout
 
     def __str__(self) -> str:
         return f'{self.repo}:{self.image}:{self.tag}'
@@ -117,7 +119,7 @@ def _call_registry_api_tags_list(self, page_url=None) -> requests.Response:
         if self.auth_token:
             headers['Authorization'] = f'Bearer {self.auth_token}'
 
-        response = requests.get(docker_repo_url, headers=headers)
+        response = requests.get(docker_repo_url, headers=headers, timeout=self.config.timeout)
         return response
 
     def _call_docker_registry_api_auth(self, authenticate_header: str) -> str:
@@ -131,7 +133,7 @@ def _call_docker_registry_api_auth(self, authenticate_header: str) -> str:
                                    (key, parsed_hearder['bearer'][key]))
         auth_url = f'{realm}?{"&".join(auth_params)}'
         headers = {'Content-Type': 'application/json'}
-        response = requests.get(auth_url, headers=headers)
+        response = requests.get(auth_url, headers=headers, timeout=self.config.timeout)
 
         if response.status_code == 200:
             content = json.loads(response.content.decode('utf-8'))
@@ -152,7 +154,7 @@ def _get_tag_date(self, tag: str) -> datetime:
         if self.auth_token:
             headers['Authorization'] = f'Bearer {self.auth_token}'
 
-        response = requests.get(docker_repo_url, headers=headers)
+        response = requests.get(docker_repo_url, headers=headers, timeout=self.config.timeout)
         if response.status_code == 200:
             content = json.loads(response.content.decode('utf-8'))
             tag_date = None
@@ -188,7 +190,7 @@ def _get_date_from_manifest(self, digest: str) -> datetime:
         if self.auth_token:
             headers['Authorization'] = f'Bearer {self.auth_token}'
 
-        response = requests.get(api_url, headers=headers)
+        response = requests.get(api_url, headers=headers, timeout=self.config.timeout)
         if response.status_code == 200:
             content = json.loads(response.content.decode('utf-8'))
             return self._get_tag_date_from_config(content['config']['digest'])
@@ -210,7 +212,7 @@ def _get_tag_date_from_config(self, digest: str) -> datetime:
         if self.auth_token:
             headers['Authorization'] = f'Bearer {self.auth_token}'
 
-        response = requests.get(api_url, headers=headers)
+        response = requests.get(api_url, headers=headers, timeout=self.config.timeout)
         if response.status_code == 200:
             content = json.loads(response.content.decode('utf-8'))
             if 'created' in content:
@@ -242,8 +244,9 @@ def parse_config(self, common_config: CommonConfig,
         includes = watcher_config.get('includes', [])
         excludes = watcher_config.get('excludes', [])
         name = watcher_config.get('name', f'{repo}:{image}')
+        timeout = watcher_config.get('timeout', common_config.docker.timeout)
 
-        return DockerRegistryWatcherConfig(name, repo, image, tag, includes, excludes)
+        return DockerRegistryWatcherConfig(name, repo, image, tag, includes, excludes, timeout)
 
     def create_watcher(self, watcher_config: DockerRegistryWatcherConfig) -> DockerRegistryWatcher:
         return DockerRegistryWatcher(watcher_config)

release_watcher/watchers/github_commit_watcher.py

@@ -78,6 +78,7 @@ def parse_config(self, common_config: CommonConfig, watcher_config: Dict):
         config = GithubCommitWatcherConfig(name, repo, branch, commit)
         config.username = watcher_config.get('username', common_config.github.username)
         config.password = watcher_config.get('password', common_config.github.password)
+        config.timeout = watcher_config.get('timeout', common_config.github.timeout)
         config.rate_limit_wait_max = watcher_config.get(
             'rate_limit_wait_max', common_config.github.rate_limit_wait_max)
 

release_watcher/watchers/github_release_watcher.py

@@ -95,6 +95,7 @@ def parse_config(self, common_config: CommonConfig, watcher_config: Dict) \
 
         config.username = watcher_config.get('username', common_config.github.username)
         config.password = watcher_config.get('password', common_config.github.password)
+        config.timeout = watcher_config.get('timeout', common_config.github.timeout)
         config.rate_limit_wait_max = watcher_config.get(
             'rate_limit_wait_max', common_config.github.rate_limit_wait_max)
 

release_watcher/watchers/github_tag_watcher.py

@@ -97,6 +97,7 @@ def parse_config(self, common_config: CommonConfig, watcher_config) -> GithubTag
         config = GithubTagWatcherConfig(name, repo, tag, includes, excludes)
         config.username = watcher_config.get('username', common_config.github.username)
         config.password = watcher_config.get('password', common_config.github.password)
+        config.timeout = watcher_config.get('timeout', common_config.github.timeout)
         config.rate_limit_wait_max = watcher_config.get(
             'rate_limit_wait_max', common_config.github.rate_limit_wait_max)