From fedccd2e147a44440a2ba7121d1a96d5ac79f3b6 Mon Sep 17 00:00:00 2001 From: thevickypedia Date: Fri, 1 Dec 2023 13:55:44 -0600 Subject: [PATCH] Extend module support up to 10,000 releases --- changelog.rst | 4 ++++ docs/index.html | 9 +++------ docs/searchindex.js | 2 +- gitverse/__init__.py | 2 +- gitverse/releases.py | 44 ++++++++++++++++++++++++++++---------------- release_notes.rst | 4 ++++ 6 files changed, 41 insertions(+), 24 deletions(-) diff --git a/changelog.rst b/changelog.rst index c27b291..8103f65 100644 --- a/changelog.rst +++ b/changelog.rst @@ -1,6 +1,10 @@ Commit History ============== +0.5.5 (12/01/2023) +------------------ +- Extend module support up to 10,000 releases + 0.5.4 (12/01/2023) ------------------ - Reduce redundancy, improve runtime diff --git a/docs/index.html b/docs/index.html index f8bcb55..7b0a694 100644 --- a/docs/index.html +++ b/docs/index.html @@ -76,14 +76,11 @@

Welcome to GitVerse’s documentation!
-gitverse.releases.get_api_releases() → Dict[str, List[str]]
+gitverse.releases.get_api_releases() → Generator[Dict[str, List[str]]]

Get release notes via git api.

-
Returns:
-

Returns release notes in the form of release version and description as key-value pairs gathered via GitHub API.

-
-
Return type:
-

Dict[str, List[str]]

+
Yields:
+

Generator[Dict[str, List[str]]] – Yields release notes in the form of release version and description as key-value pairs gathered via GitHub API.

diff --git a/docs/searchindex.js b/docs/searchindex.js index f0bc03a..614a6b0 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["GitVerse", "Welcome to GitVerse\u2019s documentation!"], "terms": {"get": [0, 1], "github": [0, 1], "": 0, "format": 0, "restructuredtext": 0, "markdown": [0, 1], "document": 0, "pip": 0, "gener": [0, 1], "from": [0, 1], "tri": 0, "us": 0, "api": [0, 1], "If": 0, "fail": 0, "messag": [0, 1], "associ": 0, "thi": [0, 1], "featur": 0, "take": [0, 1], "git_token": 0, "an": [0, 1], "environ": 0, "variabl": 0, "i": 0, "person": 0, "repo": [0, 1], "known": 0, "issu": 0, "relat": 0, "order": [0, 1], "There": 0, "where": [0, 1], "git": [0, 1], "command": [0, 1], "return": [0, 1], "incorrect": 0, "timestamp": [0, 1], "when": 0, "ar": 0, "creat": [0, 1], "differ": 0, "timezon": 0, "expect": 0, "pleas": 0, "run": [0, 1], "follow": 0, "verifi": 0, "each": [0, 1], "ref": 0, "sort": 0, "creatord": 0, "refnam": 0, "short": 0, "iso8601": 0, "output": [0, 1], "rais": 0, "log": [0, 1], "debug": [0, 1], "enabl": 0, "mode": 0, "revers": [0, 1], "b": 0, "gather": [0, 1], "specif": 0, "branch": [0, 1], "default": [0, 1], "pass": [0, 1], "onli": 0, "f": 0, "write": [0, 1], "custom": 0, "filenam": [0, 1], "t": 0, "titl": [0, 1], "index": [0, 1], "line": [0, 1], "file": [0, 1], "_": 0, "rst": 0, "changelog": 0, "precommit": 0, "ensur": 0, "doc": 0, "creation": 0, "everi": 0, "requir": 0, "sphinx": 0, "5": 0, "1": 0, "pre": 0, "2": 0, "20": 0, "0": 0, "recommonmark": 0, "7": 0, "all": [0, 1], "http": 0, "org": 0, "project": 0, "thevickypedia": 0, "io": 0, "vignesh": 0, "rao": 0, "under": [0, 1], "mit": 0, "instal": 1, "usag": 1, "lint": 1, "pypi": 1, "packag": 1, "licens": 1, "copyright": 1, "releas": 1, "generate_snippet": 1, "list": 1, "str": 1, "snippet": 1, "base": 1, "inform": 1, "readi": 1, "load": 1, "type": 1, "get_api_releas": 1, "dict": 1, "note": 1, "via": 1, "form": 1, "version": 1, "descript": 1, "kei": 1, "valu": 1, "pair": 1, "get_releas": 1, "option": 1, "union": 1, "int": 1, "map": 1, "dictionari": 1, "none": 1, "get_tag": 1, "tag": 1, "iter": 1, "over": 1, "subject": 1, "yield": 1, "date": 1, "handler": 1, "function": 1, "paramet": 1, "name": 1, "ha": 1, "store": 1, "which": 1, "run_git_cmd": 1, "cmd": 1, "argument": 1, "split": 1, "commit": 1, "gitlog": 1, "trigger": 1, "convers": 1, "process": 1, "written": 1, "get_branch": 1, "avail": 1, "get_commit": 1, "trunk": 1, "scan": 1, "number": 1, "get_gitlog": 1, "exclud": 1, "merg": 1, "histori": 1, "modul": 1, "set": 1, "up": 1, "bearer": 1, "authent": 1, "class": 1, "model": 1, "auth_bear": 1, "bearerauth": 1, "token": 1, "instanti": 1, "object": 1, "refer": 1, "new": 1, "initi": 1, "assign": 1, "member": 1, "auth": 1, "msg": 1, "overrid": 1, "bool": 1, "fals": 1, "print": 1, "incom": 1, "light": 1, "green": 1, "error": 1, "bright": 1, "red": 1, "info": 1, "warn": 1, "yellow": 1, "util": 1, "get_release_not": 1, "pathlik": 1, "test": 1, "sourc": 1, "particular": 1, "string": 1, "rst2dict": 1, "regular_dict": 1, "ordereddict": 1, "convert": 1, "python": 1, "filepath": 1, "boolean": 1, "flag": 1, "regular": 1, "arg": 1, "rst2html": 1, "src": 1, "dst": 1, "html": 1, "destin": 1, "A": 1, "wa": 1, "success": 1, "search": 1, "page": 1}, "objects": {"gitverse": [[1, 0, 0, "-", "commits"], [1, 0, 0, "-", "releases"]], "gitverse.commits": [[1, 1, 1, "", "generator"], [1, 1, 1, "", "get_branches"], [1, 1, 1, "", "get_commits"], [1, 1, 1, "", "get_gitlog"], [1, 1, 1, "", "run"]], "gitverse.models": [[1, 0, 0, "-", "auth_bearer"], [1, 0, 0, "-", "debugger"]], "gitverse.models.auth_bearer": [[1, 2, 1, "", "BearerAuth"]], "gitverse.models.debugger": [[1, 1, 1, "", "debug"], [1, 1, 1, "", "error"], [1, 1, 1, "", "info"], [1, 1, 1, "", "warning"]], "gitverse.releases": [[1, 1, 1, "", "generate_snippets"], [1, 1, 1, "", "get_api_releases"], [1, 1, 1, "", "get_releases"], [1, 1, 1, "", "get_tags"], [1, 1, 1, "", "run"], [1, 1, 1, "", "run_git_cmd"]], "gitverse.utils": [[1, 0, 0, "-", "rst"]], "gitverse.utils.rst": [[1, 1, 1, "", "get_release_notes"], [1, 1, 1, "", "rst2dict"], [1, 1, 1, "", "rst2html"]]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"]}, "titleterms": {"gitvers": [0, 1], "instal": 0, "usag": 0, "releas": 0, "note": 0, "tag": 0, "commit": 0, "histori": 0, "option": 0, "flag": 0, "sampl": 0, "lint": 0, "pypi": 0, "packag": 0, "runbook": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "read": 1, "me": 1, "releasenot": 1, "commithistori": 1, "authbear": 1, "debugg": 1, "rst": 1, "parser": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file +Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["GitVerse", "Welcome to GitVerse\u2019s documentation!"], "terms": {"get": [0, 1], "github": [0, 1], "": 0, "format": 0, "restructuredtext": 0, "markdown": [0, 1], "document": 0, "pip": 0, "gener": [0, 1], "from": [0, 1], "tri": 0, "us": 0, "api": [0, 1], "If": 0, "fail": 0, "messag": [0, 1], "associ": 0, "thi": [0, 1], "featur": 0, "take": [0, 1], "git_token": 0, "an": [0, 1], "environ": 0, "variabl": 0, "i": 0, "person": 0, "repo": [0, 1], "known": 0, "issu": 0, "relat": 0, "order": [0, 1], "There": 0, "where": [0, 1], "git": [0, 1], "command": [0, 1], "return": [0, 1], "incorrect": 0, "timestamp": [0, 1], "when": 0, "ar": 0, "creat": [0, 1], "differ": 0, "timezon": 0, "expect": 0, "pleas": 0, "run": [0, 1], "follow": 0, "verifi": 0, "each": [0, 1], "ref": 0, "sort": 0, "creatord": 0, "refnam": 0, "short": 0, "iso8601": 0, "output": [0, 1], "rais": 0, "log": [0, 1], "debug": [0, 1], "enabl": 0, "mode": 0, "revers": [0, 1], "b": 0, "gather": [0, 1], "specif": 0, "branch": [0, 1], "default": [0, 1], "pass": [0, 1], "onli": 0, "f": 0, "write": [0, 1], "custom": 0, "filenam": [0, 1], "t": 0, "titl": [0, 1], "index": [0, 1], "line": [0, 1], "file": [0, 1], "_": 0, "rst": 0, "changelog": 0, "precommit": 0, "ensur": 0, "doc": 0, "creation": 0, "everi": 0, "requir": 0, "sphinx": 0, "5": 0, "1": 0, "pre": 0, "2": 0, "20": 0, "0": 0, "recommonmark": 0, "7": 0, "all": [0, 1], "http": 0, "org": 0, "project": 0, "thevickypedia": 0, "io": 0, "vignesh": 0, "rao": 0, "under": [0, 1], "mit": 0, "instal": 1, "usag": 1, "lint": 1, "pypi": 1, "packag": 1, "licens": 1, "copyright": 1, "releas": 1, "generate_snippet": 1, "list": 1, "str": 1, "snippet": 1, "base": 1, "inform": 1, "readi": 1, "load": 1, "type": 1, "get_api_releas": 1, "dict": 1, "note": 1, "via": 1, "yield": 1, "form": 1, "version": 1, "descript": 1, "kei": 1, "valu": 1, "pair": 1, "get_releas": 1, "option": 1, "union": 1, "int": 1, "map": 1, "dictionari": 1, "none": 1, "get_tag": 1, "tag": 1, "iter": 1, "over": 1, "subject": 1, "date": 1, "handler": 1, "function": 1, "paramet": 1, "name": 1, "ha": 1, "store": 1, "which": 1, "run_git_cmd": 1, "cmd": 1, "argument": 1, "split": 1, "commit": 1, "gitlog": 1, "trigger": 1, "convers": 1, "process": 1, "written": 1, "get_branch": 1, "avail": 1, "get_commit": 1, "trunk": 1, "scan": 1, "number": 1, "get_gitlog": 1, "exclud": 1, "merg": 1, "histori": 1, "modul": 1, "set": 1, "up": 1, "bearer": 1, "authent": 1, "class": 1, "model": 1, "auth_bear": 1, "bearerauth": 1, "token": 1, "instanti": 1, "object": 1, "refer": 1, "new": 1, "initi": 1, "assign": 1, "member": 1, "auth": 1, "msg": 1, "overrid": 1, "bool": 1, "fals": 1, "print": 1, "incom": 1, "light": 1, "green": 1, "error": 1, "bright": 1, "red": 1, "info": 1, "warn": 1, "yellow": 1, "util": 1, "get_release_not": 1, "pathlik": 1, "test": 1, "sourc": 1, "particular": 1, "string": 1, "rst2dict": 1, "regular_dict": 1, "ordereddict": 1, "convert": 1, "python": 1, "filepath": 1, "boolean": 1, "flag": 1, "regular": 1, "arg": 1, "rst2html": 1, "src": 1, "dst": 1, "html": 1, "destin": 1, "A": 1, "wa": 1, "success": 1, "search": 1, "page": 1}, "objects": {"gitverse": [[1, 0, 0, "-", "commits"], [1, 0, 0, "-", "releases"]], "gitverse.commits": [[1, 1, 1, "", "generator"], [1, 1, 1, "", "get_branches"], [1, 1, 1, "", "get_commits"], [1, 1, 1, "", "get_gitlog"], [1, 1, 1, "", "run"]], "gitverse.models": [[1, 0, 0, "-", "auth_bearer"], [1, 0, 0, "-", "debugger"]], "gitverse.models.auth_bearer": [[1, 2, 1, "", "BearerAuth"]], "gitverse.models.debugger": [[1, 1, 1, "", "debug"], [1, 1, 1, "", "error"], [1, 1, 1, "", "info"], [1, 1, 1, "", "warning"]], "gitverse.releases": [[1, 1, 1, "", "generate_snippets"], [1, 1, 1, "", "get_api_releases"], [1, 1, 1, "", "get_releases"], [1, 1, 1, "", "get_tags"], [1, 1, 1, "", "run"], [1, 1, 1, "", "run_git_cmd"]], "gitverse.utils": [[1, 0, 0, "-", "rst"]], "gitverse.utils.rst": [[1, 1, 1, "", "get_release_notes"], [1, 1, 1, "", "rst2dict"], [1, 1, 1, "", "rst2html"]]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"]}, "titleterms": {"gitvers": [0, 1], "instal": 0, "usag": 0, "releas": 0, "note": 0, "tag": 0, "commit": 0, "histori": 0, "option": 0, "flag": 0, "sampl": 0, "lint": 0, "pypi": 0, "packag": 0, "runbook": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "read": 1, "me": 1, "releasenot": 1, "commithistori": 1, "authbear": 1, "debugg": 1, "rst": 1, "parser": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file diff --git a/gitverse/__init__.py b/gitverse/__init__.py index f958670..ffc5a1c 100644 --- a/gitverse/__init__.py +++ b/gitverse/__init__.py @@ -1 +1 @@ -version = "3.1" +version = "3.2" diff --git a/gitverse/releases.py b/gitverse/releases.py index 0412f0f..31a5809 100644 --- a/gitverse/releases.py +++ b/gitverse/releases.py @@ -15,12 +15,12 @@ from gitverse.models.callables import md_link_pattern, options -def get_api_releases() -> Dict[str, List[str]]: +def get_api_releases() -> Generator[Dict[str, List[str]]]: """Get release notes via git api. - Returns: - Dict[str, List[str]]: - Returns release notes in the form of release version and description as key-value pairs gathered via GitHub API. + Yields: + Generator[Dict[str, List[str]]]: + Yields release notes in the form of release version and description as key-value pairs gathered via GitHub API. """ gh_token = os.getenv('GIT_TOKEN') or os.getenv('git_token') session = requests.Session() @@ -31,15 +31,25 @@ def get_api_releases() -> Dict[str, List[str]]: session.auth = BearerAuth(token=gh_token) else: debugger.warning("Trying to collect release notes without github token") - response = session.get(url=f'https://api.github.com/repos/{owner}/{repo_name}/releases') - if response.ok: - debugger.info("Collected release notes via GitHub API") - try: - return {resp['name']: resp['body'].splitlines() for resp in response.json()} - except requests.JSONDecodeError as error: - debugger.error(error) - else: - debugger.error(f"{response.status_code} - {response.text}") + page_num = 0 + # 100 pages with 100 releases per page, should cover up to 10_000 releases + while page_num <= 100: + page_num += 1 + response = session.get(url=f'https://api.github.com/repos/{owner}/{repo_name}/releases', + params={'per_page': 100, 'page': page_num}) + if response.ok: + response_json = response.json() + if not response_json: + debugger.debug(f"Page {page_num} returned {response_json}, assuming end of releases.") + break + debugger.info(f"Collected release notes on page {page_num} via GitHub API") + try: + yield {resp['name']: resp['body'].splitlines() for resp in response_json} + except requests.JSONDecodeError as error: + debugger.error(error) + else: + debugger.error(f"{response.status_code} - {response.text}") + break def run_git_cmd(cmd: str) -> str: @@ -103,13 +113,15 @@ def get_releases() -> Union[List[Dict[str, Union[str, List[str], int, str]]], No return debugger.info(f"Git tags gathered: {len(tags)}") # Update release notes for each version, if available via GitHub API - if release_api := get_api_releases(): - debugger.info(f"Release notes gathered: {len(release_api)}") + n = 0 + for release_api in get_api_releases(): + n = len(release_api) for tag in tags: if api_description := release_api.get(tag['version']): tag['description'] = api_description else: - debugger.warning(f"{tag['version']} could not be found in releases") + debugger.warning(f"{tag['version']} is either missing or doesn't have release notes") + debugger.info(f"Release notes gathered: {n}") if options['reverse']: debugger.warning('Converting snippets to reverse order') version_updates = sorted(tags, key=lambda x: x['timestamp'], reverse=True) diff --git a/release_notes.rst b/release_notes.rst index 57319f5..dff3e5d 100644 --- a/release_notes.rst +++ b/release_notes.rst @@ -1,6 +1,10 @@ Release Notes ============= +v3.2 (12/01/2023) +----------------- +- Includes support up to 10,000 releases (previously 30) + v3.1 (12/01/2023) ----------------- - Reduces redundant code and improves runtime