Skip to content

Commit

Permalink
Use pypi-json to interact with PyPI's JSON API.
Browse files Browse the repository at this point in the history
  • Loading branch information
@domdfcoding
domdfcoding committed May 15, 2022
1 parent 62120d0 commit 9d2e4ee
Show file tree
Hide file tree
Showing 21 changed files with 1,074 additions and 55 deletions.
1 change: 1 addition & 0 deletions formate.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ known_third_party = [
"ruamel_yaml",
"sdjson",
"shippinglabel",
"shippinglabel_conda",
"southwark",
"tabulate",
"toml",
Expand Down
43 changes: 18 additions & 25 deletions repo_helper/cli/commands/suggest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
Suggest trove classifiers and keywords.
"""
#
# Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
# Copyright © 2020-2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
Expand Down Expand Up @@ -227,13 +227,12 @@ def stubs(
from itertools import chain

# 3rd party
import pypi_json
import tabulate
from apeye import URL
from apeye.requests_url import TrailingRequestsURL
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import StringList
from packaging.requirements import InvalidRequirement
from shippinglabel import normalize
from shippinglabel.pypi import PYPI_API
from shippinglabel.requirements import combine_requirements, read_requirements

# this package
Expand Down Expand Up @@ -264,28 +263,22 @@ def stubs(

suggestions = {}

for requirement in all_requirements:
if normalize(requirement.name) in {"typing-extensions"}:
continue

types_url = TrailingRequestsURL(PYPI_API / f"types-{requirement.name.lower()}" / "json/")
stubs_url = TrailingRequestsURL(PYPI_API / f"{requirement.name.lower()}-stubs" / "json/")

response = stubs_url.head()
if response.status_code == 404:
# No stubs found for -stubs
response = types_url.head()
if response.status_code == 404:
# No stubs found for types-
with pypi_json.PyPIJSON() as client:
for requirement in all_requirements:
if normalize(requirement.name) in {"typing-extensions"}:
continue
else:
response_url = URL(response.url)
suggestions[str(requirement)] = response_url.parent.name
# print(requirement, response.url)
else:
response_url = URL(response.url)
suggestions[str(requirement)] = response_url.parent.name
# print(requirement, response.url)

try:
metadata = client.get_metadata(f"types-{requirement.name.lower()}")
except InvalidRequirement:
# No stubs found for -stubs
try:
metadata = client.get_metadata(f"{requirement.name.lower()}-stubs")
except InvalidRequirement:
# No stubs found for types-
continue

suggestions[str(requirement)] = metadata.name

if not suggestions:
if sys.stdout.isatty() or force_tty:
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jinja2>=2.11.3
mkrecipe>=0.3.0
natsort>=7.1.1
packaging>=20.9
pypi-json>=0.2.1
ruamel.yaml>=0.17.4
shippinglabel>=1.0.0
shippinglabel-conda>=0.1.0
Expand Down
71 changes: 71 additions & 0 deletions tests/test_cli/test_show_/test_requirements_3_10_.tree
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,25 @@ repo_helper==2020.12.18
├── natsort>=7.1.1
├── packaging>=20.9
│ └── pyparsing!=3.0.5,>=2.0.2
├── pypi-json>=0.2.1
│ ├── apeye>=1.1.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── packaging>=21.0
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ └── requests>=2.26.0
│ ├── certifi>=2017.4.17
│ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ ├── idna<4,>=2.5; python_version >= "3"
│ └── urllib3<1.27,>=1.21.1
├── ruamel.yaml>=0.17.4
│ └── ruamel.yaml.clib>=0.2.6; platform_python_implementation == "CPython" and python_version < "3.11"
├── shippinglabel>=1.0.0
Expand Down Expand Up @@ -353,6 +372,58 @@ repo_helper==2020.12.18
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── shippinglabel-conda>=0.1.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── domdf-python-tools>=3.3.0
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── shippinglabel>=1.3.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── dist-meta>=0.1.2
│ │ ├── domdf-python-tools>=3.1.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── handy-archives>=0.1.0
│ │ │ └── domdf-python-tools>=3.1.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── dom-toml>=0.2.2
│ │ ├── domdf-python-tools>=2.8.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── toml>=0.10.2
│ ├── domdf-python-tools>=3.1.0
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── southwark>=0.7.1
│ ├── click>=7.1.2
│ ├── consolekit>=0.1.2
Expand Down
71 changes: 71 additions & 0 deletions tests/test_cli/test_show_/test_requirements_3_11__.tree
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,25 @@ repo_helper==2020.12.18
├── natsort>=7.1.1
├── packaging>=20.9
│ └── pyparsing!=3.0.5,>=2.0.2
├── pypi-json>=0.2.1
│ ├── apeye>=1.1.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── packaging>=21.0
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ └── requests>=2.26.0
│ ├── certifi>=2017.4.17
│ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ ├── idna<4,>=2.5; python_version >= "3"
│ └── urllib3<1.27,>=1.21.1
├── ruamel.yaml>=0.17.4
├── shippinglabel>=1.0.0
│ ├── apeye>=1.0.0
Expand Down Expand Up @@ -351,6 +370,58 @@ repo_helper==2020.12.18
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── shippinglabel-conda>=0.1.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── domdf-python-tools>=3.3.0
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── shippinglabel>=1.3.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── dist-meta>=0.1.2
│ │ ├── domdf-python-tools>=3.1.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── handy-archives>=0.1.0
│ │ │ └── domdf-python-tools>=3.1.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── dom-toml>=0.2.2
│ │ ├── domdf-python-tools>=2.8.0
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── toml>=0.10.2
│ ├── domdf-python-tools>=3.1.0
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── southwark>=0.7.1
│ ├── click>=7.1.2
│ ├── consolekit>=0.1.2
Expand Down
111 changes: 111 additions & 0 deletions tests/test_cli/test_show_/test_requirements_3_6_.tree
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,30 @@ repo_helper==2020.12.18
├── natsort>=7.1.1
├── packaging>=20.9
│ └── pyparsing!=3.0.5,>=2.0.2
├── pypi-json>=0.2.1
│ ├── apeye>=1.1.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── packaging>=21.0
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ └── requests>=2.26.0
│ ├── certifi>=2017.4.17
│ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ ├── idna<4,>=2.5; python_version >= "3"
│ └── urllib3<1.27,>=1.21.1
├── ruamel.yaml>=0.17.4
│ └── ruamel.yaml.clib>=0.2.6; platform_python_implementation == "CPython" and python_version < "3.11"
├── shippinglabel>=1.0.0
Expand Down Expand Up @@ -591,6 +615,93 @@ repo_helper==2020.12.18
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── shippinglabel-conda>=0.1.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── domdf-python-tools>=3.3.0
│ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ └── zipp>=0.5
│ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── shippinglabel>=1.3.0
│ ├── apeye>=1.0.0
│ │ ├── domdf-python-tools>=2.6.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── idna>=2.5
│ │ ├── platformdirs>=2.3.0
│ │ └── requests>=2.24.0
│ │ ├── certifi>=2017.4.17
│ │ ├── charset-normalizer~=2.0.0; python_version >= "3"
│ │ ├── idna<4,>=2.5; python_version >= "3"
│ │ └── urllib3<1.27,>=1.21.1
│ ├── dist-meta>=0.1.2
│ │ ├── domdf-python-tools>=3.1.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ ├── handy-archives>=0.1.0
│ │ │ └── domdf-python-tools>=3.1.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── dom-toml>=0.2.2
│ │ ├── domdf-python-tools>=2.8.0
│ │ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ │ └── zipp>=0.5
│ │ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ │ ├── natsort>=7.0.1
│ │ │ └── typing-extensions>=3.7.4.1
│ │ └── toml>=0.10.2
│ ├── domdf-python-tools>=3.1.0
│ │ ├── importlib-metadata>=3.6.0; python_version < "3.9"
│ │ │ ├── typing-extensions>=3.6.4; python_version < "3.8"
│ │ │ └── zipp>=0.5
│ │ ├── importlib-resources>=3.0.0; python_version < "3.7"
│ │ │ └── zipp>=3.1.0; python_version < "3.10"
│ │ ├── natsort>=7.0.1
│ │ └── typing-extensions>=3.7.4.1
│ ├── packaging>=20.9
│ │ └── pyparsing!=3.0.5,>=2.0.2
│ ├── platformdirs>=2.3.0
│ └── typing-extensions>=3.7.4.3
├── southwark>=0.7.1
│ ├── click>=7.1.2
│ ├── consolekit>=0.1.2
Expand Down
Loading

0 comments on commit 9d2e4ee

Please sign in to comment.