Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rework audit-licenses check [ci] #123119

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft

Rework audit-licenses check [ci] #123119

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 125 additions & 24 deletions script/licenses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from dataclasses import dataclass
from importlib import metadata
import json
import logging
from pathlib import Path
import sys
from typing import TypedDict, cast
Expand All @@ -22,6 +23,7 @@
)

licensing = get_spdx_licensing()
logger = logging.getLogger(__name__)


class PackageMetadata(TypedDict):
Expand All @@ -38,7 +40,6 @@ class PackageMetadata(TypedDict):
class PackageDefinition:
"""Package definition."""

license: str
license_expression: str | None
license_metadata: str | None
license_classifier: list[str]
Expand All @@ -48,10 +49,7 @@ class PackageDefinition:
@classmethod
def from_dict(cls, data: PackageMetadata) -> PackageDefinition:
"""Create a package definition from PackageMetadata."""
if not (license_str := "; ".join(data["license_classifier"])):
license_str = data["license_metadata"] or "UNKNOWN"
return cls(
license=license_str,
license_expression=data["license_expression"],
license_metadata=data["license_metadata"],
license_classifier=data["license_classifier"],
Expand Down Expand Up @@ -92,7 +90,7 @@ def from_dict(cls, data: PackageMetadata) -> PackageDefinition:
"ZPL-2.1",
}

OSI_APPROVED_LICENSES = {
OSI_APPROVED_LICENSE_CLASSIFIER = {
"Academic Free License (AFL)",
"Apache Software License",
"Apple Public Source License",
Expand Down Expand Up @@ -160,19 +158,6 @@ def from_dict(cls, data: PackageMetadata) -> PackageDefinition:
"Zero-Clause BSD (0BSD)",
"Zope Public License",
"zlib/libpng License",
# End license classifier
"Apache License",
"MIT",
"MPL2",
"Apache 2",
"LGPL v3",
"BSD",
"GNU-3.0",
"GPLv3",
"Eclipse Public License v2.0",
"ISC",
"GNU General Public License v3",
"GPLv2",
}

EXCEPTIONS = {
Expand Down Expand Up @@ -211,6 +196,87 @@ def from_dict(cls, data: PackageMetadata) -> PackageDefinition:
"aiocache": AwesomeVersion(
"0.12.3"
), # https://github.com/aio-libs/aiocache/blob/master/LICENSE all rights reserved?
# -- Full license text in metadata
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should these be in TODO or in exceptions?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of them are OSI licenses, so I wouldn't consider them to be "exceptions" per se. More things we should take care of which would be "todo".

I can change it though if you think it makes more sense there.

"PyNINA": AwesomeVersion("0.3.3"), # MIT
"aioconsole": AwesomeVersion("0.8.1"), # GPL
"dicttoxml": AwesomeVersion("1.7.16"), # GPL
"homematicip": AwesomeVersion("1.1.2"), # GPL
"ibmiotf": AwesomeVersion("0.3.4"), # Eclipse Public License
"matrix-nio": AwesomeVersion("0.25.2"), # ISC
"zwave-js-server-python": AwesomeVersion(
"0.58.1"
), # Apache # https://github.com/home-assistant-libs/zwave-js-server-python/pull/1029
# -- Not SPDX license strings --
"PyNaCl": AwesomeVersion("1.5.0"), # Apache License 2.0
"PySocks": AwesomeVersion("1.7.1"), # BSD
"aioairq": AwesomeVersion("0.3.2"), # Apache License, Version 2.0
"aioaquacell": AwesomeVersion("0.2.0"), # Apache License 2.0
"aioeagle": AwesomeVersion("1.1.0"), # Apache License 2.0
"aiohttp_socks": AwesomeVersion("0.9.0"), # Apache 2
"aiolivisi": AwesomeVersion("0.0.19"), # Apache License 2.0
"aiopegelonline": AwesomeVersion("0.0.10"), # Apache License 2.0
"aioshelly": AwesomeVersion("12.0.1"), # Apache License 2.0
"amcrest": AwesomeVersion("1.9.8"), # GPLv2
"async-modbus": AwesomeVersion("0.2.1"), # GNU General Public License v3
"asyncssh": AwesomeVersion("2.18.0"), # Eclipse Public License v2.0
"baidu-aip": AwesomeVersion("1.6.6.0"), # Apache License
"bs4": AwesomeVersion("0.0.2"), # MIT License
"bt-proximity": AwesomeVersion("0.2.1"), # Apache 2.0
"connio": AwesomeVersion("0.2.0"), # GPLv3+
"datapoint": AwesomeVersion("0.9.9"), # GPLv3
"electrickiwi-api": AwesomeVersion("0.8.5"), # GNU-3.0
"freebox-api": AwesomeVersion("1.1.0"), # GNU GPL v3
"gpiod": AwesomeVersion("2.2.1"), # LGPLv2.1
"insteon-frontend-home-assistant": AwesomeVersion("0.5.0"), # MIT License
"knx_frontend": AwesomeVersion("2024.9.10.221729"), # MIT License
"lcn-frontend": AwesomeVersion("0.2.1"), # MIT License
"libpyfoscam": AwesomeVersion("1.2.2"), # LGPLv3+
"london-tube-status": AwesomeVersion("0.5"), # Apache License, Version 2.0
"mutesync": AwesomeVersion("0.0.1"), # Apache License 2.0
"oemthermostat": AwesomeVersion("1.1.1"), # BSD
"paho-mqtt": AwesomeVersion(
"1.6.1"
), # Eclipse Public License v2.0 / Eclipse Distribution License v1.0
"pilight": AwesomeVersion("0.1.1"), # MIT License
"ply": AwesomeVersion("3.11"), # BSD
"protobuf": AwesomeVersion("5.28.3"), # 3-Clause BSD License
"psutil-home-assistant": AwesomeVersion("0.0.1"), # Apache License 2.0
"pure-pcapy3": AwesomeVersion("1.0.1"), # Simplified BSD
"py-vapid": AwesomeVersion("1.9.1"), # MPL2
"pyAtome": AwesomeVersion("0.1.1"), # Apache Software License
"pybotvac": AwesomeVersion("0.0.25"), # Licensed under the MIT license
"pychannels": AwesomeVersion("1.2.3"), # The MIT License
"pycognito": AwesomeVersion("2024.5.1"), # Apache License 2.0
"pycryptodome": AwesomeVersion("3.21.0"), # BSD, Public Domain
"pycryptodomex": AwesomeVersion("3.21.0"), # BSD, Public Domain
"pydanfossair": AwesomeVersion("0.1.0"), # Apache 2.0
"pydrawise": AwesomeVersion("2024.9.0"), # Apache License 2.0
"pydroid-ipcam": AwesomeVersion("2.0.0"), # Apache License 2.0
"pyebox": AwesomeVersion("1.1.4"), # Apache 2.0
"pyevilgenius": AwesomeVersion("2.0.0"), # Apache License 2.0
"pyezviz": AwesomeVersion("0.2.1.2"), # Apache Software License 2.0
"pyfido": AwesomeVersion("2.1.2"), # Apache 2.0
"pyialarm": AwesomeVersion("2.2.0"), # Apache 2.0
"pylitejet": AwesomeVersion("0.6.3"), # MIT License
"pyquery": AwesomeVersion("2.0.1"), # BSD
"pyschlage": AwesomeVersion("2024.8.0"), # Apache License 2.0
"pysuezV2": AwesomeVersion("0.2.2"), # Apache 2.0
"python-digitalocean": AwesomeVersion("1.13.2"), # LGPL v3
"python-socks": AwesomeVersion("2.5.3"), # Apache 2
"pywebpush": AwesomeVersion("1.14.1"), # MPL2
"raincloudy": AwesomeVersion("0.0.7"), # Apache License 2.0
"securetar": AwesomeVersion("2024.2.1"), # Apache License 2.0
"simplehound": AwesomeVersion("0.3"), # Apache License, Version 2.0
"sockio": AwesomeVersion("0.15.0"), # GPLv3+
"starkbank-ecdsa": AwesomeVersion("2.2.0"), # MIT License
"streamlabswater": AwesomeVersion("1.0.1"), # Apache 2.0
"vilfo-api-client": AwesomeVersion("0.5.0"), # MIT License
"voluptuous-openapi": AwesomeVersion("0.0.5"), # Apache License 2.0
"voluptuous-serialize": AwesomeVersion("2.6.0"), # Apache License 2.0
"vultr": AwesomeVersion("0.1.2"), # The MIT License (MIT)
"wallbox": AwesomeVersion("0.7.0"), # Apache 2
"zha-quirks": AwesomeVersion("0.0.124"), # Apache License Version 2.0
"zhong-hong-hvac": AwesomeVersion("1.0.13"), # Apache
}

EXCEPTIONS_AND_TODOS = EXCEPTIONS.union(TODO)
Expand Down Expand Up @@ -275,22 +341,25 @@ def check_license_status(package: PackageDefinition) -> bool:
"""Check if package licenses is OSI approved."""
if package.license_expression:
# Prefer 'License-Expression' if it exists
return check_license_expression(package.license_expression) or False
return (
check_license_expression(package.license_expression, package.name) or False
)

if (
package.license_metadata
and (check := check_license_expression(package.license_metadata)) is not None
and (check := check_license_expression(package.license_metadata, package.name))
is not None
):
# Check license metadata if it's a valid SPDX license expression
return check

for approved_license in OSI_APPROVED_LICENSES:
if approved_license in package.license:
return True
if check := check_license_classifier(package.license_classifier, package.name):
return check

return False


def check_license_expression(license_str: str) -> bool | None:
def check_license_expression(license_str: str, package_name: str) -> bool | None:
"""Check if license expression is a valid and approved SPDX license string."""
if license_str == "UNKNOWN" or "\n" in license_str:
# Ignore common errors for license metadata values
Expand All @@ -299,6 +368,11 @@ def check_license_expression(license_str: str) -> bool | None:
try:
expr = licensing.parse(license_str, validate=True)
except ExpressionError:
logger.debug(
"Not a validate metadata license for %s: %s",
package_name,
license_str,
)
return None
return check_spdx_license(expr)

Expand All @@ -314,6 +388,24 @@ def check_spdx_license(expr: LicenseExpression) -> bool:
return False


def check_license_classifier(licenses: list[str], package_name: str) -> bool | None:
"""Check license classifier are OSI approved."""
if not licenses:
return None
if len(licenses) > 1:
# It's not defined how multiple license classifier should be interpreted
# To be safe required ALL to be approved
check = all(
classifier in OSI_APPROVED_LICENSE_CLASSIFIER for classifier in licenses
)
if check is False:
logger.debug(
"Not all classifier approved for %s: %s", package_name, licenses
)
return check
return licenses[0] in OSI_APPROVED_LICENSE_CLASSIFIER


def get_license_str(package: PackageDefinition) -> str:
"""Return license string."""
return (
Expand Down Expand Up @@ -375,6 +467,9 @@ class CheckArgs(Namespace):
def main(argv: Sequence[str] | None = None) -> int:
"""Run the main script."""
parser = ArgumentParser()
parser.add_argument(
"-v", dest="verbose", action="store_true", help="Enable verbose logging"
)
subparsers = parser.add_subparsers(title="Subcommands", required=True)

parser_extract = subparsers.add_parser("extract")
Expand All @@ -398,6 +493,12 @@ def main(argv: Sequence[str] | None = None) -> int:
argv = argv or sys.argv[1:]
args = parser.parse_args(argv)

logging.basicConfig(
format="%(levelname)s:%(message)s", stream=sys.stdout, level=logging.INFO
)
if args.verbose:
logger.setLevel(logging.DEBUG)

if args.action == "extract":
args = cast(ExtractArgs, args)
return extract_licenses(args)
Expand Down