From 941f95692b1bb387c8e6af547fe4530af6d47d17 Mon Sep 17 00:00:00 2001 From: RonTamG <33351836+RonTamG@users.noreply.github.com> Date: Tue, 6 Feb 2024 09:15:15 +0000 Subject: [PATCH] fix: fixed merged python artifact --- .github/clean_imports.py | 6 +- .github/workflows/python-app.yaml | 2 +- main.py | 17 ++- src/packages.py | 128 ------------------ src/update.py | 202 ---------------------------- tests/test_packages.py | 84 ------------ tests/test_update.py | 211 ------------------------------ 7 files changed, 21 insertions(+), 629 deletions(-) delete mode 100644 src/packages.py delete mode 100644 src/update.py delete mode 100644 tests/test_packages.py delete mode 100644 tests/test_update.py diff --git a/.github/clean_imports.py b/.github/clean_imports.py index 0675f97..695d6aa 100644 --- a/.github/clean_imports.py +++ b/.github/clean_imports.py @@ -33,10 +33,12 @@ def remove_module_imports(code, modules_to_remove): code = sys.stdin.read() modules_to_remove = [ + "src.index", + "src.package", + "src.progressbar", + "src.version", "src.sources_list", "src.file_manager", - "src.update", - "src.packages", "src.install", ] diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 21ac4f3..ab30e3b 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -93,7 +93,7 @@ jobs: - name: Install dependencies run: pip install black isort - name: Merge source files - run: cat src/sources_list.py src/file_manager.py src/update.py src/packages.py src/install.py main.py | python .github/clean_imports.py | isort --float-to-top - | black - > pyapt.py + run: cat src/progress_bar.py src/sources_list.py src/version.py src/package.py src/index.py src/file_manager.py src/install.py main.py | python .github/clean_imports.py | isort --float-to-top - | black - > pyapt.py - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/main.py b/main.py index 04893d8..ca3551d 100644 --- a/main.py +++ b/main.py @@ -10,11 +10,26 @@ from src.install import create_install_script from src.progress_bar import progressbar from src.sources_list import SourcesList -from src.update import get_apt_sources DEFAULT_ARCHITECTURE = "amd64" +def get_apt_sources(url): + """ + return the apt sources of a given index package url + """ + pattern = re.compile( + r"(?P\w+://.+?/.+)/dists/(?P.+?)/(?P.+?)/binary-(?P.+?)/Packages" + ) + result = re.match(pattern, url) + if result is None: + raise ValueError("url has invalid format") + + uri, dist, component, architecture = result.groups() + + return f"{uri} {dist}/{component} {architecture}" + + def tar_dir(path, name): def set_permissions(tarinfo): tarinfo.mode = 0o755 diff --git a/src/packages.py b/src/packages.py deleted file mode 100644 index ddad3ee..0000000 --- a/src/packages.py +++ /dev/null @@ -1,128 +0,0 @@ -import posixpath - -from src.update import get_index_name - - -def extend_unique(iterator, new): - """ - append items from new into iterator if they are not already in iterator - """ - for element in new: - if element not in iterator: - iterator.append(element) - return iterator - - -def get_dependency_list(package): - if "Depends" in package: - return package["Depends"].split(",") - else: - return [] - - -def get_recommended_list(package): - if "Recommends" in package: - return package["Recommends"].split(",") - else: - return [] - - -def get_pre_dependency_list(package): - if "Pre-Depends" in package: - return package["Pre-Depends"].split(",") - else: - return [] - - -def get_package_dependencies( - current, - index, - packages=None, - with_dependencies=True, - with_recommended=True, - with_pre_dependencies=True, - with_required=False, -): - """ - return a list of all package names that are needed to install the package specified in 'name' - - by default includes all pre-dependencies, dependencies and recommended packages - these are the defaults that 'apt install' has - """ # noqa: E501 - current = get_index_name(current) - - if packages is None: - packages = [] - - if current not in index: - raise KeyError(current) - - if index[current]["Priority"] in ["required", "important"] and (not with_required): - return None - - if current in packages: - return None - else: - packages.append(current) - - if with_pre_dependencies: - try: - [ - get_package_dependencies( - dep, - index, - packages, - with_dependencies, - with_recommended, - with_pre_dependencies, - with_required, - ) - for dep in get_pre_dependency_list(index[current]) - ] - except KeyError: - raise KeyError(current) - - if with_dependencies: - try: - [ - get_package_dependencies( - dep, - index, - packages, - with_dependencies, - with_recommended, - with_pre_dependencies, - with_required, - ) - for dep in get_dependency_list(index[current]) - ] - except KeyError: - raise KeyError(current) - - if with_recommended: - try: - [ - get_package_dependencies( - dep, - index, - packages, - with_dependencies, - with_recommended, - with_pre_dependencies, - with_required, - ) - for dep in get_recommended_list(index[current]) - ] - except KeyError: - pass - - return packages - - -def get_package_url(name, index): - """ - return the url to request in order to download the package - """ - URI = 0 - package = index[name] - return posixpath.join(package["Apt-Source"].split()[URI], package["Filename"]) diff --git a/src/update.py b/src/update.py deleted file mode 100644 index 93cf481..0000000 --- a/src/update.py +++ /dev/null @@ -1,202 +0,0 @@ -import logging -import re - - -def generate_index_dictionary(index_data): - """ - create a dictionary of the package index from an index file's data - - original file usually named Packages.xz - """ - index = {} - for data in index_data.strip().split("\n\n"): - matches = re.finditer( - r"^(?P\S+?):(?P(.*)(\n\s.+)?)\n?", data, re.MULTILINE - ) - groups = [(match.group("name"), match.group("value")) for match in matches] - values = {name: value.replace("\n", "").strip() for (name, value) in groups} - - try: - if values["Package"] not in index: - index[values["Package"]] = values - else: - # add to the index if the version is a higher version than the previous - new_version = values["Version"] - current_version = index[values["Package"]]["Version"] - if debian_upstream_compare(new_version, current_version) > 0: - index[values["Package"]] = values - except KeyError: - logging.warning(f'failed to parse: "{data}" as package') - - return index - - -def get_apt_sources(url): - """ - return the apt sources of a given index package url - """ - pattern = re.compile( - r"(?P\w+://.+?/.+)/dists/(?P.+?)/(?P.+?)/binary-(?P.+?)/Packages" - ) - result = re.match(pattern, url) - if result is None: - raise ValueError("url has invalid format") - - uri, dist, component, architecture = result.groups() - - return f"{uri} {dist}/{component} {architecture}" - - -def get_index_name(index): - """ - returns the name of the index without any version or architecture infromation - """ - # remove version - name = index.split()[0].strip() - # remove architecture - if ":" in name: - (name, architecture) = name.split(":") - if architecture != "any": - logging.warning( - f"error processing package {name}: no support for architecture {architecture}" # noqa: E501 - ) - - return name - - -def get_provides_list(index): - """ - this list includes the names of virtual packages that point to this one - """ - if "Provides" in index: - return index["Provides"].split(",") - else: - return [] - - -def add_virtual_indexes(index): - """ - add virtual packages to an index - - modifies the given index. - """ - virtual_packages = {} - - for values in index.values(): - provides = get_provides_list(values) - for virtual_index in provides: - virtual_packages[get_index_name(virtual_index)] = values - - index.update(virtual_packages) - - return index - - -def add_apt_source_field(package_index, source): - """ - add the Apt-Source field to every package in the given package index - """ - for package in package_index.values(): - package["Apt-Source"] = source - - return package_index - - -def split_debian_version(version): - """ - split a debian version string into an epoch, upstream version and debian revision - """ - pattern = re.compile( - r"((\d+):)?(([0-9A-Za-z.+~-]+(?=-))|([0-9A-Za-z.+~]+))(-([0-9A-Za-z.+~]+))?" - ) - result = re.match(pattern, version) - - if result is None: - raise AssertionError( - f'function failed to match "{version}" as a dpkg version string' - ) - if result.string != version: - raise AssertionError( - f"function matched {result.string} instead of {version} when must match all input string" # noqa: E501 - ) - - _, epoch, upstream, _, _, _, revision = result.groups("0") - - return epoch, upstream, revision - - -def order(c): - """ - give a weight to the character to order in the debian version comparison. - """ - if c.isdigit(): - return 0 - elif c.isalpha(): - return ord(c) - elif c == "~": - return -1 - elif c == " ": - return 0 - elif c: - return ord(c) + 256 - else: - return 0 - - -def debian_upstream_compare(first, second): - """ - compare a debian upstream version or revision string with another - """ - first = first.ljust(len(second), " ") - second = second.ljust(len(first), " ") - - for a, b in zip(first, second): - if not a.isdigit() or not b.isdigit(): - ac = order(a) - bc = order(b) - if ac != bc: - return ac - bc - - if a.isdigit() and b.isdigit(): - diff = int(a) - int(b) - if diff != 0: - return diff - else: - return 0 - - -def dpkg_version_compare(a, b): - """ - compare 2 debain version strings - """ - a_epoch, a_version, a_revision = split_debian_version(a) - b_epoch, b_version, b_revision = split_debian_version(b) - - if int(a_epoch) > int(b_epoch): - return 1 - if int(a_epoch) < int(b_epoch): - return -1 - - rc = debian_upstream_compare(a_version, b_version) - if rc: - return rc - - return debian_upstream_compare(a_revision, b_revision) - - -def index_to_package_file_format(package): - return "\n".join([f"{key}: {value}" for key, value in package.items()]) - - -def combine_indexes(index_1, index_2): - """ - combine two indexes into one. - in case of conflicting packages, keep package with latest version. - """ - for key, value in index_2.items(): - if key not in index_1: - index_1[key] = value - elif dpkg_version_compare(index_1[key]["Version"], value["Version"]) < 0: - index_1[key] = value - - return index_1 diff --git a/tests/test_packages.py b/tests/test_packages.py deleted file mode 100644 index 397ae0c..0000000 --- a/tests/test_packages.py +++ /dev/null @@ -1,84 +0,0 @@ -from pathlib import Path - -from src.packages import get_package_dependencies, get_package_url -from src.update import ( - add_apt_source_field, - add_virtual_indexes, - generate_index_dictionary, -) - - -def test_package_dependencies_collection_with_dependencies_and_recommended_is_successfull(): # noqa: E501 - index = valid_index() - - result = get_package_dependencies("python3", index) - - assert len(result) == 43 - - -def test_package_dependencies_collection_without_recommended_packages_is_successful(): - index = valid_index() - - result = get_package_dependencies("python3", index, with_recommended=False) - - assert len(result) == 33 - - -def test_package_dependencies_collection_with_required_packages_is_successful(): - index = valid_index() - - result = get_package_dependencies("python3", index, with_required=True) - - assert len(result) == 104 - - -def test_package_dependencies_collection_without_pre_dependencies_is_successful(): - index = valid_index() - - result = get_package_dependencies("python3", index, with_pre_dependencies=False) - - assert len(result) == 42 - - -def test_package_dependencies_collection_ignores_missing_recommended_packages(): - index = index_missing_ca_certificates_package() - - result = get_package_dependencies("python3", index) - - assert len(result) == 41 - - -def test_package_download_url_generation_is_successfull(): - source = "http://deb.debian.org/debian" - index = valid_index_with_apt_source(source) - expected = "http://deb.debian.org/debian/./python3_3.11.4-5+b1_amd64.deb" - - result = get_package_url("python3", index) - - assert result == expected - - -def valid_index(): - with open( - Path("tests", "resources", "python_Packages"), "r", encoding="utf-8" - ) as index_file: - index_data = index_file.read() - - index = generate_index_dictionary(index_data) - add_virtual_indexes(index) - - return index - - -def valid_index_with_apt_source(source): - index = valid_index() - add_apt_source_field(index, source) - - return index - - -def index_missing_ca_certificates_package(): - index = valid_index() - index.pop("ca-certificates") - - return index diff --git a/tests/test_update.py b/tests/test_update.py deleted file mode 100644 index 6a63baf..0000000 --- a/tests/test_update.py +++ /dev/null @@ -1,211 +0,0 @@ -from pathlib import Path - -import pytest - -from src.update import ( - add_apt_source_field, - add_virtual_indexes, - combine_indexes, - dpkg_version_compare, - generate_index_dictionary, - get_apt_sources, - index_to_package_file_format, - split_debian_version, -) - - -def full_index_data(): - with open( - Path("tests", "resources", "python_Packages"), "r", encoding="utf-8" - ) as index_file: - return index_file.read() - - -def index_data_with_multiline_field(): - index_data = full_index_data() - return index_data.replace( - "Depends: python3.11 (>= 3.11.4-1~), libpython3-stdlib (= 3.11.4-5+b1)", - "Depends: python3.11 (>= 3.11.4-1~),\n libpython3-stdlib (= 3.11.4-5+b1)", - ) - - -def genius_version_1_package_data(): - return """\ -Package: genius -Version: 1.0.25-2 -Installed-Size: 774 -Maintainer: Felipe Sateler -Architecture: amd64 -Depends: libc6 (>= 2.14), libglib2.0-0 (>= 2.35.9), libgmp10, libmpfr6 (>= 3.1.3), libreadline8 (>= 6.0), libtinfo6 (>= 6), genius-common (>= 1.0.25-2), genius-common (<= 1.0.25-2.) -Description: advanced general purpose calculator program (CLI frontend) -Homepage: https://www.5z.com/jirka/genius.html -Description-md5: 91ce686a0384efccfc97b0de617f8732 -Tag: field::mathematics, role::program, uitoolkit::ncurses -Section: gnome -Priority: optional -Filename: pool/main/g/genius/genius_1.0.25-2_amd64.deb -Size: 348540 -MD5sum: a0521c13eb57afa16d9c5aa4cef01100 -SHA256: c7d6931b92e4902814e59194a16c3936d8622eda2cb5cfc222ee17ae99bb7f82 -""" # noqa: E501 - - -def genius_version_2_package_data(): - return """\ -Package: genius -Version: 2.0.25-2 -Installed-Size: 774 -Maintainer: Felipe Sateler -Architecture: amd64 -Depends: libc6 (>= 2.14), libglib2.0-0 (>= 2.35.9), libgmp10, libmpfr6 (>= 3.1.3), libreadline8 (>= 6.0), libtinfo6 (>= 6), genius-common (>= 1.0.25-2), genius-common (<= 1.0.25-2.) -Description: advanced general purpose calculator program (CLI frontend) -Homepage: https://www.5z.com/jirka/genius.html -Description-md5: 91ce686a0384efccfc97b0de617f8732 -Tag: field::mathematics, role::program, uitoolkit::ncurses -Section: gnome -Priority: optional -Filename: pool/main/g/genius/genius_2.0.25-2_amd64.deb -Size: 348540 -MD5sum: a0521c13eb57afa16d9c5aa4cef01100 -SHA256: c7d6931b92e4902814e59194a16c3936d8622eda2cb5cfc222ee17ae99bb7f82 -""" # noqa: E501 - - -def genius_package_data_with_provides_field(): - data = genius_version_2_package_data() - return data + "Provides: genius_virtual\n" - - -def package_file_urls_with_sources(): - urls = [ - "http://deb.debian.org/debian/dists/bullseye/main/binary-amd64/Packages.xz", - "http://security.debian.org/debian-security/dists/bullseye-security/main/binary-amd64/Packages.xz", - ] - expected = [ - "http://deb.debian.org/debian bullseye/main amd64", - "http://security.debian.org/debian-security bullseye-security/main amd64", - ] - - return urls, expected - - -def valid_index(): - return generate_index_dictionary(full_index_data()) - - -def test_full_index_data_generates_full_index_dictionary(): - index_data = full_index_data() - - result = generate_index_dictionary(index_data) - - assert len(result) == 103 - - -def test_index_data_with_multiline_fields_combines_the_lines(): - index_data = index_data_with_multiline_field() - - index = generate_index_dictionary(index_data) - result = index["python3"]["Depends"] - - assert result == "python3.11 (>= 3.11.4-1~), libpython3-stdlib (= 3.11.4-5+b1)" - - -def test_multiple_indexes_with_same_package_when_combined_keep_latest_version(): - index_data_1 = genius_version_1_package_data() - index_data_2 = genius_version_2_package_data() - - index_1 = generate_index_dictionary(index_data_1) - index_2 = generate_index_dictionary(index_data_2) - - result = combine_indexes(index_1, index_2) - - assert result["genius"]["Version"] == "2.0.25-2" - - -def test_index_can_have_virtual_packages_added_according_to_provider_fields(): - index_data = genius_package_data_with_provides_field() - - index = generate_index_dictionary(index_data) - result = add_virtual_indexes(index) - - assert "genius_virtual" in result - - -def test_package_file_urls_can_be_converted_to_original_apt_source(): - package_urls, expected_sources = package_file_urls_with_sources() - - result = [get_apt_sources(url) for url in package_urls] - - assert all(item in result for item in expected_sources) - - -def test_index_adding_apt_source_field_is_successful(): - index = valid_index() - source = "http://deb.debian.org/debian bullseye/main amd64" - - result = add_apt_source_field(index, source=source) - - assert all(["Apt-Source" in package for package in result.values()]) - assert all([package["Apt-Source"] == source for package in result.values()]) - - -@pytest.mark.parametrize( - "version, expected_epoch, expected_upstream, expected_revision", - [ - ("1:2.30.2-1+deb11u2", "1", "2.30.2", "1+deb11u2"), - ("4.7-1", "0", "4.7", "1"), - ("0.0.14", "0", "0.0.14", "0"), - ("1:8.4p1-5+deb11u1", "1", "8.4p1", "5+deb11u1"), - ("3.0043", "0", "3.0043", "0"), - ("0.11b-20160615-2", "0", "0.11b-20160615", "2"), - ("1:0", "1", "0", "0"), - ], -) -def test_debian_version_split_into_base_parts_is_successfull( - version, expected_epoch, expected_upstream, expected_revision -): - epoch, upstream, revision = split_debian_version(version) - - assert epoch == expected_epoch - assert upstream == expected_upstream - assert revision == expected_revision - - -@pytest.mark.parametrize( - "a, b, result", - [ - ("1:0", "2:0", -1), - ("0:1-1", "0:2-1", -1), - ("0:1-1", "0:1-2", -1), - # Test for version equality. - ("0:0-0", "0:0-0", 0), - ("0:0-00", "0:00-0", 0), - ("1:2-3", "1:2-3", 0), - # Test for epoch difference. - ("0:0-0", "1:0-0", -1), - ("1:0-0", "0:0-0", 1), - ("11.1+deb11u6", "1:11.1+deb11u6", -1), - # Test for version component difference. - ("0:a-0", "0:b-0", -1), - ("0:b-0", "0:a-0", 1), - ("11.1+deb11u6", "11.2+deb11u6", -1), - ("11.1+deb11u6", "11.1+deb11u6", 0), - ("11.1+deb11u6", "11.0+deb11u6", 1), - # Test for revision component difference. - ("0:0-a", "0:0-b", -1), - ("0:0-b", "0:0-a", 1), - ("1.6.1-5", "1.6.1-5+deb11u1", -299), - ("1:16.28.0~dfsg-0+deb11u1", "1:16.28.0~dfsg-0+deb11u2", -1), - ], -) -def test_debian_version_compare_is_successfull(a, b, result): - assert dpkg_version_compare(a, b) == result - - -def test_index_can_return_to_package_file(): - expected = genius_version_1_package_data().strip() - index = generate_index_dictionary(expected) - - result = index_to_package_file_format(index["genius"]) - - assert result == expected