Parse and import both rpm and deb packages metadata

.gitignore

@@ -67,8 +67,6 @@ yarn-error.log
 # This should never be used since we use Yarn, but avoid anyone accidentally committing it
 package-lock.json
 
-rel-eng/custom/__pycache__
-
 # Intellij IDEA
 .idea/
 *.iml
@@ -86,6 +84,12 @@ python/.vscode
 # Python
 venv/
 .venv/
+*.egg-info/
+*.egg
+wheels/
+__pycache__/
+build/
+.pytest_cache/
 
 # Schema
 

python/lzreposync/README.md

@@ -0,0 +1,75 @@
+# lzreposync
+
+TODO: project description
+
+## How to work in this project
+
+1. Create a new virtual environment
+```sh
+$ python3.11 -m venv .venv
+$ . .venv/bin/activate
+```
+2. Install `lzreposync` in *editable* mode
+``` sh
+$ pip install -e .
+```
+3. Install other required dependencies (required by spacewalk and other modules)
+```sh
+pip install pytest
+pip install pycurl
+pip install pyopenssl
+pip install rpm
+pip install psycopg2-binary
+```
+4. Add a path configuration file (**Important!**)
+```
+echo "absolute/path/to/uyuni/python/" > .venv/lib64/python3.11/site-packages/uyuni_python_paths.pth
+# This is a temporary solution that  will allow the lzreposync service to recognize/locate other modules like spacewalk, etc...
+```
+5. Add configuration environment variables
+```sh
+vim /etc/rhn/rhn.conf: # create directory/file if not exists
+
+DB_BACKEND=postgresql
+DB_USER=spacewalk
+DB_PASSWORD=spacewalk
+DB_NAME=susemanager
+DB_HOST=127.0.0.1 # might not work with 'localhost'
+DB_PORT=5432
+PRODUCT_NAME=any
+TRACEBACK_MAIL=any
+DB_SSL_ENABLED=
+DB_SSLROOTCERT=any
+DEBUG=1
+ENABLE_NVREA=1
+MOUNT_POINT=/tmp
+SYNC_SOURCE_PACKAGES=0
+
+# Some values might not be the right ones
+```
+6. Try `lzreposync`
+``` sh
+$ lzreposync -u https://download.opensuse.org/update/leap/15.5/oss/ --type yum [--import-updates]
+```
+
+### How do I ...?
+
+- add new a dependency? Add the *pypi* name to the `dependencies` list in the `[project]` section in `pyproject.toml`.
+
+## Tests
+We're using a special postgres db docker container that contains all the `susemanager` database schema built and ready.
+
+To pull and start the database, you should:
+```sh
+cd /uyuni/java
+sudo make -f Makefile.docker EXECUTOR=podman dockerrun_pg
+# Wait a few seconds until the db is fully initialized
+```
+
+After installing with `pip install .` (or `pip install -e .`), `pytest tests/` runs all tests. Sometimes a `rehash` is required to ensure `.venv/bin/pytest` is used by your shell.
+
+You can connect to the database by:
+```sh
+psql -h localhost -d susemanager -U spacewalk # password: spacewalk
+```
+

python/lzreposync/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lzreposync"
+version = "0.1"
+dependencies = [
+     "memory_profiler",
+     "pytest",
+     "requests",
+     "python-gnupg",
+     "more_itertools"
+]
+
+[project.scripts]
+lzreposync = "lzreposync:main"

python/lzreposync/src/lzreposync/__init__.py

@@ -0,0 +1,218 @@
+#  pylint: disable=missing-module-docstring
+
+import argparse
+import logging
+
+from lzreposync import db_utils, updates_util
+from lzreposync.db_utils import (
+    get_compatible_arches,
+    get_channel_info_by_label,
+    get_all_arches,
+    create_channel,
+    ChannelAlreadyExistsException,
+)
+from lzreposync.deb_repo import DebRepo
+from lzreposync.import_utils import (
+    import_package_batch,
+    batched,
+    import_repository_packages_in_batch,
+)
+from lzreposync.rpm_repo import RPMRepo
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Lazy reposync service",
+        conflict_handler="resolve",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--url",
+        "-u",
+        help="The target url of the remote repository of which we'll "
+        "parse the metadata",
+        dest="url",
+        type=str,
+        default=None,
+    )
+
+    parser.add_argument(
+        "-n",
+        "--name",
+        help="Name of the repository",
+        dest="name",
+        type=str,
+        default="noname",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--debug",
+        help="Show debug messages",
+        action="store_const",
+        dest="loglevel",
+        const=logging.DEBUG,
+        default=logging.INFO,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--cache",
+        help="Path to the cache directory",
+        dest="cache",
+        default=".",
+        type=str,
+    )
+
+    parser.add_argument(
+        "-b",
+        "--batch-size",
+        help="Size of the batch (num of packages by batch)",
+        dest="batch_size",
+        default=20,
+        type=int,
+    )
+
+    parser.add_argument(
+        "-a",
+        "--arch",
+        help="A filter for package architecture. Can be a regex, for example: 'x86_64',  '(x86_64|arch_64)'",
+        default=".*",
+        dest="arch",
+        type=str,
+    )
+
+    parser.add_argument(
+        "--channel",
+        help="The channel label of which you want to synchronize repositories",
+        dest="channel",
+        type=str,
+        default=None,
+    )
+
+    parser.add_argument(
+        "--type",
+        help="Repo type (yum or deb)",
+        dest="repo_type",
+        type=str,
+        default=None,
+    )
+
+    parser.add_argument(
+        "--import-updates",
+        help="Import related patches/updates",
+        action="store_true",
+        dest="import_updates",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--create-channel",
+        help="Create a new channel by providing the 'channel_label', and the 'channel_arch' eg: x86_64.\n"
+        "Eg: --create-channel test_channel x86_64",
+        dest="channel_info",
+        type=str,
+        nargs=2,
+    )
+
+    args = parser.parse_args()
+
+    # Creating a new channel
+    if args.channel_info:
+        channel_label, channel_arch = args.channel_info[0], args.channel_info[1]
+        print(
+            f"Creating a new channel with label: {channel_label}, and arch: {channel_arch}"
+        )
+        try:
+            channel = create_channel(
+                channel_label=channel_label, channel_arch=channel_arch
+            )
+            print(
+                f"Info: successfully created channel: {channel_label} -> id={channel.get_id()}, name={channel.get_label()}"
+            )
+        except ChannelAlreadyExistsException:
+            print(f"Warn: failed to create channel {channel_label}. Already exists !!")
+        return
+
+    arch = args.arch
+    if arch != ".*":
+        # pylint: disable-next=consider-using-f-string
+        arch = "(noarch|{})".format(args.arch)
+
+    logging.getLogger().setLevel(args.loglevel)
+    if args.url:
+        if not args.repo_type:
+            print("ERROR: --type (yum/deb) must be specified when using --url")
+            return
+        if args.repo_type == "yum":
+            repo = RPMRepo(args.name, args.cache, args.url, arch)
+        elif args.repo_type == "deb":
+            repo = DebRepo(args.name, args.cache, args.url, None)
+        else:
+            print(f"ERROR: not supported repo_type: {args.repo_type}")
+            return
+        compatible_arches = get_all_arches()
+        failed = import_repository_packages_in_batch(
+            repo, args.batch_size, compatible_arches=compatible_arches
+        )
+        logging.debug("Completed import with %d failed packages", failed)
+
+    else:
+        # No url specified
+        if args.channel:
+            channel_label = args.channel
+            channel = get_channel_info_by_label(channel_label)
+            if not channel:
+                logging.error("Couldn't fetch channel with label %s", channel_label)
+                return
+            compatible_arches = get_compatible_arches(channel_label)
+            if args.arch and args.arch != ".*" and args.arch not in compatible_arches:
+                logging.error(
+                    "Not compatible arch: %s for channel: %s",
+                    args.channel_arch,
+                    args.channel,
+                )
+                return
+            target_repos = db_utils.get_repositories_by_channel_label(channel_label)
+            for repo in target_repos:
+                if repo.repo_type == "yum":
+                    rpm_repo = RPMRepo(
+                        repo.repo_label, args.cache, repo.source_url, repo.channel_arch
+                    )
+                    logging.debug("Importing package for repo %s", repo.repo_label)
+                    failed = import_repository_packages_in_batch(
+                        rpm_repo,
+                        args.batch_size,
+                        channel,
+                        compatible_arches=compatible_arches,
+                        import_updates=args.import_updates,
+                    )
+                    logging.debug(
+                        "Completed import for repo %s with %d failed packages",
+                        repo.repo_label,
+                        failed,
+                    )
+                elif repo.repo_type == "deb":
+                    dep_repo = DebRepo(
+                        repo.repo_label, args.cache, repo.source_url, repo.channel_label
+                    )
+                    logging.debug("Importing package for repo %s", repo.repo_label)
+                    failed = import_repository_packages_in_batch(
+                        dep_repo,
+                        args.batch_size,
+                        channel,
+                        compatible_arches=compatible_arches,
+                    )
+                    logging.debug(
+                        "Completed import for repo %s with %d failed packages",
+                        repo.repo_label,
+                        failed,
+                    )
+                else:
+                    # TODO: handle repositories other than yum and deb
+                    logging.debug("Not supported repo type: %s", repo.repo_type)
+                    continue
+
+        else:
+            logging.error("Either --url or --channel must be specified")