Add command component sync

commodore/cli.py

@@ -15,15 +15,17 @@
 from .config import Config, Migration, parse_dynamic_facts_from_cli
 from .helpers import clean_working_tree
 from .compile import compile as _compile
-from .component.template import ComponentTemplater
+from .component import Component
 from .component.compile import compile_component
+from .component.template import ComponentTemplater
+from .dependency_syncer import sync_dependencies
 from .inventory.render import extract_components, extract_packages, extract_parameters
 from .inventory.parameters import InventoryFacts
 from .inventory.lint import LINTERS
 from .login import login, fetch_token
+from .package import Package
 from .package.compile import compile_package
 from .package.template import PackageTemplater
-from .package.sync import sync_packages
 
 pass_config = click.make_pass_decorator(Config)
 
@@ -590,6 +592,77 @@ def component_compile(
     compile_component(config, path, alias, values, search_paths, output, name)
 
 
+@component.command("sync", short_help="Synchronize components to template")
+@verbosity
+@pass_config
+@click.argument(
+    "component_list", type=click.Path(file_okay=True, dir_okay=False, exists=True)
+)
+@click.option(
+    "--github-token",
+    help="GitHub API token",
+    envvar="COMMODORE_GITHUB_TOKEN",
+    default="",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Don't create or update PRs", default=False
+)
+@click.option(
+    "--pr-branch",
+    "-b",
+    metavar="BRANCH",
+    default="template-sync",
+    type=str,
+    help="Branch name to use for updates from template",
+)
+@click.option(
+    "--pr-label",
+    "-l",
+    metavar="LABEL",
+    default=[],
+    multiple=True,
+    help="Labels to set on the PR. Can be repeated",
+)
+def component_sync(
+    config: Config,
+    verbose: int,
+    component_list: str,
+    github_token: str,
+    dry_run: bool,
+    pr_branch: str,
+    pr_label: Iterable[str],
+):
+    """This command processes all components listed in the provided `COMPONENT_LIST`
+    YAML file.
+
+    Currently, the command only supports updating components hosted on GitHub. The
+    command expects that the YAML file contains a single document with a list of GitHub
+    repositories in form `organization/repository-name`.
+
+    The command clones each component and runs `component update` on the local copy. If
+    there are any changes, the command creates a PR for the changes. For each component,
+    the command parses the component's `.cruft.json` to determine the template repository
+    and template version for the component. The command bases each PR on the default
+    branch of the corresponding component repository as reported by the GitHub API.
+
+    The command requires a GitHub Access token with the 'public_repo' permission, which
+    is required to create PRs on public repositories. If you want to manage private
+    repos, the access token may require additional permissions.
+    """
+    config.update_verbosity(verbose)
+    config.github_token = github_token
+
+    sync_dependencies(
+        config,
+        Path(component_list),
+        dry_run,
+        pr_branch,
+        pr_label,
+        Component,
+        ComponentTemplater,
+    )
+
+
 @commodore.group(short_help="Interact with a Commodore config package")
 @verbosity
 @pass_config
@@ -879,7 +952,15 @@ def package_sync(
     config.update_verbosity(verbose)
     config.github_token = github_token
 
-    sync_packages(config, Path(package_list), dry_run, pr_branch, pr_label)
+    sync_dependencies(
+        config,
+        Path(package_list),
+        dry_run,
+        pr_branch,
+        pr_label,
+        Package,
+        PackageTemplater,
+    )
 
 
 @commodore.group(short_help="Interact with a Commodore inventory")

commodore/component/__init__.py

@@ -20,6 +20,18 @@ class Component:
     _dir: P
     _sub_path: str
 
+    @classmethod
+    def clone(cls, cfg, clone_url: str, name: str, version: str = "master"):
+        cdep = cfg.register_dependency_repo(clone_url)
+        c = Component(
+            name,
+            cdep,
+            directory=component_dir(cfg.work_dir, name),
+            version=version,
+        )
+        c.checkout()
+        return c
+
     # pylint: disable=too-many-arguments
     def __init__(
         self,
@@ -114,6 +126,10 @@ def repo_directory(self) -> P:
     def target_directory(self) -> P:
         return self._dir / self._sub_path
 
+    @property
+    def target_dir(self) -> P:
+        return self.target_directory
+
     @property
     def class_file(self) -> P:
         return self.target_directory / "class" / f"{self.name}.yml"

commodore/package/sync.py → commodore/dependency_syncer.py

@@ -4,7 +4,7 @@
 import time
 
 from pathlib import Path
-from typing import Iterable
+from typing import Iterable, Union, Type
 
 import click
 import git
@@ -16,61 +16,68 @@
 from commodore.config import Config
 from commodore.helpers import yaml_load
 
-from . import Package
-from .template import PackageTemplater
+from commodore.component import Component
+from commodore.package import Package
 
+from commodore.dependency_templater import Templater
 
-def sync_packages(
+
+def sync_dependencies(
     config: Config,
-    package_list: Path,
+    dependency_list: Path,
     dry_run: bool,
     pr_branch: str,
     pr_label: Iterable[str],
+    deptype: Type[Union[Component, Package]],
+    templater: Type[Templater],
 ) -> None:
     if not config.github_token:
         raise click.ClickException("Can't continue, missing GitHub API token.")
 
+    deptype_str = deptype.__name__.lower()
+
     try:
-        pkgs = yaml_load(package_list)
-        if not isinstance(pkgs, list):
-            typ = type(pkgs)
-            raise ValueError(f"unexpected type: {typ}")
+        deps = yaml_load(dependency_list)
+        if not isinstance(deps, list):
+            raise ValueError(f"unexpected type: {type_name(deps)}")
     except ValueError as e:
-        raise click.ClickException(f"Expected a list in '{package_list}', but got {e}")
+        raise click.ClickException(
+            f"Expected a list in '{dependency_list}', but got {e}"
+        )
     except (yaml.parser.ParserError, yaml.scanner.ScannerError):
-        raise click.ClickException(f"Failed to parse YAML in '{package_list}'")
+        raise click.ClickException(f"Failed to parse YAML in '{dependency_list}'")
 
     gh = github.Github(config.github_token)
-    pkg_count = len(pkgs)
-    for i, pn in enumerate(pkgs, start=1):
-        click.secho(f"Synchronizing {pn}", bold=True)
-        porg, preponame = pn.split("/")
-        pname = preponame.replace("package-", "", 1)
+    dep_count = len(deps)
+    for i, dn in enumerate(deps, start=1):
+        click.secho(f"Synchronizing {dn}", bold=True)
+        _, dreponame = dn.split("/")
+        dname = dreponame.replace(f"{deptype_str}-", "", 1)
 
-        # Clone package
+        # Clone dependency
         try:
-            gr = gh.get_repo(pn)
+            gr = gh.get_repo(dn)
         except github.UnknownObjectException:
-            click.secho(f" > Repository {pn} doesn't exist, skipping...", fg="yellow")
+            click.secho(f" > Repository {dn} doesn't exist, skipping...", fg="yellow")
             continue
-        p = Package.clone(config, gr.clone_url, pname, version=gr.default_branch)
+        d = deptype.clone(config, gr.clone_url, dname, version=gr.default_branch)
 
-        if not (p.target_dir / ".cruft.json").is_file():
-            click.echo(f" > Skipping repo {pn} which doesn't have `.cruft.json`")
+        if not (d.target_dir / ".cruft.json").is_file():
+            click.echo(f" > Skipping repo {dn} which doesn't have `.cruft.json`")
             continue
 
-        # Run `package update`
-        t = PackageTemplater.from_existing(config, p.target_dir)
+        # Update the dependency
+        t = templater.from_existing(config, d.target_dir)
         changed = t.update(print_completion_message=False)
 
         # Create or update PR if there were updates
         if changed:
-            ensure_branch(p, pr_branch)
-            msg = ensure_pr(p, pn, gr, dry_run, pr_branch, pr_label)
+            ensure_branch(d, pr_branch)
+            msg = ensure_pr(d, dn, gr, dry_run, pr_branch, pr_label)
             click.secho(msg, bold=True)
 
-            if i < pkg_count:
-                # except when processing the last package in the list, sleep for 1-2
+            if i < dep_count:
+                # except when processing the last dependency in the list, sleep for 1-2
                 # seconds to avoid hitting secondary rate-limits for PR creation. No
                 # need to sleep if we're not creating a PR.
                 # Without the #nosec annotations bandit warns (correctly) that
@@ -80,7 +87,7 @@ def sync_packages(
                 backoff = 1.0 + random.random()  # nosec
                 time.sleep(backoff)
         else:
-            click.secho(f"Package {pn} already up-to-date", bold=True)
+            click.secho(f"Package {dn} already up-to-date", bold=True)
 
 
 def message_body(c: git.objects.commit.Commit) -> str:
@@ -92,12 +99,14 @@ def message_body(c: git.objects.commit.Commit) -> str:
     return "\n\n".join(paragraphs[1:])
 
 
-def ensure_branch(p: Package, branch_name: str):
+def ensure_branch(d: Union[Component, Package], branch_name: str):
     """Create or reset `template-sync` branch pointing to our new template update
     commit."""
-    if not p.repo:
-        raise ValueError("package repo not initialized")
-    r = p.repo.repo
+    deptype = type_name(d)
+
+    if not d.repo:
+        raise ValueError(f"{deptype} repo not initialized")
+    r = d.repo.repo
     has_sync_branch = any(h.name == branch_name for h in r.heads)
 
     if not has_sync_branch:
@@ -109,32 +118,34 @@ def ensure_branch(p: Package, branch_name: str):
 
 
 def ensure_pr(
-    p: Package,
-    pn: str,
+    d: Union[Component, Package],
+    dn: str,
     gr: Repository,
     dry_run: bool,
     branch_name: str,
     pr_labels: Iterable[str],
 ) -> str:
     """Create or update template sync PR."""
-    if not p.repo:
-        raise ValueError("package repo not initialized")
+    deptype = type_name(d)
+
+    if not d.repo:
+        raise ValueError(f"{deptype} repo not initialized")
 
     prs = gr.get_pulls(state="open")
     has_sync_pr = any(pr.head.ref == branch_name for pr in prs)
 
     cu = "update" if has_sync_pr else "create"
     if dry_run:
-        return f"Would {cu} PR for {pn}"
+        return f"Would {cu} PR for {dn}"
 
-    r = p.repo.repo
+    r = d.repo.repo
     r.remote().push(branch_name, force=True)
     pr_body = message_body(r.head.commit)
 
     try:
         if not has_sync_pr:
             sync_pr = gr.create_pull(
-                "Update from package template",
+                f"Update from {deptype} template",
                 pr_body,
                 gr.default_branch,
                 branch_name,
@@ -145,8 +156,12 @@ def ensure_pr(
         sync_pr.add_to_labels(*list(pr_labels))
     except github.UnknownObjectException:
         return (
-            f"Unable to {cu} PR for {pn}. "
+            f"Unable to {cu} PR for {dn}. "
             + "Please make sure your GitHub token has permission 'public_repo'"
         )
 
-    return f"PR for package {pn} successfully {cu}d"
+    return f"PR for {deptype} {dn} successfully {cu}d"
+
+
+def type_name(o: object) -> str:
+    return type(o).__name__.lower()

commodore/dependency_templater.py

@@ -58,6 +58,11 @@ def __init__(
 
             self.output_dir = odir
 
+    @classmethod
+    @abstractmethod
+    def from_existing(cls, config: Config, path: Path):
+        ...
+
     @classmethod
     def _base_from_existing(cls, config: Config, path: Path, deptype: str):
         if not path.is_dir():

commodore/package/__init__.py

@@ -16,7 +16,7 @@ class Package:
 
     @classmethod
     def clone(cls, cfg, clone_url: str, name: str, version: str = "master"):
-        pdep = MultiDependency(clone_url, cfg.inventory.dependencies_dir)
+        pdep = cfg.register_dependency_repo(clone_url)
         p = Package(
             name,
             pdep,

docs/modules/ROOT/pages/reference/cli.adoc

@@ -230,6 +230,27 @@ This command doesn't have any command line options.
 *--help*::
   Show component new usage and options then exit.
 
+== Component Sync
+
+*--github-token* TEXT::
+  The GitHub access token to use when interacting with the GitHub API.
+  We recommend passing the token in environment variable `COMMODORE_GITHUB_TOKEN`.
+
+*--dry-run*::
+  If this flag is provided, the sync command doesn't push the template branch to GitHub and doesn't create or update any PRs.
+
+*-b, --pr-branch* BRANCH::
+  The branch name to use when pushing updates to GitHub.
+  By default `template-sync` is used used as the branch name.
++
+NOTE: Changing this flag will orphan any open update PRs created with a different branch name.
+
+*-l, --pr-label* LABEL::
+  Labels to set on the PR.
+  Can be repeated.
++
+When changing the set of labels, new labels will be added to open PRs.
+However, labels added by previous runs can't be removed since we've got no easy way to distinguish between old labels and externally added labels.
 
 == Inventory Components / Packages / Show