Skip to content

Commit

Permalink
Simplify tasks.py usage
Browse files Browse the repository at this point in the history 
- Introduce 'alias' names for combinations of hostname (IP) and
  target nixosConfigurations.
- Add task `alias-list` to list all alias names currently configured.
- Change the task.py so that all tasks take the 'alias' name as an
  argument, instead of separate hostname and configuration name.
  This makes it less likely to accidentally apply a configuration to a
  wrong host.
- This also makes it possible to apply tasks to all alias names. As an
  example, `deploy` without arguments could deploy all specified alias
  configurations at once. Such changes to tasks will be implemented
  later in a separate PR.

Signed-off-by: Henri Rosten <henri.rosten@unikie.com>
  • Loading branch information
@henrirosten
henrirosten committed Nov 6, 2023
1 parent f2361c9 commit 592c16c
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 76 deletions.
1 change: 1 addition & 0 deletions shell.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pkgs.mkShell {
python3.pkgs.invoke
python3.pkgs.pycodestyle
python3.pkgs.pylint
python3.pkgs.tabulate
reuse
sops
ssh-to-age
Expand Down
192 changes: 116 additions & 76 deletions tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Union
from collections import OrderedDict
from dataclasses import dataclass

from tabulate import tabulate
from colorlog import ColoredFormatter, default_log_colors
from deploykit import DeployHost, DeployGroup, HostKeyCheck
from deploykit import DeployHost, HostKeyCheck
from invoke import task


################################################################################

ROOT = Path(__file__).parent.resolve()
Expand All @@ -55,6 +59,37 @@
################################################################################


@dataclass(eq=False)
class TargetHost:
"""Represents target host"""

hostname: str
nixosconfig: str


# Below dictionary defines the set of ghaf-infra hosts:
# - Name (e.g. 'build01-dev) defines the aliasname for each target.
# - TargetHost.hostname: host name or IP address of the target.
# - TargetHost.nixosconfig: name of the nixosConfiguration installed/deployed
# on the given host.
TARGETS = OrderedDict(
{
"build01-dev": TargetHost(hostname="51.12.57.124", nixosconfig="build01"),
"ghafhydra-dev": TargetHost(hostname="51.12.56.79", nixosconfig="ghafhydra"),
}
)


def _get_target(alias: str) -> TargetHost:
if alias not in TARGETS:
LOG.fatal("Unknown alias '%s'", alias)
sys.exit(1)
return TARGETS[alias]


################################################################################


def set_log_verbosity(verbosity: int = 1) -> None:
"""Set logging verbosity (0=NOTSET, 1=INFO, or 2=DEBUG)"""
log_levels = [logging.NOTSET, logging.INFO, logging.DEBUG]
Expand Down Expand Up @@ -96,11 +131,17 @@ def _init_logging(verbosity: int = 1) -> None:
set_log_verbosity(1)


def exec_cmd(cmd, raise_on_error=True):
def exec_cmd(cmd, raise_on_error=True, capture_output=True):
"""Run shell command cmd"""
LOG.debug("Running: %s", cmd)
LOG.info("Running: %s", cmd)
try:
return subprocess.run(cmd.split(), capture_output=True, text=True, check=True)
if capture_output:
return subprocess.run(
cmd.split(), capture_output=True, text=True, check=True
)
return subprocess.run(
cmd.split(), text=True, check=True, stdout=subprocess.PIPE
)
except subprocess.CalledProcessError as error:
warn = [f"'{cmd}':"]
if error.stdout:
Expand All @@ -116,6 +157,23 @@ def exec_cmd(cmd, raise_on_error=True):
################################################################################


@task
def alias_list(_c: Any) -> None:
"""
List available targets (i.e. configurations and alias names)
Example usage:
inv list-name
"""
table_rows = []
table_rows.append(["alias", "nixosconfig", "hostname"])
for alias, host in TARGETS.items():
row = [alias, host.nixosconfig, host.hostname]
table_rows.append(row)
table = tabulate(table_rows, headers="firstrow", tablefmt="fancy_outline")
print(f"\nCurrent ghaf-infra targets:\n\n{table}")


@task
def update_sops_files(c: Any) -> None:
"""
Expand All @@ -135,15 +193,16 @@ def update_sops_files(c: Any) -> None:


@task
def print_keys(_c: Any, target: str) -> None:
def print_keys(_c: Any, alias: str) -> None:
"""
Decrypt host private key, print ssh and age public keys for `target`.
Decrypt host private key, print ssh and age public keys for `alias` config.
Example usage:
inv print-keys --target ghafhydra
inv print-keys --target ghafhydra-dev
"""
with TemporaryDirectory() as tmpdir:
decrypt_host_key(target, tmpdir)
nixosconfig = _get_target(alias).nixosconfig
decrypt_host_key(nixosconfig, tmpdir)
key = f"{tmpdir}/etc/ssh/ssh_host_ed25519_key"
pubkey = subprocess.run(
["ssh-keygen", "-y", "-f", f"{key}"],
Expand All @@ -162,28 +221,28 @@ def print_keys(_c: Any, target: str) -> None:
)


def get_deploy_host(target: str = "", hostname: str = "") -> DeployHost:
def get_deploy_host(alias: str = "") -> DeployHost:
"""
Return DeployHost object, given `hostname` and `target`
Return DeployHost object, given `alias`
"""
hostname = _get_target(alias).hostname
deploy_host = DeployHost(
host=hostname,
meta={"target": target},
host_key_check=HostKeyCheck.NONE,
# verbose_ssh=True,
)
return deploy_host


@task
def deploy(_c: Any, target: str, hostname: str) -> None:
def deploy(_c: Any, alias: str) -> None:
"""
Deploy NixOS configuration `target` to host `hostname`.
Deploy the configuration for `alias`.
Example usage:
inv deploy --target ghafhydra --hostname 192.168.1.107
inv deploy --alias ghafhydra-dev
"""
h = get_deploy_host(target, hostname)
h = get_deploy_host(alias)
command = "sudo nixos-rebuild"
res = h.run_local(
["nix", "flake", "archive", "--to", f"ssh://{h.host}", "--json"],
Expand All @@ -193,12 +252,13 @@ def deploy(_c: Any, target: str, hostname: str) -> None:
path = data["path"]
LOG.debug("data['path']: %s", path)
flags = "--option accept-flake-config true"
h.run(f"{command} switch {flags} --flake {path}#{h.meta['target']}")
nixosconfig = _get_target(alias).nixosconfig
h.run(f"{command} switch {flags} --flake {path}#{nixosconfig}")


def decrypt_host_key(target: str, tmpdir: str) -> None:
def decrypt_host_key(nixosconfig: str, tmpdir: str) -> None:
"""
Run sops to extract `target` secret 'ssh_host_ed25519_key'
Run sops to extract `nixosconfig` secret 'ssh_host_ed25519_key'
"""

def opener(path: str, flags: int) -> Union[str, int]:
Expand All @@ -217,13 +277,15 @@ def opener(path: str, flags: int) -> Union[str, int]:
"--extract",
'["ssh_host_ed25519_key"]',
"--decrypt",
f"{ROOT}/hosts/{target}/secrets.yaml",
f"{ROOT}/hosts/{nixosconfig}/secrets.yaml",
],
check=True,
stdout=fh,
)
except subprocess.CalledProcessError:
LOG.warning("Failed reading secret 'ssh_host_ed25519_key' for '%s'", target)
LOG.warning(
"Failed reading secret 'ssh_host_ed25519_key' for '%s'", nixosconfig
)
ask = input("Still continue? [y/N] ")
if ask != "y":
sys.exit(1)
Expand All @@ -240,27 +302,28 @@ def opener(path: str, flags: int) -> Union[str, int]:


@task
def install(c: Any, target: str, hostname: str) -> None:
def install(c: Any, alias) -> None:
"""
Install `target` on `hostname` using nixos-anywhere, deploying host private key.
Note: this will automatically partition and re-format `hostname` hard drive,
Install `alias` configuration using nixos-anywhere, deploying host private key.
Note: this will automatically partition and re-format the target hard drive,
meaning all data on the target will be completely overwritten with no option
to rollback.
Example usage:
inv install --target ghafscan --hostname 192.168.1.109
inv install --alias ghafscan-dev
"""
ask = input(f"Install configuration '{target}' on host '{hostname}'? [y/N] ")
h = get_deploy_host(alias)

ask = input(f"Install configuration '{alias}'? [y/N] ")
if ask != "y":
return

h = get_deploy_host(target, hostname)
# Check sudo nopasswd
try:
h.run("sudo -nv", become_root=True)
except subprocess.CalledProcessError:
LOG.warning(
"sudo on '%s' needs password: installation will likely fail", hostname
"sudo on '%s' needs password: installation will likely fail", h.host
)
ask = input("Still continue? [y/N] ")
if ask != "y":
Expand All @@ -271,7 +334,7 @@ def install(c: Any, target: str, hostname: str) -> None:
except subprocess.CalledProcessError:
pass
else:
LOG.warning("Above address(es) on '%s' use dynamic addressing.", hostname)
LOG.warning("Above address(es) on '%s' use dynamic addressing.", h.host)
LOG.warning(
"This might cause issues if you assume the target host is reachable "
"from any such address also after kexec switch. "
Expand All @@ -282,55 +345,40 @@ def install(c: Any, target: str, hostname: str) -> None:
if ask != "y":
sys.exit(1)

nixosconfig = _get_target(alias).nixosconfig
with TemporaryDirectory() as tmpdir:
decrypt_host_key(target, tmpdir)
decrypt_host_key(nixosconfig, tmpdir)
command = "nix run github:numtide/nixos-anywhere --"
command += f" {hostname} --extra-files {tmpdir} --flake .#{target}"
command += f" {h.host} --extra-files {tmpdir} --flake .#{nixosconfig}"
command += " --option accept-flake-config true"
LOG.warning(command)
c.run(command)

# Reboot
print(f"Wait for {hostname} to start", end="")
wait_for_port(hostname, 22)
reboot(c, hostname)
print(f"Wait for {h.host} to start", end="")
wait_for_port(h.host, 22)
reboot(c, alias)


@task
def build_local(_c: Any, target: str = "") -> None:
def build_local(_c: Any, alias: str = "") -> None:
"""
Build NixOS configuration `target` locally.
If `target` is not specificied, builds all nixosConfigurations in the flake.
Build NixOS configuration `alias` locally.
If `alias` is not specificied, builds all TARGETS.
Example usage:
inv build-local --target ghafhydra
inv build-local --alias ghafhydra-dev
"""
if target:
# For local builds, we pretend hostname is the target
g = DeployGroup([get_deploy_host(hostname=target)])
if alias:
target_configs = [_get_target(alias).nixosconfig]
else:
res = subprocess.run(
["nix", "flake", "show", "--json"],
check=True,
text=True,
stdout=subprocess.PIPE,
)
data = json.loads(res.stdout)
targets = data["nixosConfigurations"]
g = DeployGroup([get_deploy_host(hostname=t) for t in targets])

def _build_local(h: DeployHost) -> None:
h.run_local(
[
"nixos-rebuild",
"build",
"--option",
"accept-flake-config",
"true",
"--flake",
f".#{h.host}",
]
target_configs = [target.nixosconfig for _, target in TARGETS.items()]
for nixosconfig in target_configs:
cmd = (
"nixos-rebuild build --option accept-flake-config true "
f" -v --flake .#{nixosconfig}"
)

g.run_function(_build_local)
exec_cmd(cmd, capture_output=False)


def wait_for_port(host: str, port: int, shutdown: bool = False) -> None:
Expand All @@ -351,14 +399,14 @@ def wait_for_port(host: str, port: int, shutdown: bool = False) -> None:


@task
def reboot(_c: Any, hostname: str) -> None:
def reboot(_c: Any, alias: str) -> None:
"""
Reboot host `hostname`.
Reboot host identified as `alias`.
Example usage:
inv reboot --hostname 192.168.1.112
inv reboot --alias ghafhydra-dev
"""
h = get_deploy_host(hostname=hostname)
h = get_deploy_host(alias)
h.run("sudo reboot &")

print(f"Wait for {h.host} to shutdown", end="")
Expand All @@ -383,42 +431,34 @@ def pre_push(c: Any) -> None:
cmd = "find . -type f -name *.py ! -path *result* ! -path *eggs*"
ret = exec_cmd(cmd)
pyfiles = ret.stdout.replace("\n", " ")
LOG.info("Running black")
cmd = f"black -q {pyfiles}"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Running pylint")
cmd = f"pylint --disable duplicate-code -rn {pyfiles}"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Running pycodestyle")
cmd = f"pycodestyle --max-line-length=90 {pyfiles}"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Running reuse lint")
cmd = "reuse lint"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Running terraform fmt")
cmd = "terraform fmt -check -recursive"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
LOG.warning("Run `terraform fmt -recursive` locally to fix formatting")
sys.exit(1)
LOG.info("Running nix fmt")
cmd = "nix fmt"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Running nix flake check")
cmd = "nix flake check -v --log-format raw"
cmd = "nix flake check -vv"
ret = exec_cmd(cmd, raise_on_error=False)
if not ret:
sys.exit(1)
LOG.info("Building all nixosConfigurations")
build_local(c)
LOG.info("All pre-push checks passed")

0 comments on commit 592c16c

Please sign in to comment.