diff --git a/README.md b/README.md index 574f282..e0a3ee3 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Please use the interactive mode and doublecheck commands before # INSTALLATION -Automatix requires Python ≥ 3.6. +Automatix requires Python ≥ 3.8. ``` pip install automatix diff --git a/automatix/automatix.py b/automatix/automatix.py index eb33c06..f034f85 100644 --- a/automatix/automatix.py +++ b/automatix/automatix.py @@ -1,9 +1,11 @@ import logging from argparse import Namespace from collections import OrderedDict +from functools import cached_property from typing import List -from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS +from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS, ReloadFromFile +from .config import get_script from .environment import PipelineEnvironment @@ -31,9 +33,17 @@ def __init__( logger=logging.getLogger(config['logger']), ) + @cached_property + def command_lists(self) -> dict: + return { + 'always': self.build_command_list(pipeline='always'), + 'main': self.build_command_list(pipeline='pipeline'), + 'cleanup': self.build_command_list(pipeline='cleanup'), + } + def build_command_list(self, pipeline: str) -> List[Command]: command_list = [] - for index, cmd in enumerate(self.script[pipeline]): + for index, cmd in enumerate(self.script.get(pipeline, [])): new_cmd = self.cmd_class( cmd=cmd, index=index, @@ -45,6 +55,10 @@ def build_command_list(self, pipeline: str) -> List[Command]: self.env.vars[new_cmd.assignment_var] = f'{{{new_cmd.assignment_var}}}' return command_list + def reload_script(self): + self.script = get_script(args=self.env.cmd_args) + del self.command_lists # Clear cache + def print_main_data(self): self.env.LOG.info('\n\n') self.env.LOG.info(f' ------ Overview ------') @@ -58,32 +72,43 @@ def print_command_line_steps(self, command_list: List[Command]): for cmd in command_list: self.env.LOG.info(f"({cmd.index}) [{cmd.orig_key}]: {cmd.get_resolved_value()}") - def execute_main_pipeline(self, command_list: List[Command]): - self.env.LOG.info('\n------------------------------') - self.env.LOG.info(' --- Start MAIN pipeline ---') + def _execute_command_list(self, name: str, start_index: int, treat_as_main: bool): + try: + steps = self.script.get('steps') + for cmd in self.command_lists[name][start_index:]: + if treat_as_main: + if steps and (self.script['exclude'] == (cmd.index in steps)): + # Case 1: exclude is True and index is in steps => skip + # Case 2: exclude is False and index is in steps => execute + self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip') + continue + cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force) + else: + cmd.execute() + except ReloadFromFile as exc: + self.env.LOG.info(f'\n({exc.index}) Reload script from file and retry.') + self.reload_script() + self._execute_command_list(name=name, start_index=exc.index, treat_as_main=treat_as_main) + + def execute_pipeline(self, name: str): + if not self.command_lists[name]: + return + + if name == 'main': + treat_as_main = True + start_index = self.env.cmd_args.jump_to + else: + treat_as_main = False + start_index = 0 - steps = self.script.get('steps') + self.env.LOG.info('\n------------------------------') + self.env.LOG.info(f' --- Start {name.upper()} pipeline ---') - for cmd in command_list[self.env.cmd_args.jump_to:]: - if steps and (self.script['exclude'] == (cmd.index in steps)): - # Case 1: exclude is True and index is in steps => skip - # Case 2: exclude is False and index is in steps => execute - self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip') - continue - cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force) + self._execute_command_list(name=name, start_index=start_index, treat_as_main=treat_as_main) - self.env.LOG.info('\n --- End MAIN pipeline ---') + self.env.LOG.info(f'\n --- End {name.upper()} pipeline ---') self.env.LOG.info('------------------------------\n') - def execute_extra_pipeline(self, pipeline: str): - if self.script.get(pipeline): - self.env.LOG.info('\n------------------------------') - self.env.LOG.info(f' --- Start {pipeline.upper()} pipeline ---') - for cmd in self.build_command_list(pipeline=pipeline): - cmd.execute() - self.env.LOG.info(f'\n --- End {pipeline.upper()} pipeline ---') - self.env.LOG.info('------------------------------\n') - def run(self): self.env.LOG.info('\n\n') self.env.LOG.info('//////////////////////////////////////////////////////////////////////') @@ -92,24 +117,22 @@ def run(self): PERSISTENT_VARS.clear() - command_list = self.build_command_list(pipeline='pipeline') - - self.execute_extra_pipeline(pipeline='always') + self.execute_pipeline(name='always') self.print_main_data() - self.print_command_line_steps(command_list) + self.print_command_line_steps(command_list=self.command_lists['main']) if self.env.cmd_args.print_overview: exit() try: - self.execute_main_pipeline(command_list=command_list) + self.execute_pipeline(name='main') except (AbortException, SkipBatchItemException): self.env.LOG.debug('Abort requested. Cleaning up.') - self.execute_extra_pipeline(pipeline='cleanup') + self.execute_pipeline(name='cleanup') self.env.LOG.debug('Clean up done. Exiting.') raise - self.execute_extra_pipeline(pipeline='cleanup') + self.execute_pipeline(name='cleanup') self.env.LOG.info('---------------------------------------------------------------') self.env.LOG.info('Automatix finished: Congratulations and have a N.I.C.E. day :-)') diff --git a/automatix/command.py b/automatix/command.py index cd39389..1655bc5 100644 --- a/automatix/command.py +++ b/automatix/command.py @@ -24,6 +24,7 @@ def __setattr__(self, key: str, value): POSSIBLE_ANSWERS = { 'p': 'proceed (default)', 'r': 'retry', + 'R': 'reload from file and retry command (same index)', 's': 'skip', 'a': 'abort', 'c': 'abort & continue to next (CSV processing)', @@ -99,8 +100,8 @@ def execute(self, interactive: bool = False, force: bool = False): if self.get_type() == 'manual' or interactive: self.env.LOG.debug('Ask for user interaction.') - answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'a']) - # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass + answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'R', 'a']) + # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass if answer == 's': return @@ -122,8 +123,8 @@ def execute(self, interactive: bool = False, force: bool = False): if force: return - err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'a']) - # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass + err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'R', 'a']) + # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass if err_answer == 'r': return self.execute(interactive) @@ -142,19 +143,21 @@ def _ask_user(self, question: str, allowed_options: list) -> str: if self.env.batch_mode: allowed_options.append('c') - options = ', '.join([f'{k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options]) + options = '\n'.join([f' {k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options]) answer = None while answer not in allowed_options: if answer is not None: self.env.LOG.info('Invalid input. Try again.') - answer = input(f'{question} ({options})\a') + answer = input(f'{question}\n{options}\nYour answer: \a') if answer == '': # default answer = 'p' if answer == 'a': raise AbortException(1) + if answer == 'R': + raise ReloadFromFile(index=self.index) if self.env.batch_mode and answer == 'c': raise SkipBatchItemException() @@ -204,6 +207,8 @@ def _python_action(self) -> int: 'Seems you are trying to use bundlewrap functions without having bundlewrap support enabled.' ' Please check your configuration.') return 1 + if isinstance(exc.__context__, ReloadFromFile): + exc.__suppress_context__ = True self.env.LOG.exception('Unknown error occured:') return 1 @@ -344,3 +349,11 @@ class SkipBatchItemException(Exception): class UnknownCommandException(Exception): pass + + +class ReloadFromFile(Exception): + def __init__(self, index: int): + self.index = index + + def __int__(self): + return self.index diff --git a/automatix/command_test.py b/automatix/command_test.py index fc2ce93..7a7a227 100644 --- a/automatix/command_test.py +++ b/automatix/command_test.py @@ -9,7 +9,7 @@ def test__execute_remote_cmd(ssh_up): - cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, env=environment) + cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, pipeline='pipeline', env=environment) cmd.execute() try: run_command_and_check('ssh docker-test ls /test_remote_cmd >/dev/null') @@ -23,7 +23,7 @@ def test__execute_local_cmd(capfd): # empty captured stdin and stderr _ = capfd.readouterr() - cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, env=environment) + cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, pipeline='pipeline', env=environment) cmd.execute() out, err = capfd.readouterr() @@ -44,7 +44,7 @@ def test__execute_local_with_condition(capfd): # empty captured stdin and stderr _ = capfd.readouterr() - cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, index=2, env=environment) + cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, pipeline='pipeline', index=2, env=environment) cmd.execute() out, err = capfd.readouterr() @@ -59,10 +59,10 @@ def test__execute_python_cmd(): PERSISTENT_VARS.update(locals()) """ - cmd = Command(cmd={'python': test_cmd}, index=2, env=environment) + cmd = Command(cmd={'python': test_cmd}, index=2, pipeline='pipeline', env=environment) cmd.execute() - cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, env=environment) + cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, pipeline='pipeline', env=environment) cmd.execute() diff --git a/automatix/config.py b/automatix/config.py index 35c8a34..d30b93e 100644 --- a/automatix/config.py +++ b/automatix/config.py @@ -2,8 +2,8 @@ import logging import os import re -import sys from collections import OrderedDict +from importlib import metadata from time import sleep import yaml @@ -16,11 +16,6 @@ def read_yaml(yamlfile: str) -> dict: return yaml.load(file.read(), Loader=yaml.SafeLoader) -if sys.version_info >= (3, 8): - from importlib import metadata -else: - import importlib_metadata as metadata - try: from argcomplete import autocomplete from .bash_completion import ScriptFileCompleter diff --git a/example.automatix.cfg.yaml b/example.automatix.cfg.yaml index 57cb666..836b216 100644 --- a/example.automatix.cfg.yaml +++ b/example.automatix.cfg.yaml @@ -1,42 +1,32 @@ # Path to scripts directory - script_dir: ~/automatix_script_files # Global constants for use in pipeline scripts - constants: apt_update: 'apt-get -qy update' apt_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends upgrade' apt_full_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends full-upgrade' # Encoding - encoding: utf-8 # Path to shell imports - import_path: '.' # SSH Command used for remote connections - ssh_cmd: 'ssh {hostname} sudo ' # Temporary directory on remote machines for shell imports - remote_tmp_dir: 'automatix_tmp' # Logger - logger: mylogger # Logging library - logging_lib: mylib.logging # Bundlewrap support, bundlewrap has to be installed (default: false) - bundlewrap: true # Teamvault / Secret support, bundlewrap-teamvault has to be installed (default: false) - teamvault: true diff --git a/pytest.ini b/pytest.ini index 3bdab05..7073109 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] addopts = --durations=0 --docker-compose=tests/docker-compose.yml -testpaths = automatix \ No newline at end of file +testpaths = automatix diff --git a/setup.py b/setup.py index 0662a3d..26ab7e1 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,10 @@ author='Johannes Paul, //SEIBERT/MEDIA GmbH', author_email='jpaul@seibert-media.net', license='MIT', - python_requires='>=3.6', + python_requires='>=3.8', install_requires=[ + 'cython<3.0.0', 'pyyaml>=5.1', - 'importlib-metadata >= 1.0 ; python_version < "3.8"', ], extras_require={ 'bash completion': 'argcomplete', diff --git a/tests/README.md b/tests/README.md index e17b623..5a45af0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,6 +4,7 @@ * Install docker and docker-compose * Install pytest and pytest-docker-compose via pip + * If install fails, have a look at https://github.com/yaml/pyyaml/issues/601#issuecomment-1813963845 * Generate a ssh keypair and place it in tests with `ssh-keygen -t rsa -f tests/id_rsa_tests` * In tests: Copy docker-compose.example.yml to docker-compose.yml and replace the public key * Put something like the following in your ~.ssh/config @@ -16,4 +17,7 @@ StrictHostKeyChecking no -* Run `make test` in the automatix root directory. \ No newline at end of file +* Run `make test` in the automatix root directory. + +Note: Testing remote commands on MacOs with podman seems broken (for me). +Maybe this needs some adjustment or further investigation. diff --git a/tests/test_environment.py b/tests/test_environment.py index cfd9380..8a6ebf9 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -5,8 +5,9 @@ import pytest -from automatix import get_script, collect_vars, CONFIG, cmdClass, SCRIPT_FIELDS +from automatix import get_script, collect_vars, CONFIG, Command, SCRIPT_FIELDS from automatix.automatix import Automatix +from automatix.logger import init_logger SELFDIR = dirname(abspath(__file__)) @@ -32,7 +33,7 @@ script=script, variables=variables, config=CONFIG, - cmd_class=cmdClass, + cmd_class=Command, script_fields=SCRIPT_FIELDS, cmd_args=default_args, ) @@ -47,6 +48,8 @@ environment = testauto.env +init_logger(name=CONFIG['logger'], debug=True) + def run_command_and_check(cmd): subprocess.run(cmd, shell=True).check_returncode() @@ -55,7 +58,7 @@ def run_command_and_check(cmd): @pytest.fixture(scope='function') def ssh_up(function_scoped_container_getter): max_retries = 20 - for i in range(max_retries): + for _ in range(max_retries): sleep(1) try: run_command_and_check(