Merge pull request #51 from seibert-media/reload_on_try

Reload on retry

README.md

@@ -39,7 +39,7 @@ Please use the interactive mode and doublecheck commands before
 
 # INSTALLATION
 
-Automatix requires Python ≥ 3.6.
+Automatix requires Python ≥ 3.8.
 
 ```
 pip install automatix

automatix/automatix.py

@@ -1,9 +1,11 @@
 import logging
 from argparse import Namespace
 from collections import OrderedDict
+from functools import cached_property
 from typing import List
 
-from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS
+from .command import Command, AbortException, SkipBatchItemException, PERSISTENT_VARS, ReloadFromFile
+from .config import get_script
 from .environment import PipelineEnvironment
 
 
@@ -31,9 +33,17 @@ def __init__(
             logger=logging.getLogger(config['logger']),
         )
 
+    @cached_property
+    def command_lists(self) -> dict:
+        return {
+            'always': self.build_command_list(pipeline='always'),
+            'main': self.build_command_list(pipeline='pipeline'),
+            'cleanup': self.build_command_list(pipeline='cleanup'),
+        }
+
     def build_command_list(self, pipeline: str) -> List[Command]:
         command_list = []
-        for index, cmd in enumerate(self.script[pipeline]):
+        for index, cmd in enumerate(self.script.get(pipeline, [])):
             new_cmd = self.cmd_class(
                 cmd=cmd,
                 index=index,
@@ -45,6 +55,10 @@ def build_command_list(self, pipeline: str) -> List[Command]:
                 self.env.vars[new_cmd.assignment_var] = f'{{{new_cmd.assignment_var}}}'
         return command_list
 
+    def reload_script(self):
+        self.script = get_script(args=self.env.cmd_args)
+        del self.command_lists  # Clear cache
+
     def print_main_data(self):
         self.env.LOG.info('\n\n')
         self.env.LOG.info(f' ------ Overview ------')
@@ -58,32 +72,43 @@ def print_command_line_steps(self, command_list: List[Command]):
         for cmd in command_list:
             self.env.LOG.info(f"({cmd.index}) [{cmd.orig_key}]: {cmd.get_resolved_value()}")
 
-    def execute_main_pipeline(self, command_list: List[Command]):
-        self.env.LOG.info('\n------------------------------')
-        self.env.LOG.info(' --- Start MAIN pipeline ---')
+    def _execute_command_list(self, name: str, start_index: int, treat_as_main: bool):
+        try:
+            steps = self.script.get('steps')
+            for cmd in self.command_lists[name][start_index:]:
+                if treat_as_main:
+                    if steps and (self.script['exclude'] == (cmd.index in steps)):
+                        # Case 1: exclude is True  and index is in steps => skip
+                        # Case 2: exclude is False and index is in steps => execute
+                        self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip')
+                        continue
+                    cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force)
+                else:
+                    cmd.execute()
+        except ReloadFromFile as exc:
+            self.env.LOG.info(f'\n({exc.index}) Reload script from file and retry.')
+            self.reload_script()
+            self._execute_command_list(name=name, start_index=exc.index, treat_as_main=treat_as_main)
+
+    def execute_pipeline(self, name: str):
+        if not self.command_lists[name]:
+            return
+
+        if name == 'main':
+            treat_as_main = True
+            start_index = self.env.cmd_args.jump_to
+        else:
+            treat_as_main = False
+            start_index = 0
 
-        steps = self.script.get('steps')
+        self.env.LOG.info('\n------------------------------')
+        self.env.LOG.info(f' --- Start {name.upper()} pipeline ---')
 
-        for cmd in command_list[self.env.cmd_args.jump_to:]:
-            if steps and (self.script['exclude'] == (cmd.index in steps)):
-                # Case 1: exclude is True  and index is in steps => skip
-                # Case 2: exclude is False and index is in steps => execute
-                self.env.LOG.notice(f'\n({cmd.index}) Not selected for execution: skip')
-                continue
-            cmd.execute(interactive=self.env.cmd_args.interactive, force=self.env.cmd_args.force)
+        self._execute_command_list(name=name, start_index=start_index, treat_as_main=treat_as_main)
 
-        self.env.LOG.info('\n --- End MAIN pipeline ---')
+        self.env.LOG.info(f'\n --- End {name.upper()} pipeline ---')
         self.env.LOG.info('------------------------------\n')
 
-    def execute_extra_pipeline(self, pipeline: str):
-        if self.script.get(pipeline):
-            self.env.LOG.info('\n------------------------------')
-            self.env.LOG.info(f' --- Start {pipeline.upper()} pipeline ---')
-            for cmd in self.build_command_list(pipeline=pipeline):
-                cmd.execute()
-            self.env.LOG.info(f'\n --- End {pipeline.upper()} pipeline ---')
-            self.env.LOG.info('------------------------------\n')
-
     def run(self):
         self.env.LOG.info('\n\n')
         self.env.LOG.info('//////////////////////////////////////////////////////////////////////')
@@ -92,24 +117,22 @@ def run(self):
 
         PERSISTENT_VARS.clear()
 
-        command_list = self.build_command_list(pipeline='pipeline')
-
-        self.execute_extra_pipeline(pipeline='always')
+        self.execute_pipeline(name='always')
 
         self.print_main_data()
-        self.print_command_line_steps(command_list)
+        self.print_command_line_steps(command_list=self.command_lists['main'])
         if self.env.cmd_args.print_overview:
             exit()
 
         try:
-            self.execute_main_pipeline(command_list=command_list)
+            self.execute_pipeline(name='main')
         except (AbortException, SkipBatchItemException):
             self.env.LOG.debug('Abort requested. Cleaning up.')
-            self.execute_extra_pipeline(pipeline='cleanup')
+            self.execute_pipeline(name='cleanup')
             self.env.LOG.debug('Clean up done. Exiting.')
             raise
 
-        self.execute_extra_pipeline(pipeline='cleanup')
+        self.execute_pipeline(name='cleanup')
 
         self.env.LOG.info('---------------------------------------------------------------')
         self.env.LOG.info('Automatix finished: Congratulations and have a N.I.C.E. day :-)')

automatix/command.py

@@ -24,6 +24,7 @@ def __setattr__(self, key: str, value):
 POSSIBLE_ANSWERS = {
     'p': 'proceed (default)',
     'r': 'retry',
+    'R': 'reload from file and retry command (same index)',
     's': 'skip',
     'a': 'abort',
     'c': 'abort & continue to next (CSV processing)',
@@ -99,8 +100,8 @@ def execute(self, interactive: bool = False, force: bool = False):
         if self.get_type() == 'manual' or interactive:
             self.env.LOG.debug('Ask for user interaction.')
 
-            answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'a'])
-            # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass
+            answer = self._ask_user(question='[MS] Proceed?', allowed_options=['p', 's', 'R', 'a'])
+            # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass
             if answer == 's':
                 return
 
@@ -122,8 +123,8 @@ def execute(self, interactive: bool = False, force: bool = False):
             if force:
                 return
 
-            err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'a'])
-            # answers 'a' and 'c' are handled by _ask_user, 'p' means just pass
+            err_answer = self._ask_user(question='[CF] What should I do?', allowed_options=['p', 'r', 'R', 'a'])
+            # answers 'a', 'c' and 'R' are handled by _ask_user, 'p' means just pass
             if err_answer == 'r':
                 return self.execute(interactive)
 
@@ -142,19 +143,21 @@ def _ask_user(self, question: str, allowed_options: list) -> str:
         if self.env.batch_mode:
             allowed_options.append('c')
 
-        options = ', '.join([f'{k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options])
+        options = '\n'.join([f' {k}: {POSSIBLE_ANSWERS[k]}' for k in allowed_options])
 
         answer = None
         while answer not in allowed_options:
             if answer is not None:
                 self.env.LOG.info('Invalid input. Try again.')
 
-            answer = input(f'{question} ({options})\a')
+            answer = input(f'{question}\n{options}\nYour answer: \a')
 
             if answer == '':  # default
                 answer = 'p'
             if answer == 'a':
                 raise AbortException(1)
+            if answer == 'R':
+                raise ReloadFromFile(index=self.index)
             if self.env.batch_mode and answer == 'c':
                 raise SkipBatchItemException()
 
@@ -204,6 +207,8 @@ def _python_action(self) -> int:
                     'Seems you are trying to use bundlewrap functions without having bundlewrap support enabled.'
                     ' Please check your configuration.')
                 return 1
+            if isinstance(exc.__context__, ReloadFromFile):
+                exc.__suppress_context__ = True
 
             self.env.LOG.exception('Unknown error occured:')
             return 1
@@ -344,3 +349,11 @@ class SkipBatchItemException(Exception):
 
 class UnknownCommandException(Exception):
     pass
+
+
+class ReloadFromFile(Exception):
+    def __init__(self, index: int):
+        self.index = index
+
+    def __int__(self):
+        return self.index

automatix/command_test.py

@@ -9,7 +9,7 @@
 
 
 def test__execute_remote_cmd(ssh_up):
-    cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, env=environment)
+    cmd = Command(cmd={'remote@testsystem': 'touch /test_remote_cmd'}, index=2, pipeline='pipeline', env=environment)
     cmd.execute()
     try:
         run_command_and_check('ssh docker-test ls /test_remote_cmd >/dev/null')
@@ -23,7 +23,7 @@ def test__execute_local_cmd(capfd):
     # empty captured stdin and stderr
     _ = capfd.readouterr()
 
-    cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, env=environment)
+    cmd = Command(cmd={'local': f'echo {test_string}'}, index=2, pipeline='pipeline', env=environment)
     cmd.execute()
 
     out, err = capfd.readouterr()
@@ -44,7 +44,7 @@ def test__execute_local_with_condition(capfd):
         # empty captured stdin and stderr
         _ = capfd.readouterr()
 
-        cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, index=2, env=environment)
+        cmd = Command(cmd={f'{condition_var}?local': 'pwd'}, pipeline='pipeline', index=2, env=environment)
         cmd.execute()
 
         out, err = capfd.readouterr()
@@ -59,10 +59,10 @@ def test__execute_python_cmd():
 PERSISTENT_VARS.update(locals())
 """
 
-    cmd = Command(cmd={'python': test_cmd}, index=2, env=environment)
+    cmd = Command(cmd={'python': test_cmd}, index=2, pipeline='pipeline', env=environment)
     cmd.execute()
 
-    cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, env=environment)
+    cmd = Command(cmd={'python': 'print(uuid4())'}, index=2, pipeline='pipeline', env=environment)
     cmd.execute()
 
 

automatix/config.py

@@ -2,8 +2,8 @@
 import logging
 import os
 import re
-import sys
 from collections import OrderedDict
+from importlib import metadata
 from time import sleep
 
 import yaml
@@ -16,11 +16,6 @@ def read_yaml(yamlfile: str) -> dict:
         return yaml.load(file.read(), Loader=yaml.SafeLoader)
 
 
-if sys.version_info >= (3, 8):
-    from importlib import metadata
-else:
-    import importlib_metadata as metadata
-
 try:
     from argcomplete import autocomplete
     from .bash_completion import ScriptFileCompleter

example.automatix.cfg.yaml

@@ -1,42 +1,32 @@
 # Path to scripts directory
-
 script_dir: ~/automatix_script_files
 
 # Global constants for use in pipeline scripts
-
 constants:
   apt_update: 'apt-get -qy update'
   apt_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends upgrade'
   apt_full_upgrade: 'DEBIAN_FRONTEND=noninteractive apt-get -qy -o Dpkg::Options::=--force-confold --no-install-recommends full-upgrade'
 
 # Encoding
-
 encoding: utf-8
 
 # Path to shell imports
-
 import_path: '.'
 
 # SSH Command used for remote connections
-
 ssh_cmd: 'ssh {hostname} sudo '
 
 # Temporary directory on remote machines for shell imports
-
 remote_tmp_dir: 'automatix_tmp'
 
 # Logger
-
 logger: mylogger
 
 # Logging library
-
 logging_lib: mylib.logging
 
 # Bundlewrap support, bundlewrap has to be installed (default: false)
-
 bundlewrap: true
 
 # Teamvault / Secret support, bundlewrap-teamvault has to be installed (default: false)
-
 teamvault: true

pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
 addopts = --durations=0 --docker-compose=tests/docker-compose.yml
-testpaths = automatix
+testpaths = automatix

setup.py

@@ -14,10 +14,10 @@
     author='Johannes Paul, //SEIBERT/MEDIA GmbH',
     author_email='jpaul@seibert-media.net',
     license='MIT',
-    python_requires='>=3.6',
+    python_requires='>=3.8',
     install_requires=[
+        'cython<3.0.0',
         'pyyaml>=5.1',
-        'importlib-metadata >= 1.0 ; python_version < "3.8"',
     ],
     extras_require={
         'bash completion': 'argcomplete',

tests/README.md

@@ -4,6 +4,7 @@
 
 * Install docker and docker-compose
 * Install pytest and pytest-docker-compose via pip
+  * If install fails, have a look at https://github.com/yaml/pyyaml/issues/601#issuecomment-1813963845
 * Generate a ssh keypair and place it in tests with `ssh-keygen -t rsa -f tests/id_rsa_tests`
 * In tests: Copy docker-compose.example.yml to docker-compose.yml and replace the public key
 * Put something like the following in your ~.ssh/config
@@ -16,4 +17,7 @@
     StrictHostKeyChecking no
 
 
-* Run `make test` in the automatix root directory.
+* Run `make test` in the automatix root directory.
+
+Note: Testing remote commands on MacOs with podman seems broken (for me).
+Maybe this needs some adjustment or further investigation.