add guard container

casm/cmds/__init__.py

@@ -10,11 +10,12 @@
 from xarg import commands
 from xarg import run_command
 
-from ..utils import __url_home__
 from ..utils import __project__
+from ..utils import __url_home__
 from ..utils import __version__
 from ..utils import assemble_file
 from .modify import add_cmd_modify
+from .podman import add_cmd_guard
 from .podman import add_cmd_system
 from .podman import add_cmd_systemd
 from .podman_compose import add_cmd_down
@@ -48,7 +49,8 @@ def add_cmd(_arg: argp):
 @run_command(add_cmd, add_cmd_pull, add_cmd_up, add_cmd_down,
              add_cmd_start, add_cmd_stop, add_cmd_restart,
              add_cmd_pause, add_cmd_unpause, add_cmd_exec, add_cmd_logs,
-             add_cmd_system, add_cmd_systemd, add_cmd_services, add_cmd_modify)
+             add_cmd_system, add_cmd_systemd, add_cmd_guard,
+             add_cmd_services, add_cmd_modify)
 def run_cmd(cmds: commands) -> int:
     instance: str = DEF_INSTANCE
     if isinstance(cmds.args.instance, list):

casm/cmds/podman/__init__.py

@@ -10,16 +10,18 @@
 
 from ...utils import __url_home__
 from ...utils import __version__
+from .container import add_cmd_container
+from .guard import add_cmd_guard  # noqa:F401
 from .system import add_cmd_system
-from .systemd import add_cmd_systemd
+from .systemd import add_cmd_systemd  # noqa:F401
 
 
 @add_command("cman")
 def add_cmd(_arg: argp):
     pass
 
 
-@run_command(add_cmd, add_cmd_system)
+@run_command(add_cmd, add_cmd_container, add_cmd_system)
 def run_cmd(cmds: commands) -> int:
     return 0
 

casm/cmds/podman/container.py

@@ -0,0 +1,41 @@
+# coding:utf-8
+
+from typing import Iterable
+
+from xarg import add_command
+from xarg import argp
+from xarg import commands
+from xarg import run_command
+
+from ...utils import podman_container
+
+
+def add_pos_containers(_arg: argp):
+    _arg.add_argument(dest="containers", type=str, nargs="*",
+                      metavar="CONTAINER", action="extend",
+                      choices=list(podman_container.list()) + [[]],
+                      help="Specify containers, default ALL")
+
+
+@add_command("guard", help="Guard podman container")
+def add_cmd_container_guard(_arg: argp):
+    add_pos_containers(_arg)
+
+
+@run_command(add_cmd_container_guard)
+def run_cmd_container_guard(cmds: commands) -> int:
+    containers: Iterable[str] = cmds.args.containers or podman_container.list()
+    for container_name in containers:
+        cmds.logger.info(f"guard container {container_name}")
+        podman_container(container_name).guard()
+    return 0
+
+
+@add_command("container", help="Manage podman containers")
+def add_cmd_container(_arg: argp):
+    pass
+
+
+@run_command(add_cmd_container, add_cmd_container_guard)
+def run_cmd_container(cmds: commands) -> int:
+    return 0

casm/cmds/podman/guard.py

@@ -0,0 +1,42 @@
+# coding:utf-8
+
+from typing import List
+
+from xarg import add_command
+from xarg import argp
+from xarg import commands
+from xarg import run_command
+
+from ...utils import assemble_file
+from ...utils import podman_container
+from ..service import add_pos_services
+
+
+@add_command("generate", help="Generate container guard crontab")
+def add_cmd_guard_generate(_arg: argp):
+    add_pos_services(_arg)
+
+
+@run_command(add_cmd_guard_generate)
+def run_cmd_guard_generate(cmds: commands) -> int:
+    assemble: assemble_file = cmds.args.assemble_file
+    assert isinstance(assemble, assemble_file)
+    services: List[str] = cmds.args.services
+    for service in assemble.template.services:
+        cmds.logger.debug(f"{service.title}: {service.container_name}")
+        if len(services) > 0 and service.title not in services:
+            continue
+        container_name = assemble.safe_substitute(service.container_name)
+        cmds.logger.info(f"generate container {container_name} guard")
+        podman_container(container_name).generate_guard_task()
+    return 0
+
+
+@add_command("guard", help="Manage containers guard")
+def add_cmd_guard(_arg: argp):
+    pass
+
+
+@run_command(add_cmd_guard, add_cmd_guard_generate)
+def run_cmd_guard(cmds: commands) -> int:
+    return 0

casm/cmds/podman/systemd.py

@@ -12,7 +12,7 @@
 from ..service import add_pos_services
 
 
-@add_command("enable", help="Enable systemd for containers")
+@add_command("enable", help="Enable systemd unit for containers")
 def add_cmd_enable(_arg: argp):
     _arg.add_argument("--restart-policy", dest="restart_policy",
                       type=str, nargs=1, metavar="STR", default=["on-failure"],
@@ -37,7 +37,7 @@ def run_cmd_enable(cmds: commands) -> int:
     return 0
 
 
-@add_command("disable", help="Disable systemd for containers")
+@add_command("disable", help="Disable systemd unit for containers")
 def add_cmd_disable(_arg: argp):
     add_pos_services(_arg)
 

casm/utils/podman.py

@@ -1,12 +1,14 @@
 # coding:utf-8
 
+import getpass
 import os
 import shutil
 from typing import Any
 from typing import Dict
 from typing import Iterable
 from typing import List
 from typing import Optional
+from typing import Tuple
 
 from podman import PodmanClient
 from podman.domain.containers import Container
@@ -95,7 +97,7 @@ def __init__(self, container: Container):
         self.__container: Container = container
         self.__info: Dict[str, Any] = container.inspect()
         self.__state: Optional[podman_container_inspect.state_struct] = None
-        self.__host_config: Optional[podman_container_inspect.host_config_struct] = None  # noqa: E501
+        self.__host_config: Optional[podman_container_inspect.host_config_struct] = None  # noqa:E501
 
     @property
     def container(self) -> Container:
@@ -130,7 +132,7 @@ def State(self) -> state_struct:
     @property
     def HostConfig(self) -> host_config_struct:
         if self.__host_config is None:
-            self.__host_config = self.host_config_struct(self.info["HostConfig"])  # noqa: E501
+            self.__host_config = self.host_config_struct(self.info["HostConfig"])  # noqa:E501
         return self.__host_config
 
 
@@ -168,9 +170,44 @@ def inspect(self) -> podman_container_inspect:
     def service_unit(self) -> str:
         return f"container-{self.container_name}.service"
 
+    @property
+    def healthy(self) -> bool:
+        inspect = self.inspect()
+
+        def __restarting() -> bool:
+            return inspect.State.Restarting
+
+        def __running() -> bool:
+            return inspect.State.Status == "running" and\
+                inspect.State.Running and\
+                not inspect.State.Paused and\
+                not inspect.State.OOMKilled and\
+                not inspect.State.Dead
+
+        def __healthy() -> bool:
+            health = inspect.State.Health
+            return health is None or health.Status == "healthy"
+
+        return __restarting() or __running() and __healthy()
+
+    def stop(self) -> int:
+        return os.system(f"podman container stop {self.container_name}")
+
+    def start(self) -> int:
+        return os.system(f"podman container start {self.container_name}")
+
+    def restart(self) -> int:
+        return os.system(f"podman container restart {self.container_name}")
+
     def stop_service(self) -> int:
         return os.system(f"systemctl stop {self.service_unit}")
 
+    def start_service(self) -> int:
+        return os.system(f"systemctl start {self.service_unit}")
+
+    def restart_service(self) -> int:
+        return os.system(f"systemctl restart {self.service_unit}")
+
     def generate_service(self, restart_policy: str = "on-failure",
                          restart_sect: Optional[int] = None,
                          start_timeout: Optional[int] = None,
@@ -187,8 +224,8 @@ def generate_service(self, restart_policy: str = "on-failure",
             raise FileNotFoundError("podman command not found")
 
         container_inspect: podman_container_inspect = self.inspect()
-        mounts: List[Optional[str]] = [mountpoint(bind.split(":")[0]) for bind in  # noqa: E501
-                                       container_inspect.HostConfig.Binds if bind.startswith("/")]  # noqa: E501
+        mounts: List[Optional[str]] = [mountpoint(bind.split(":")[0]) for bind in  # noqa:E501
+                                       container_inspect.HostConfig.Binds if bind.startswith("/")]  # noqa:E501
         mountpoints: List[str] = ["/run/containers/storage"]
         mountpoints.extend([m for m in mounts if isinstance(m, str)])
         content: str = f"""
@@ -237,6 +274,26 @@ def disable_service(self) -> int:
         systemd_service.delete_unit(unit=self.service_unit)
         return 0
 
+    def generate_guard_task(self, interval: int = 3) -> int:
+        container_name: str = self.container_name
+        with open(f"/etc/cron.d/guard-container-{container_name}.sh", "w") as hdl:  # noqa:E501
+            username: str = getpass.getuser()
+            hdl.write(f"PATH={os.environ['PATH']}\n")
+            hdl.write(f"*/{interval} * * * * {username} cman container guard {container_name}\n")  # noqa:E501
+        return 0
+
+    def guard(self) -> bool:
+        def __restart() -> bool:
+            return self.restart_service() == 0 or self.restart() == 0
+        return __restart() if not self.healthy else True
+
+    @classmethod
+    def list(cls) -> Tuple[str, ...]:
+        client: PodmanClient = PodmanClient(base_url=cls.BASEURL)
+        containers: List[Container] = client.containers.list()
+        container_names = [container.name for container in containers]
+        return tuple(name for name in container_names if isinstance(name, str))
+
 
 class podman_cmd:
     '''Execute podman command