API Updates (#311)

* Use 24hr timeformat instead of 12hr

* Add last_backup_seconds_taken to api

* Update pydantic

* Add last_backup_seconds_taken to API docs

* No longer fail backup upon missing nested destination directory

* Add `source-dir-required` label

* Remove `version: 3` from docker compose docs

* Update dependency fastapi to v0.114.0

* Bump CURL version

* Docs update

.devcontainer/scripts/nb.sh

@@ -8,13 +8,13 @@ show_help() {
     echo "  build-run      - Build and run Nautical container"
     echo ""
     echo "  unit-test      - Run Unit tests locally using mounts"
-    echo "  unit-test-full - Build and run Unit tests locally"
-    echo "  build-test     - Build and run Nautical Testing container"
+    echo "  integration    - Build and run integration tests locally"
+    # echo "  build-test     - Build and run Nautical Testing container"
     echo "  test           - Run already built test Nautical container"
-    echo "  build-test-run - Build and run Nautical Testing container"
+    # echo "  build-test-run - Build and run Nautical Testing container"
     echo ""
-    echo "  dev            - Run Nautical Development container"
-    echo "  build-dev      - Build and run Nautical Development container"
+    # echo "  dev            - Run Nautical Development container"
+    # echo "  build-dev      - Build and run Nautical Development container"
     echo "  api            - Run the Python API locally"
     echo "  pytest         - Pytest locally and capture coverage"
     echo "  format         - Format all python code with black"
@@ -56,21 +56,21 @@ execute_command() {
         cd $APP_HOME/tests
         docker compose run nautical-backup-test4 --exit-code-from nautical-backup-test4
         ;;
-    dev)
-        cd $APP_HOME/tests
-        docker compose run nautical-backup-test5 --exit-code-from nautical-backup-test5
-        ;;
-    build-test)
-        clear
-        cecho CYAN "Building Test Nautical container..."
-        cd $APP_HOME
-        docker build -t minituff/nautical-test --no-cache --progress=plain --build-arg='NAUTICAL_VERSION=testing' --build-arg='TEST_MODE=0' .
-        ;;
-    build-dev)
-        cd $APP_HOME
-        nb build-test
-        ;;
-    test)
+    # dev)
+    #     cd $APP_HOME/tests
+    #     docker compose run nautical-backup-test5 --exit-code-from nautical-backup-test5
+    #     ;;
+    # build-test)
+    #     clear
+    #     cecho CYAN "Building Test Nautical container..."
+    #     cd $APP_HOME
+    #     docker build -t minituff/nautical-test --no-cache --progress=plain --build-arg='NAUTICAL_VERSION=testing' --build-arg='TEST_MODE=0' .
+    #     ;;
+    # build-dev)
+    #     cd $APP_HOME
+    #     nb build-test
+    #     ;;
+    integration)
         cecho CYAN "Running Nautical integration tests..."
         cd $APP_HOME
 
@@ -108,11 +108,11 @@ execute_command() {
         rm -rf source destination config
 
         ;;
-    build-test-run)
-        nb build-test
-        clear
-        nb test
-        ;;
+    # build-test-run)
+    #     nb build-test
+    #     clear
+    #     nb test
+    #     ;;
     api)
         cd $APP_HOME
         cecho CYAN "Running Nautical API locally..."

Dockerfile

@@ -59,9 +59,9 @@ ENV DOS2UNIX_VERSION="7.4.4-r1"
 # renovate: datasource=repology depName=alpine_3_18/jq versioning=loose
 ENV JQ_VERSION="1.6-r4"
 # renovate: datasource=repology depName=alpine_3_18/curl versioning=loose
-ENV CURL_VERSION="8.9.0-r0"
+ENV CURL_VERSION="8.9.1-r0"
 # renovate: datasource=repology depName=alpine_3_18/python3 versioning=loose
-ENV PYTHON_VERSION="3.11.8-r0"
+ENV PYTHON_VERSION="3.11.8-r1"
 # renovate: datasource=repology depName=alpine_3_18/py3-pip versioning=loose
 ENV PIP_VERSION="23.1.2-r0"
 # renovate: datasource=repology depName=alpine_3_18/ruby-full versioning=loose

app/api/nautical_router.py

@@ -36,6 +36,7 @@ def dashboard(username: Annotated[str, Depends(authorize)]) -> JSONResponse:
         "next_run": next_crons.get("1", [None, None])[1] if next_crons else None,
         "last_cron": db.get("last_cron", "None"),
         "number_of_containers": db.get("number_of_containers", 0),
+        "last_backup_seconds_taken": db.get("last_backup_seconds_taken", 0),
         "completed": db.get("containers_completed", 0),
         "skipped": db.get("containers_skipped", 0),
         "errors": db.get("errors", 0),

app/backup.py

@@ -47,14 +47,14 @@ def __init__(self, docker_client: docker.DockerClient):
         if self.env.REPORT_FILE == True and self.env.REPORT_FILE_ON_BACKUP_ONLY == False:
             self.logger._create_new_report_file()
 
-        self.verify_source_location(self.env.SOURCE_LOCATION)
-        self.verify_destination_location(self.env.DEST_LOCATION)
+        self.verify_nautcical_mounted_source_location(self.env.SOURCE_LOCATION)
+        self.verify_nautiucal_mounted_destination_location(self.env.DEST_LOCATION, create_if_not_exists=False)
 
     def log_this(self, log_message, log_priority="INFO", log_type=LogType.DEFAULT) -> None:
         """Wrapper for log this"""
         return self.logger.log_this(log_message, log_priority, log_type)
 
-    def verify_source_location(self, src_dir: str):
+    def verify_nautcical_mounted_source_location(self, src_dir: str):
         self.log_this(f"Verifying source directory '{src_dir}'...", "DEBUG", LogType.INIT)
         if not os.path.isdir(src_dir):
             self.log_this(f"Source directory '{src_dir}' does not exist.", "ERROR", LogType.INIT)
@@ -65,8 +65,12 @@ def verify_source_location(self, src_dir: str):
 
         self.log_this(f"Source directory '{src_dir}' READ access verified", "TRACE", LogType.INIT)
 
-    def verify_destination_location(self, dest_dir: str):
-        self.log_this(f"Verifying destination directory '{dest_dir}'...", "DEBUG", LogType.INIT)
+    def verify_nautiucal_mounted_destination_location(self, dest_dir: Union[str, Path], create_if_not_exists=True):
+        self.log_this(f"Verifying Nautical destination directory '{dest_dir}'...", "DEBUG", LogType.INIT)
+
+        if not os.path.exists(dest_dir) and create_if_not_exists:
+            os.makedirs(dest_dir, exist_ok=True)
+
         if not os.path.isdir(dest_dir):
             self.log_this(
                 f"Destination directory '{dest_dir}' does not exist. Please mount it to /app/destination",
@@ -83,6 +87,28 @@ def verify_destination_location(self, dest_dir: str):
 
         self.log_this(f"Destination directory '{dest_dir}' READ/WRITE access verified", "TRACE", LogType.INIT)
 
+    def verify_destination_location(self, dest_dir: Union[str, Path], create_if_not_exists=True) -> bool:
+        """Verify the destination location (for containers) exists and is writable"""
+        self.log_this(f"Verifying destination directory '{dest_dir}'...", "DEBUG", LogType.INIT)
+
+        if not os.path.exists(dest_dir) and create_if_not_exists:
+            os.makedirs(dest_dir, exist_ok=True)
+
+        if not os.path.isdir(dest_dir):
+            self.log_this(
+                f"Destination directory '{dest_dir}' does not exist. Please mount it to /app/destination", "ERROR"
+            )
+            return False
+        elif not os.access(dest_dir, os.R_OK):
+            self.log_this(f"No read access to destination directory '{dest_dir}'.", "ERROR")
+            return False
+        elif not os.access(dest_dir, os.W_OK):
+            self.log_this(f"No write access to destination directory '{dest_dir}'.", "ERROR")
+            raise PermissionError(f"No write access to destination directory '{dest_dir}'")
+
+        self.log_this(f"Destination directory '{dest_dir}' READ/WRITE access verified", "TRACE")
+        return True
+
     def _should_skip_container(self, c: Container) -> bool:
         """Use logic to determine if a container should be skipped by nautical completely"""
 
@@ -465,8 +491,8 @@ def _backup_additional_folders_standalone(self, when: BeforeOrAfter, base_dest_d
             return
 
         base_src_dir = Path(self.env.SOURCE_LOCATION)
-        # base_dest_dir = Path(self.env.DEST_LOCATION)
 
+        self.verify_destination_location(base_dest_dir)
         if not os.path.exists(base_dest_dir):
             self.log_this(
                 f"Destination directory '{base_dest_dir}' does not exist during {BeforeOrAfter.BEFORE.name}", "ERROR"
@@ -508,6 +534,7 @@ def _backup_additional_folders(self, c: Container, base_dest_dir: Path):
                 if not os.path.exists(dest_dir):
                     os.makedirs(dest_dir, exist_ok=True)
 
+            self.verify_destination_location(dest_dir)
             self.log_this(f"Backing up additional folder '{folder}' for container {c.name}")
             self._run_rsync(c, rsync_args, src_dir, dest_dir)
 
@@ -521,14 +548,21 @@ def _backup_container_folders(self, c: Container, dest_path: Optional[Path] = No
         else:  # Only container given (no secondary dest)
             dest_path = Path(self.env.DEST_LOCATION)
 
-        if not dest_dir.exists():
-            self.log_this(f"Destination directory '{dest_dir}' does not exist", "ERROR")
+        src_dir_required = str(c.labels.get("nautical-backup.source-dir-required", "true")).lower()
+        if src_dir_required == "true":
+            self.verify_destination_location(dest_path)
+            if not dest_dir.exists():
+                self.log_this(f"Destination directory '{dest_dir}' does not exist", "ERROR")
 
         if src_dir.exists():
             self.log_this(f"Backing up {c.name}...", "INFO")
 
             rsync_args = self._get_rsync_args(c)
             self._run_rsync(c, rsync_args, src_dir, dest_dir)
+        elif src_dir_required == "false":
+            # Do nothing. This container is still started and stopped, but there is nothing to backup
+            # Likely this container is part of a group and the source directory is not required
+            pass
         else:
             self.log_this(f"Source directory {src_dir} does not exist. Skipping", "DEBUG")
 
@@ -587,14 +621,26 @@ def _get_rsync_args(self, c: Optional[Container], log=False) -> str:
 
         return f"{default_rsync_args} {custom_rsync_args}"
 
+    def reset_db(self) -> None:
+        """Reset the database values to their defaults"""
+        self.db.put("containers_completed", 0)
+        self.db.put("containers_skipped", 0)
+        self.db.put("last_backup_seconds_taken", 0)
+        self.db.put("last_cron", "None")
+        self.db.put("completed", "0")
+        self.db.put("backup_running", False)
+
     def backup(self):
         if self.env.REPORT_FILE == True:
             self.logger._create_new_report_file()
 
         self.log_this("Starting backup...", "INFO")
 
+        self.reset_db()
         self.db.put("backup_running", True)
-        self.db.put("last_cron", datetime.now().strftime("%m/%d/%y %I:%M"))
+
+        start_time = datetime.now()
+        self.db.put("last_cron", start_time.strftime("%m/%d/%y %H:%M"))
 
         self._run_exec(None, BeforeAfterorDuring.BEFORE, attached_to_container=False)
 
@@ -626,13 +672,15 @@ def backup(self):
 
                 src_dir, src_dir_no_path = self._get_src_dir(c)
                 if not src_dir.exists():
-                    src_dir_required = str(c.labels.get("nautical-backup.source-dir-required-to-stop", "true")).lower()
+                    src_dir_required = str(c.labels.get("nautical-backup.source-dir-required", "true")).lower()
                     if src_dir_required == "false":
-                        self.log_this(f"{c.name} - Source directory '{src_dir}' does, but that's okay", "DEBUG")
-
-                    self.log_this(f"{c.name} - Source directory '{src_dir}' does not exist. Skipping", "DEBUG")
-                    self.containers_skipped.add(c.name)
-                    continue
+                        self.log_this(
+                            f"{c.name} - Source directory '{src_dir}' does not exist, but that's okay", "DEBUG"
+                        )
+                    else:
+                        self.log_this(f"{c.name} - Source directory '{src_dir}' does not exist. Skipping", "DEBUG")
+                        self.containers_skipped.add(c.name)
+                        continue
 
                 stop_result = self._stop_container(c)  # Stop containers
 
@@ -686,12 +734,18 @@ def backup(self):
             self._backup_additional_folders_standalone(BeforeOrAfter.AFTER, dir)
         self._run_exec(None, BeforeAfterorDuring.AFTER, attached_to_container=False)
 
+        end_time = datetime.now()
+        exeuction_time = end_time - start_time
+        duration = datetime.fromtimestamp(exeuction_time.total_seconds())
+
         self.db.put("backup_running", False)
         self.db.put("containers_completed", len(self.containers_completed))
         self.db.put("containers_skipped", len(self.containers_skipped))
+        self.db.put("last_backup_seconds_taken", round(exeuction_time.total_seconds()))
 
         self.log_this("Containers completed: " + self.logger.set_to_string(self.containers_completed), "DEBUG")
         self.log_this("Containers skipped: " + self.logger.set_to_string(self.containers_skipped), "DEBUG")
+        self.log_this(f"Completed in {duration.strftime('%Mm %Ss')}", "INFO")
 
         self.log_this(
             f"Success. {len(self.containers_completed)} containers backed up! {len(self.containers_skipped)} skipped.",

app/db.sh

@@ -46,7 +46,7 @@ add_current_datetime() {
 
     # Format the current date and time
     local datetime_format1=$(date +"%A, %B %d, %Y at %I:%M %p")
-    local datetime_format2=$(date +"%m/%d/%y %I:%M")
+    local datetime_format2=$(date +"%m/%d/%y %H:%M")
 
     # Update or create the key with the formatted date and time
     jq --arg key "$key" --arg datetime1 "$datetime_format1" --arg datetime2 "$datetime_format2" \

docs/advanced/nfs-share.md

@@ -44,7 +44,7 @@ The above example created a local directory of `/mnt/nfs/docker_backups` which i
 Here is how we can use this new mount withing Nautical:
 === "Docker Compose"
     ```yaml  hl_lines="9"
-    ------8<------ "docker-compose-example.yml:0:9"
+    ------8<------ "docker-compose-example.yml::8"
           - /mnt/nfs/docker_backups:/app/destination #(3) <-- NFS Share
 
     ```

docs/arguments.md

@@ -117,7 +117,7 @@ ADDITIONAL_FOLDERS_WHEN=after
     The `additional2` folder already exists within the `/opt/volume-data` so it does not need a mount point.
 
     ```yaml
-    ------8<------ "docker-compose-example-no-tooltips.yml:3:9"
+    ------8<------ "docker-compose-example-no-tooltips.yml:1:7"
           - /opt/volume-data:/app/source #(4)!
           - /mnt/nfs-share/backups:/app/destination
           - /mnt/additional:/app/source/additional #(1)!
@@ -154,7 +154,7 @@ SECONDARY_DEST_DIRS=/path/one,/path/two
         Pay attention to the newly added highlighed lines:
 
         ```yaml hl_lines="10-13"
-        ------8<------ "docker-compose-example-no-tooltips.yml:3:9"
+        ------8<------ "docker-compose-example-no-tooltips.yml:1:7"
               - /opt/volume-data:/app/source
               - /mnt/nfs-share/backups:/app/destination
               - /mnt/nfs-share/destination1:/nautical/destination1 #(1)!

docs/introduction.md

@@ -2,7 +2,7 @@
 Essentially, this is an automated and configurable backup tool built around [rsync](https://en.wikipedia.org/wiki/Rsync). 
 
 ## The Basics
-Nautical runs `Bash` commands on a `CRON` schedule to:
+Nautical runs on a `CRON` schedule to:
 
 1. Stop the container <small>(if configured)</small>
 2. Run the backup via `rsync`
@@ -11,7 +11,6 @@ Nautical runs `Bash` commands on a `CRON` schedule to:
 ⚗️ **Need more control?** There are many more options available via [variables](./arguments.md) and [labels](./labels.md).
 
 
-
 ##  Sample Configuration
 Nautical requires almost no configuration when container volumes are all in a folder matching its `container-name` within the source directory.  <small>Of course, we can use [variables](./arguments.md) and [labels](./labels.md) to override these defaults. </small>
 
@@ -26,7 +25,7 @@ Let's take a look at an example:
 !!! example "Here is how Nautical fits into the *Sample Configuration*"
     === "Docker Compose"
         ```yaml
-        ------8<------ "docker-compose-example.yml:3:9"
+        ------8<------ "docker-compose-example.yml:1:7"
               - /opt/docker-volumes:/app/source #(2)!
               - /mnt/nfs-share/backups:/app/destination #(3)!
         ```