Merge pull request #21 from gregcorbett/check_db_dump_recent_rework_r…

…ework

Refactor to ignore a failure for the grace period

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+      # For GitHub Actions, "/" checks for workflow files in .github/workflows.
+    directory: "/"
+    schedule:
+      # By default, this is on Monday.
+      interval: "weekly"

.github/workflows/unit-test.yml

@@ -0,0 +1,43 @@
+name: Run Unit Tests
+
+on: [push, pull_request]
+
+jobs:
+  unit-test:
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }} test
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.6 is the version readily available on EL7. Requires ubuntu-20.04.
+        # 3.9 would seem to be the highest python version readily available
+        # on EL8.
+        # Also test against the very latest 3 series available.
+        python-version: [3.6, 3.9, 3.x]
+        os: [ubuntu-20.04, ubuntu-latest]
+        exclude:
+        - os: ubuntu-latest
+          python-version: 3.6
+        - os: ubuntu-20.04
+          python-version: 3.9
+        - os: ubuntu-20.04
+          python-version: 3.x
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up dependencies
+      run: pip install -r requirements-test.txt
+
+    - name: Run unit tests
+      run: coverage run --branch --source=check -m unittest discover --buffer
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3.1.4

README.md

@@ -51,6 +51,10 @@ This repo contains the service and cron scripts used to run a failover gocdb ins
   check
       |_ check_db_dump_recent.py  # Python script to checks the failover
                                   # process is happening
+
+  test
+      |_ test_check_db_dump_recent.py  # Unit tests for the
+                                       # check/check_db_dump_recent.py script
 ```
 
 ## /root/autoEngageFailover/

check/check_db_dump_recent.py

@@ -4,7 +4,7 @@
 This file contains a script to check the GOCDB failover process is happening.
 
 Specifically, it checks the log file passed as the first argument for evidence
-the process has succeeded / failed recently.
+the process has succeeded recently.
 """
 from datetime import datetime, timedelta
 import sys
@@ -23,6 +23,9 @@ def __init__(self, grace_period):
         # successful run was this long ago.
         self.GRACE_PERIOD = grace_period
 
+        # The string the database update script outputs on a successful restore.
+        self.OK_STRING = "completed ok"
+
     def run(self, log_file_path):
         # Wrap everything in a try...except block so we can return
         # RETURN_CODE_UNKNOWN on a unexpected failure.
@@ -31,9 +34,8 @@ def run(self, log_file_path):
             # Use a inner try...except block to handle the more expected error
             # of "Couldn't open/read the provided file" differently.
             try:
-                with open(log_file_path, 'r') as log_file_path:
-                    lines = log_file_path.read().splitlines()
-                    last_line = lines[-1]
+                with open(log_file_path, 'r') as log_file:
+                    line_list = log_file.read().splitlines()
             except IOError:
                 print(
                     "An error occured trying to open/read {0}".format(
@@ -43,17 +45,30 @@ def run(self, log_file_path):
 
                 return RETURN_CODE_CRITICAL
 
-            # Convert the last line of the log to a timestamp.
-            # Use a inner try...except block to handle the somewhat more
-            # expected error of "the failover process ran, but something
-            # went wrong" differently.
-            try:
-                last_success = datetime.strptime(
-                    last_line,
-                    "%Y-%m-%dT%H:%M:%S%z",
+            # Assume the failover process has never run, then attempt to
+            # disprove that by looping through the logs.
+            last_success = None
+            for line in reversed(line_list):
+                # If OK_STRING is in the line we are looking at, we need to extract
+                # the timestamp from that line to determine when the failover
+                # process last succeeded.
+                if self.OK_STRING in line:
+                    last_success_timestamp = line.split(" ")[0]
+                    last_success = datetime.strptime(
+                        last_success_timestamp,
+                        "%Y-%m-%dT%H:%M:%S+0000",
+                    )
+                    # We only want the most recent success, so once we have
+                    # found it, break out of this loop
+                    break
+
+            if last_success is None:
+                print(
+                    "The failover process has never succeeded, "
+                    "according to %s." % (
+                        log_file_path
+                    )
                 )
-            except ValueError:
-                print("An error occured: {0}".format(last_line))
                 return RETURN_CODE_CRITICAL
 
             print("The failover process last succeeded at %s" % last_success)

importDBdmpFile/1_runDbUpdate.sh

@@ -170,7 +170,8 @@ mv $dmpTarget $dmpTargetFN.dmp
 # and move back up a dir as before
 cd ..
 
-# Do not remove following line - another process relies on this exact string
-# being the last line in the log file
+# Do not remove following lines - other processes rely on these exact
+# strings being the last line in the relevant log file.
+echo "$(date --iso-8601='seconds') INFO: completed ok"
 logger "completed ok"
 # Do not add any further "logger" statements after above line

requirements-test.txt

@@ -0,0 +1,2 @@
+# Requirements for the unit and coverage tests only.
+codecov