diff --git a/README.md b/README.md index d9332ab..7c87887 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ --- title: Backup Container -description: A simple containerized backup solution for backing up one or more postgres or mongo databases to a secondary location. +description: A simple containerized backup solution for backing up one or more supported databases to a secondary location. author: WadeBarnes resourceType: Components personas: @@ -12,19 +12,25 @@ labels: - backups - postgres - mongo + - mssql - database --- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) # Backup Container -[Backup Container](https://github.com/BCDevOps/backup-container) is a simple containerized backup solution for backing up one or more postgres or mongo databases to a secondary location. _Code and documentation was originally pulled from the [HETS Project](https://github.com/bcgov/hets)_ +[Backup Container](https://github.com/BCDevOps/backup-container) is a simple containerized backup solution for backing up one or more supported databases to a secondary location. _Code and documentation was originally pulled from the [HETS Project](https://github.com/bcgov/hets)_ + +# Supported Databases +MongoDB +MSSQL - Currently MSSQL requires that the nfs db volume be shared with the database for backups to function correctly. +PostgresSQL # Backup Container Options -You can run the Backup Container for postgres and mongo databases separately or in a mixed environment. +You can run the Backup Container for supported databases separately or in a mixed environment. For a mixed environment: 1) You MUST use the recommended `backup.conf` configuration. 2) Within the `backup.conf`, you MUST specify the `DatabaseType` for each listed database. -3) You will need to create two builds and two deployment configs. One for a postgres backup container and the other for a mongo backup container. +3) You will need to create a build and deployment config for each type of supported backup container in use. 4) Mount the same `backup.conf` file (ConfigMap) to each deployed container. ## Backups in OpenShift @@ -76,7 +82,7 @@ Together, the scripts and templates provided in the [openshift](./openshift) dir The following environment variables are defaults used by the `backup` app. -**NOTE**: These environment variables MUST MATCH those used by the postgresql container(s) you are planning to backup. +**NOTE**: These environment variables MUST MATCH those used by the database container(s) you are planning to backup. | Name | Default (if not set) | Purpose | | ---- | ------- | ------- | @@ -283,6 +289,9 @@ Plugin Examples: - [backup.mongo.plugin](./docker/backup.mongo.plugin) - Mongo backup implementation. +- [backup.mssql.plugin](./docker/backup.mssql.plugin) + - MSSQL backup implementation. + - [backup.null.plugin](./docker/backup.null.plugin) - Sample/Template backup implementation that simply outputs log messages for the various operations. diff --git a/config/backup.conf b/config/backup.conf index a3b4388..3541cfb 100644 --- a/config/backup.conf +++ b/config/backup.conf @@ -9,9 +9,9 @@ # - :/ # - =/ # - =:/ -# can be postgres or mongo +# can be postgres, mongo or mssql # MUST be specified when you are sharing a -# single backup.conf file between postgres and mongo +# single backup.conf file between postgres, mongo and mssql # backup containers. If you do not specify # the listed databases are assumed to be valid for the # backup container in which the configuration is mounted. @@ -21,6 +21,7 @@ # - postgres=postgresql:5432/my_database # - mongo=mongodb/my_database # - mongo=mongodb:27017/my_database +# - mssql=mssql_server:1433/my_database # ----------------------------------------------------------- # Cron Scheduling: # ----------------------------------------------------------- @@ -42,6 +43,7 @@ # postgres=postgresql:5432/TheOrgBook_Database # mongo=mender-mongodb:27017/useradm # postgres=wallet-db/tob_issuer +# mssql=pims-db-dev:1433/pims # # 0 1 * * * default ./backup.sh -s # 0 4 * * * default ./backup.sh -s -v all diff --git a/docker/Dockerfile_MSSQL b/docker/Dockerfile_MSSQL new file mode 100644 index 0000000..189e868 --- /dev/null +++ b/docker/Dockerfile_MSSQL @@ -0,0 +1,50 @@ +FROM mcr.microsoft.com/mssql/rhel/server:2019-CU1-rhel-8 + +# Change timezone to PST for convenience +ENV TZ=PST8PDT + +# Set the workdir to be root +WORKDIR / + +# Load the backup scripts into the container (must be executable). +COPY backup.* / + +COPY webhook-template.json / + +# ======================================================================================================== +# Install go-crond (from https://github.com/BCDevOps/go-crond) +# - Adds some additional logging enhancements on top of the upstream project; +# https://github.com/webdevops/go-crond +# +# CRON Jobs in OpenShift: +# - https://blog.danman.eu/cron-jobs-in-openshift/ +# -------------------------------------------------------------------------------------------------------- +ARG SOURCE_REPO=BCDevOps +ARG GOCROND_VERSION=0.6.3 +ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond-64-linux /usr/bin/go-crond + +USER root + +RUN chmod ug+x /usr/bin/go-crond +# ======================================================================================================== + +# ======================================================================================================== +# Perform operations that require root privilages here ... +# -------------------------------------------------------------------------------------------------------- +RUN echo $TZ > /etc/timezone +# ======================================================================================================== +COPY uid_entrypoint /opt/mssql-tools/bin/ +RUN chmod -R a+rwx /opt/mssql-tools/bin/uid_entrypoint + +ENV PATH=${PATH}:/opt/mssql/bin:/opt/mssql-tools/bin +RUN mkdir -p /var/opt/mssql/data && \ + chmod -R g=u /var/opt/mssql /etc/passwd + +# Important - Reset to the base image's user account. +USER 10001 + +# Set the default CMD. +CMD sh /backup.sh + +### user name recognition at runtime w/ an arbitrary uid - for OpenShift deployments +ENTRYPOINT [ "/opt/mssql-tools/bin/uid_entrypoint" ] \ No newline at end of file diff --git a/docker/backup.container.utils b/docker/backup.container.utils index 3bb4115..2fce0c7 100644 --- a/docker/backup.container.utils +++ b/docker/backup.container.utils @@ -22,6 +22,16 @@ function isMongo(){ ) } +function isMsSql(){ + ( + if isInstalled "sqlcmd"; then + return 0 + else + return 1 + fi + ) +} + function getContainerType(){ ( local _containerType=${UNKNOWN_DB} @@ -31,6 +41,8 @@ function getContainerType(){ _containerType=${POSTGRE_DB} elif isMongo; then _containerType=${MONGO_DB} + elif isMsSql; then + _containerType=${MSSQL_DB} else _containerType=${UNKNOWN_DB} _rtnCd=1 diff --git a/docker/backup.mssql.plugin b/docker/backup.mssql.plugin new file mode 100644 index 0000000..e9bbc3a --- /dev/null +++ b/docker/backup.mssql.plugin @@ -0,0 +1,229 @@ +#!/bin/bash +# ================================================================================================================= +# MSSQL Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/var/opt/mssql/data" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + #TODO: add support for backing up transaction log as well. + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "BACKUP DATABASE ${_database} TO DISK = N'${_fileName}' WITH NOFORMAT, NOINIT, SKIP, NOREWIND, NOUNLOAD, STATS = 10" + return ${?} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + #force single user mode on database to ensure restore works properly + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "ALTER DATABASE ${_database} SET SINGLE_USER WITH ROLLBACK AFTER 30;RESTORE DATABASE ${_database} FROM DISK = N'${_fileName}' WITH FILE = 1, NOUNLOAD, REPLACE, STATS=5;ALTER DATABASE ${_database} SET MULTI_USER" + return ${?} + ) +} + +function onStartServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + /opt/mssql/bin/sqlservr --accept-eula & + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SHUTDOWN" + ) +} + +function onCleanup(){ + ( + if ! dirIsEmpty ${serverDataDirectory}; then + # Delete the database files and configuration + echo -e "Cleaning up ...\n" >&2 + rm -rf ${serverDataDirectory}/* + else + echo -e "Already clean ...\n" >&2 + fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + if sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SELECT 1" >/dev/null 2>&1; then + return 0 + else + return 1 + fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + tables=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT table_name FROM information_schema.tables WHERE TABLE_CATALOG = '${_database}' AND table_type='BASE TABLE';") + rtnCd=${?} + + # Get the size of the restored database + if (( ${rtnCd} == 0 )); then + size=$(getDbSize -l "${_databaseSpec}") + rtnCd=${?} + fi + + if (( ${rtnCd} == 0 )); then + numResults=$(echo "${tables}"| wc -l) + if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then + # All good + verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." + else + # Not so good + verificationLog="\nNo tables were found in the restored database ${_database}." + rtnCd="3" + fi + fi + + echo ${verificationLog} + return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + size=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT CONVERT(VARCHAR,SUM(size)*8/1024)+' MB' AS 'size' FROM sys.master_files m INNER JOIN sys.databases d ON d.database_id = m.database_id WHERE d.name = '${_database}' AND m.type_desc = 'ROWS' GROUP BY d.name") + rtnCd=${?} + + echo ${size} + return ${rtnCd} + ) +} + +function getLocalOrDBUsername(){ + ( + _databaseSpec=${1} + _localhost="127.0.0.1" + _hostname=${2} + if [ "$_hostname" == "$_localhost" ]; then + _username=sa + else + _username=$(getUsername ${_databaseSpec}) + fi + echo ${_username} + ) +} +# ================================================================================================================= \ No newline at end of file diff --git a/docker/backup.settings b/docker/backup.settings index 7de738c..deb5d4f 100644 --- a/docker/backup.settings +++ b/docker/backup.settings @@ -48,6 +48,7 @@ export PRUNE="prune" export UNKNOWN_DB="null" export MONGO_DB="mongo" export POSTGRE_DB="postgres" +export MSSQL_DB="mssql" export CONTAINER_TYPE="$(getContainerType)" # Other: diff --git a/docker/backup.usage b/docker/backup.usage index 32238fd..05f092e 100644 --- a/docker/backup.usage +++ b/docker/backup.usage @@ -5,7 +5,7 @@ function usage () { cat <<-EOF - Automated backup script for PostgreSQL and MongoDB databases. + Automated backup script for PostgreSQL, MongoDB and MSSQL databases. There are two modes of scheduling backups: - Cron Mode: diff --git a/docker/uid_entrypoint b/docker/uid_entrypoint new file mode 100644 index 0000000..ba474c9 --- /dev/null +++ b/docker/uid_entrypoint @@ -0,0 +1,7 @@ +#!/bin/sh +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-sqlservr}:x:$(id -u):0:${USER_NAME:-sqlservr} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi +exec "$@" \ No newline at end of file diff --git a/openshift/templates/backup/backup-build.json b/openshift/templates/backup/backup-build.json index 89f4b05..9e8d0ca 100644 --- a/openshift/templates/backup/backup-build.json +++ b/openshift/templates/backup/backup-build.json @@ -59,7 +59,7 @@ { "name": "NAME", "displayName": "Name", - "description": "The name assigned to all of the resources. Use 'backup-postgres' for Postgres builds or 'backup-mongo' for MongoDB builds.", + "description": "The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider", "required": true, "value": "backup-postgres" }, @@ -87,7 +87,7 @@ { "name": "DOCKER_FILE_PATH", "displayName": "Docker File", - "description": "The path and file of the docker file defining the build. Choose either 'Dockerfile' for Postgres builds or 'Dockerfile_Mongo' for MongoDB builds.", + "description": "The path and file of the docker file defining the build. Choose either 'Dockerfile' for Postgres builds or 'Dockerfile_Mongo' for MongoDB builds or 'Dockerfile_MSSQL' for MSSQL builds.", "required": false, "value": "Dockerfile" }, diff --git a/openshift/templates/backup/backup-deploy.json b/openshift/templates/backup/backup-deploy.json index 3cbe6cd..4b55673 100644 --- a/openshift/templates/backup/backup-deploy.json +++ b/openshift/templates/backup/backup-deploy.json @@ -289,14 +289,14 @@ { "name": "NAME", "displayName": "Name", - "description": "The name assigned to all of the resources. Use 'backup-postgres' for Postgres deployments or 'backup-mongo' for MongoDB deployments.", + "description": "The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider", "required": true, "value": "backup-postgres" }, { "name": "SOURCE_IMAGE_NAME", "displayName": "Source Image Name", - "description": "The name of the image to use for this resource. Use 'backup-postgres' for Postgres deployments or 'backup-mongo' for MongoDB deployments.", + "description": "The name of the image to use for this resource. Use 'backup-{database name}' depending on your database provider", "required": true, "value": "backup-postgres" }, @@ -356,6 +356,12 @@ "required": true, "value": "database-password" }, + { + "name": "MSSQL_SA_PASSWORD", + "displayName": "MSSQL SA Password", + "description": "The database password to use for the local backup database.", + "required": false + }, { "name": "TABLE_SCHEMA", "displayName": "Table Schema", @@ -527,7 +533,7 @@ { "name": "VERIFICATION_VOLUME_MOUNT_PATH", "displayName": "Verification Volume Mount Path", - "description": "The path on which to mount the verification volume. This is used by the database server to contain the database configuration and data files. For Mongo, please use /var/lib/mongodb/data", + "description": "The path on which to mount the verification volume. This is used by the database server to contain the database configuration and data files. For Mongo, please use /var/lib/mongodb/data . For MSSQL, please use /var/opt/mssql/data", "required": true, "value": "/var/lib/pgsql/data" },