From 2b4d7e0e621abb235c538d8ce987d8bb57d89ecf Mon Sep 17 00:00:00 2001 From: Dmitry Ratushnyy Date: Tue, 27 Jun 2023 04:46:00 +0000 Subject: [PATCH] [WIP] Adding backup --- actions.yaml | 16 + lib/charms/data_platform_libs/v0/s3.py | 723 +++++++++++++++++ lib/charms/mongodb/v0/mongodb_backups.py | 229 ++++++ metadata.yaml | 3 + src/charm.py | 130 +++- tests/integration/backup_tests/__init__.py | 2 + tests/integration/backup_tests/helpers.py | 149 ++++ .../integration/backup_tests/test_backups.py | 404 ++++++++++ tests/unit/test_mongodb_backups.py | 729 ++++++++++++++++++ 9 files changed, 2367 insertions(+), 18 deletions(-) create mode 100644 lib/charms/data_platform_libs/v0/s3.py create mode 100644 lib/charms/mongodb/v0/mongodb_backups.py create mode 100644 tests/integration/backup_tests/__init__.py create mode 100644 tests/integration/backup_tests/helpers.py create mode 100644 tests/integration/backup_tests/test_backups.py create mode 100644 tests/unit/test_mongodb_backups.py diff --git a/actions.yaml b/actions.yaml index 32e883f99..a82f6a978 100644 --- a/actions.yaml +++ b/actions.yaml @@ -17,6 +17,22 @@ set-password: password: type: string description: The password will be auto-generated if this option is not specified. + +create-backup: + description: Create a database backup. + S3 credentials are retrieved from a relation with the S3 integrator charm. + +list-backups: + description: List available backup_ids in the S3 bucket and path provided by the S3 integrator charm. + +restore: + description: Restore a database backup. + S3 credentials are retrieved from a relation with the S3 integrator charm. + params: + backup-id: + type: string + description: A backup-id to identify the backup to restore. Format of <%Y-%m-%dT%H:%M:%SZ> + set-tls-private-key: description: Set the privates key, which will be used for certificate signing requests (CSR). Run for each unit separately. params: diff --git a/lib/charms/data_platform_libs/v0/s3.py b/lib/charms/data_platform_libs/v0/s3.py new file mode 100644 index 000000000..9fb518a56 --- /dev/null +++ b/lib/charms/data_platform_libs/v0/s3.py @@ -0,0 +1,723 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A library for communicating with the S3 credentials providers and consumers. + +This library provides the relevant interface code implementing the communication +specification for fetching, retrieving, triggering, and responding to events related to +the S3 provider charm and its consumers. + +### Provider charm + +The provider is implemented in the `s3-provider` charm which is meant to be deployed +alongside one or more consumer charms. The provider charm is serving the s3 credentials and +metadata needed to communicate and work with an S3 compatible backend. + +Example: +```python + +from charms.data_platform_libs.v0.s3 import CredentialRequestedEvent, S3Provider + + +class ExampleProviderCharm(CharmBase): + def __init__(self, *args) -> None: + super().__init__(*args) + self.s3_provider = S3Provider(self, "s3-credentials") + + self.framework.observe(self.s3_provider.on.credentials_requested, + self._on_credential_requested) + + def _on_credential_requested(self, event: CredentialRequestedEvent): + if not self.unit.is_leader(): + return + + # get relation id + relation_id = event.relation.id + + # get bucket name + bucket = event.bucket + + # S3 configuration parameters + desired_configuration = {"access-key": "your-access-key", "secret-key": + "your-secret-key", "bucket": "your-bucket"} + + # update the configuration + self.s3_provider.update_connection_info(relation_id, desired_configuration) + + # or it is possible to set each field independently + + self.s3_provider.set_secret_key(relation_id, "your-secret-key") + + +if __name__ == "__main__": + main(ExampleProviderCharm) + + +### Requirer charm + +The requirer charm is the charm requiring the S3 credentials. +An example of requirer charm is the following: + +Example: +```python + +from charms.data_platform_libs.v0.s3 import ( + CredentialsChangedEvent, + CredentialsGoneEvent, + S3Requirer +) + +class ExampleRequirerCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + + bucket_name = "test-bucket" + # if bucket name is not provided the bucket name will be generated + # e.g., ('relation-{relation.id}') + + self.s3_client = S3Requirer(self, "s3-credentials", bucket_name) + + self.framework.observe(self.s3_client.on.credentials_changed, self._on_credential_changed) + self.framework.observe(self.s3_client.on.credentials_gone, self._on_credential_gone) + + def _on_credential_changed(self, event: CredentialsChangedEvent): + + # access single parameter credential + secret_key = event.secret_key + access_key = event.access_key + + # or as alternative all credentials can be collected as a dictionary + credentials = self.s3_client.get_s3_credentials() + + def _on_credential_gone(self, event: CredentialsGoneEvent): + # credentials are removed + pass + + if __name__ == "__main__": + main(ExampleRequirerCharm) +``` + +""" +import json +import logging +from collections import namedtuple +from typing import Dict, List, Optional + +import ops.charm +import ops.framework +import ops.model +from ops.charm import ( + CharmBase, + CharmEvents, + EventSource, + Object, + ObjectEvents, + RelationBrokenEvent, + RelationChangedEvent, + RelationEvent, + RelationJoinedEvent, +) +from ops.model import Relation + +# The unique Charmhub library identifier, never change it +LIBID = "fca396f6254246c9bfa565b1f85ab528" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 2 + +logger = logging.getLogger(__name__) + +Diff = namedtuple("Diff", "added changed deleted") +Diff.__doc__ = """ +A tuple for storing the diff between two data mappings. + +added - keys that were added +changed - keys that still exist but have new values +deleted - key that were deleted""" + + +def diff(event: RelationChangedEvent, bucket: str) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + bucket: bucket of the databag (app or unit) + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + # Retrieve the old data from the data key in the application relation databag. + old_data = json.loads(event.relation.data[bucket].get("data", "{}")) + # Retrieve the new data from the event relation databag. + new_data = { + key: value for key, value in event.relation.data[event.app].items() if key != "data" + } + + # These are the keys that were added to the databag and triggered this event. + added = new_data.keys() - old_data.keys() + # These are the keys that were removed from the databag and triggered this event. + deleted = old_data.keys() - new_data.keys() + # These are the keys that already existed in the databag, + # but had their values changed. + changed = {key for key in old_data.keys() & new_data.keys() if old_data[key] != new_data[key]} + + # TODO: evaluate the possibility of losing the diff if some error + # happens in the charm before the diff is completely checked (DPE-412). + # Convert the new_data to a serializable format and save it for a next diff check. + event.relation.data[bucket].update({"data": json.dumps(new_data)}) + + # Return the diff with all possible changes. + return Diff(added, changed, deleted) + + +class BucketEvent(RelationEvent): + """Base class for bucket events.""" + + @property + def bucket(self) -> Optional[str]: + """Returns the bucket was requested.""" + return self.relation.data[self.relation.app].get("bucket") + + +class CredentialRequestedEvent(BucketEvent): + """Event emitted when a set of credential is requested for use on this relation.""" + + +class S3CredentialEvents(CharmEvents): + """Event descriptor for events raised by S3Provider.""" + + credentials_requested = EventSource(CredentialRequestedEvent) + + +class S3Provider(Object): + """A provider handler for communicating S3 credentials to consumers.""" + + on = S3CredentialEvents() + + def __init__( + self, + charm: CharmBase, + relation_name: str, + ): + super().__init__(charm, relation_name) + self.charm = charm + self.local_app = self.charm.model.app + self.local_unit = self.charm.unit + self.relation_name = relation_name + + # monitor relation changed event for changes in the credentials + self.framework.observe(charm.on[relation_name].relation_changed, self._on_relation_changed) + + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """React to the relation changed event by consuming data.""" + if not self.charm.unit.is_leader(): + return + diff = self._diff(event) + # emit on credential requested if bucket is provided by the requirer application + if "bucket" in diff.added: + self.on.credentials_requested.emit(event.relation, app=event.app, unit=event.unit) + + def _load_relation_data(self, raw_relation_data: dict) -> dict: + """Loads relation data from the relation data bag. + + Args: + raw_relation_data: Relation data from the databag + Returns: + dict: Relation data in dict format. + """ + connection_data = dict() + for key in raw_relation_data: + try: + connection_data[key] = json.loads(raw_relation_data[key]) + except (json.decoder.JSONDecodeError, TypeError): + connection_data[key] = raw_relation_data[key] + return connection_data + + # def _diff(self, event: RelationChangedEvent) -> Diff: + # """Retrieves the diff of the data in the relation changed databag. + + # Args: + # event: relation changed event. + + # Returns: + # a Diff instance containing the added, deleted and changed + # keys from the event relation databag. + # """ + # # Retrieve the old data from the data key in the application relation databag. + # old_data = json.loads(event.relation.data[self.local_app].get("data", "{}")) + # # Retrieve the new data from the event relation databag. + # new_data = { + # key: value for key, value in event.relation.data[event.app].items() if key != "data" + # } + + # # These are the keys that were added to the databag and triggered this event. + # added = new_data.keys() - old_data.keys() + # # These are the keys that were removed from the databag and triggered this event. + # deleted = old_data.keys() - new_data.keys() + # # These are the keys that already existed in the databag, + # # but had their values changed. + # changed = { + # key for key in old_data.keys() & new_data.keys() if old_data[key] != new_data[key] + # } + + # # TODO: evaluate the possibility of losing the diff if some error + # # happens in the charm before the diff is completely checked (DPE-412). + # # Convert the new_data to a serializable format and save it for a next diff check. + # event.relation.data[self.local_app].update({"data": json.dumps(new_data)}) + + # # Return the diff with all possible changes. + # return Diff(added, changed, deleted) + + def _diff(self, event: RelationChangedEvent) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + return diff(event, self.local_app) + + def fetch_relation_data(self) -> dict: + """Retrieves data from relation. + + This function can be used to retrieve data from a relation + in the charm code when outside an event callback. + + Returns: + a dict of the values stored in the relation data bag + for all relation instances (indexed by the relation id). + """ + data = {} + for relation in self.relations: + data[relation.id] = { + key: value for key, value in relation.data[relation.app].items() if key != "data" + } + return data + + def update_connection_info(self, relation_id: int, connection_data: dict) -> None: + """Updates the credential data as set of key-value pairs in the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + connection_data: dict containing the key-value pairs + that should be updated. + """ + # check and write changes only if you are the leader + if not self.local_unit.is_leader(): + return + + relation = self.charm.model.get_relation(self.relation_name, relation_id) + + if not relation: + return + + # configuration options that are list + s3_list_options = ["attributes", "tls-ca-chain"] + + # update the databag, if connection data did not change with respect to before + # the relation changed event is not triggered + updated_connection_data = {} + for configuration_option, configuration_value in connection_data.items(): + if configuration_option in s3_list_options: + updated_connection_data[configuration_option] = json.dumps(configuration_value) + else: + updated_connection_data[configuration_option] = configuration_value + + relation.data[self.local_app].update(updated_connection_data) + logger.debug(f"Updated S3 connection info: {updated_connection_data}") + + @property + def relations(self) -> List[Relation]: + """The list of Relation instances associated with this relation_name.""" + return list(self.charm.model.relations[self.relation_name]) + + def set_bucket(self, relation_id: int, bucket: str) -> None: + """Sets bucket name in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + bucket: the bucket name. + """ + self.update_connection_info(relation_id, {"bucket": bucket}) + + def set_access_key(self, relation_id: int, access_key: str) -> None: + """Sets access-key value in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + access_key: the access-key value. + """ + self.update_connection_info(relation_id, {"access-key": access_key}) + + def set_secret_key(self, relation_id: int, secret_key: str) -> None: + """Sets the secret key value in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + secret_key: the value of the secret key. + """ + self.update_connection_info(relation_id, {"secret-key": secret_key}) + + def set_path(self, relation_id: int, path: str) -> None: + """Sets the path value in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + path: the path value. + """ + self.update_connection_info(relation_id, {"path": path}) + + def set_endpoint(self, relation_id: int, endpoint: str) -> None: + """Sets the endpoint address in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + endpoint: the endpoint address. + """ + self.update_connection_info(relation_id, {"endpoint": endpoint}) + + def set_region(self, relation_id: int, region: str) -> None: + """Sets the region location in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + region: the region address. + """ + self.update_connection_info(relation_id, {"region": region}) + + def set_s3_uri_style(self, relation_id: int, s3_uri_style: str) -> None: + """Sets the S3 URI style in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + s3_uri_style: the s3 URI style. + """ + self.update_connection_info(relation_id, {"s3-uri-style": s3_uri_style}) + + def set_storage_class(self, relation_id: int, storage_class: str) -> None: + """Sets the storage class in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + storage_class: the storage class. + """ + self.update_connection_info(relation_id, {"storage-class": storage_class}) + + def set_tls_ca_chain(self, relation_id: int, tls_ca_chain: List[str]) -> None: + """Sets the tls_ca_chain value in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + tls_ca_chain: the TLS Chain value. + """ + self.update_connection_info(relation_id, {"tls-ca-chain": tls_ca_chain}) + + def set_s3_api_version(self, relation_id: int, s3_api_version: str) -> None: + """Sets the S3 API version in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + s3_api_version: the S3 version value. + """ + self.update_connection_info(relation_id, {"s3-api-version": s3_api_version}) + + def set_attributes(self, relation_id: int, attributes: List[str]) -> None: + """Sets the connection attributes in application databag. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + attributes: the attributes value. + """ + self.update_connection_info(relation_id, {"attributes": attributes}) + + +class S3Event(RelationEvent): + """Base class for S3 storage events.""" + + @property + def bucket(self) -> Optional[str]: + """Returns the bucket name.""" + return self.relation.data[self.relation.app].get("bucket") + + @property + def access_key(self) -> Optional[str]: + """Returns the access key.""" + return self.relation.data[self.relation.app].get("access-key") + + @property + def secret_key(self) -> Optional[str]: + """Returns the secret key.""" + return self.relation.data[self.relation.app].get("secret-key") + + @property + def path(self) -> Optional[str]: + """Returns the path where data can be stored.""" + return self.relation.data[self.relation.app].get("path") + + @property + def endpoint(self) -> Optional[str]: + """Returns the endpoint address.""" + return self.relation.data[self.relation.app].get("endpoint") + + @property + def region(self) -> Optional[str]: + """Returns the region.""" + return self.relation.data[self.relation.app].get("region") + + @property + def s3_uri_style(self) -> Optional[str]: + """Returns the s3 uri style.""" + return self.relation.data[self.relation.app].get("s3-uri-style") + + @property + def storage_class(self) -> Optional[str]: + """Returns the storage class name.""" + return self.relation.data[self.relation.app].get("storage-class") + + @property + def tls_ca_chain(self) -> Optional[List[str]]: + """Returns the TLS CA chain.""" + tls_ca_chain = self.relation.data[self.relation.app].get("tls-ca-chain") + if tls_ca_chain is not None: + return json.loads(tls_ca_chain) + return None + + @property + def s3_api_version(self) -> Optional[str]: + """Returns the S3 API version.""" + return self.relation.data[self.relation.app].get("s3-api-version") + + @property + def attributes(self) -> Optional[List[str]]: + """Returns the attributes.""" + attributes = self.relation.data[self.relation.app].get("attributes") + if attributes is not None: + return json.loads(attributes) + return None + + +class CredentialsChangedEvent(S3Event): + """Event emitted when S3 credential are changed on this relation.""" + + +class CredentialsGoneEvent(RelationEvent): + """Event emitted when S3 credential are removed from this relation.""" + + +class S3CredentialRequiresEvents(ObjectEvents): + """Event descriptor for events raised by the S3Provider.""" + + credentials_changed = EventSource(CredentialsChangedEvent) + credentials_gone = EventSource(CredentialsGoneEvent) + + +S3_REQUIRED_OPTIONS = ["access-key", "secret-key"] + + +class S3Requirer(Object): + """Requires-side of the s3 relation.""" + + on = S3CredentialRequiresEvents() + + def __init__(self, charm: ops.charm.CharmBase, relation_name: str, bucket_name: str = None): + """Manager of the s3 client relations.""" + super().__init__(charm, relation_name) + + self.relation_name = relation_name + self.charm = charm + self.local_app = self.charm.model.app + self.local_unit = self.charm.unit + self.bucket = bucket_name + + self.framework.observe( + self.charm.on[self.relation_name].relation_changed, self._on_relation_changed + ) + + self.framework.observe( + self.charm.on[self.relation_name].relation_joined, self._on_relation_joined + ) + + self.framework.observe( + self.charm.on[self.relation_name].relation_broken, + self._on_relation_broken, + ) + + def _generate_bucket_name(self, event: RelationJoinedEvent): + """Returns the bucket name generated from relation id.""" + return f"relation-{event.relation.id}" + + def _on_relation_joined(self, event: RelationJoinedEvent) -> None: + """Event emitted when the application joins the s3 relation.""" + if self.bucket is None: + self.bucket = self._generate_bucket_name(event) + self.update_connection_info(event.relation.id, {"bucket": self.bucket}) + + def fetch_relation_data(self) -> dict: + """Retrieves data from relation. + + This function can be used to retrieve data from a relation + in the charm code when outside an event callback. + + Returns: + a dict of the values stored in the relation data bag + for all relation instances (indexed by the relation id). + """ + data = {} + + for relation in self.relations: + data[relation.id] = self._load_relation_data(relation.data[self.charm.app]) + return data + + def update_connection_info(self, relation_id: int, connection_data: dict) -> None: + """Updates the credential data as set of key-value pairs in the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + connection_data: dict containing the key-value pairs + that should be updated. + """ + # check and write changes only if you are the leader + if not self.local_unit.is_leader(): + return + + relation = self.charm.model.get_relation(self.relation_name, relation_id) + + if not relation: + return + + # update the databag, if connection data did not change with respect to before + # the relation changed event is not triggered + # configuration options that are list + s3_list_options = ["attributes", "tls-ca-chain"] + updated_connection_data = {} + for configuration_option, configuration_value in connection_data.items(): + if configuration_option in s3_list_options: + updated_connection_data[configuration_option] = json.dumps(configuration_value) + else: + updated_connection_data[configuration_option] = configuration_value + + relation.data[self.local_app].update(updated_connection_data) + logger.debug(f"Updated S3 credentials: {updated_connection_data}") + + def _load_relation_data(self, raw_relation_data: dict) -> dict: + """Loads relation data from the relation data bag. + + Args: + raw_relation_data: Relation data from the databag + Returns: + dict: Relation data in dict format. + """ + connection_data = dict() + for key in raw_relation_data: + try: + connection_data[key] = json.loads(raw_relation_data[key]) + except (json.decoder.JSONDecodeError, TypeError): + connection_data[key] = raw_relation_data[key] + return connection_data + + def _diff(self, event: RelationChangedEvent) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + return diff(event, self.local_unit) + + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """Notify the charm about the presence of S3 credentials.""" + # check if the mandatory options are in the relation data + contains_required_options = True + # get current credentials data + credentials = self.get_s3_connection_info() + # records missing options + missing_options = [] + for configuration_option in S3_REQUIRED_OPTIONS: + if configuration_option not in credentials: + contains_required_options = False + missing_options.append(configuration_option) + # emit credential change event only if all mandatory fields are present + if contains_required_options: + self.on.credentials_changed.emit(event.relation, app=event.app, unit=event.unit) + else: + logger.warning( + f"Some mandatory fields: {missing_options} are not present, do not emit credential change event!" + ) + + def get_s3_connection_info(self) -> Dict: + """Return the s3 credentials as a dictionary.""" + relation = self.charm.model.get_relation(self.relation_name) + if not relation: + return {} + return self._load_relation_data(relation.data[relation.app]) + + def _on_relation_broken(self, event: RelationBrokenEvent) -> None: + """Notify the charm about a broken S3 credential store relation.""" + self.on.credentials_gone.emit(event.relation, app=event.app, unit=event.unit) + + @property + def relations(self) -> List[Relation]: + """The list of Relation instances associated with this relation_name.""" + return list(self.charm.model.relations[self.relation_name]) diff --git a/lib/charms/mongodb/v0/mongodb_backups.py b/lib/charms/mongodb/v0/mongodb_backups.py new file mode 100644 index 000000000..7147741a0 --- /dev/null +++ b/lib/charms/mongodb/v0/mongodb_backups.py @@ -0,0 +1,229 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""In this class, we manage backup configurations and actions. + +Specifically backups are handled with Percona Backup MongoDB (pbm). +A user for PBM is created when MongoDB is first started during the start phase. +This user is named "backup". +""" + +import logging +from typing import Dict + +from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer +from ops.framework import Object +from ops.model import ( + ActiveStatus, + BlockedStatus, + MaintenanceStatus, + StatusBase, + WaitingStatus, + ModelError, + RuntimeError, +) + +# The unique Charmhub library identifier, never change it +LIBID = "9f2b91c6128d48d6ba22724bf365da3b" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + +logger = logging.getLogger(__name__) + +S3_PBM_OPTION_MAP = { + "region": "storage.s3.region", + "bucket": "storage.s3.bucket", + "path": "storage.s3.prefix", + "access-key": "storage.s3.credentials.access-key-id", + "secret-key": "storage.s3.credentials.secret-access-key", + "endpoint": "storage.s3.endpointUrl", + "storage-class": "storage.s3.storageClass", +} +S3_RELATION = "s3-credentials" +REMAPPING_PATTERN = r"\ABackup doesn't match current cluster topology - it has different replica set names. Extra shards in the backup will cause this, for a simple example. The extra/unknown replica set names found in the backup are: ([^,\s]+)([.] Backup has no data for the config server or sole replicaset)?\Z" +PBM_CONFIG_FILE_PATH = "/etc/pbm_config.yaml" + +class ResyncError(Exception): + """Raised when pbm is resyncing configurations and is not ready to be used.""" + + +class SetPBMConfigError(Exception): + """Raised when pbm cannot configure a given option.""" + + +class PBMBusyError(Exception): + """Raised when PBM is busy and cannot run another operation.""" + + +class MongoDBBackups(Object): + """Manages MongoDB backups.""" + + def __init__(self, charm, substrate): + """Manager of MongoDB client relations.""" + super().__init__(charm, "client-relations") + self.charm = charm + self.substrate = substrate + + # s3 relation handles the config options for s3 backups + self.s3_client = S3Requirer(self.charm, S3_RELATION) + self.framework.observe( + self.s3_client.on.credentials_changed, self._on_s3_credential_changed + ) + self.framework.observe(self.charm.on.create_backup_action, self._on_create_backup_action) + self.framework.observe(self.charm.on.list_backups_action, self._on_list_backups_action) + self.framework.observe(self.charm.on.restore_action, self._on_restore_action) + + def _on_s3_credential_changed(self, event: CredentialsChangedEvent): + """Sets pbm credentials, resyncs if necessary and reports config errors.""" + # handling PBM configurations requires that MongoDB is running and the pbm snap is + # installed. + if not self.charm.db_initialised: + logger.debug("Cannot set PBM configurations, MongoDB has not yet started.") + event.defer() + return + + try: + pbm_service = self.charm.get_backup_service() + except ModelError as e: + logger.debug(f"Cannot set PBM configurations, pbm-agent service not found. {e}") + event.defer() + except RuntimeError as e: + logger.error(f"Cannot set PBM configurations. Failed to get pbm serivice. {e}") + self.charm.unit.status = BlockedStatus("couldn't configure s3 backup options.") + # TODO + self._set_config_options(self._get_pbm_configs(), pbm_service) + + def _on_create_backup_action(self, event) -> None: + if self.model.get_relation(S3_RELATION) is None: + event.fail("Relation with s3-integrator charm missing, cannot create backup.") + return + + # only leader can create backups. This prevents multiple backups from being attempted at + # once. + if not self.charm.unit.is_leader(): + event.fail("The action can be run only on leader unit.") + return + + # cannot create backup if pbm is not ready. This could be due to: resyncing, incompatible, + # options, incorrect credentials, or already creating a backup + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + if isinstance(pbm_status, MaintenanceStatus): + event.fail( + "Can only create one backup at a time, please wait for current backup to finish." + ) + return + if isinstance(pbm_status, WaitingStatus): + event.defer() + logger.debug( + "Sync-ing configurations needs more time, must wait before creating a backup." + ) + return + if isinstance(pbm_status, BlockedStatus): + event.fail(f"Cannot create backup {pbm_status.message}.") + return + + # TODO create backup + + def _get_pbm_status(self) -> StatusBase: + """Retrieve pbm status.""" + # TODO check pbm status + return ActiveStatus("") + + def _on_list_backups_action(self, event) -> None: + if self.model.get_relation(S3_RELATION) is None: + event.fail("Relation with s3-integrator charm missing, cannot list backups.") + return + + # cannot list backups if pbm is resyncing, or has incompatible options or incorrect + # credentials + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + if isinstance(pbm_status, WaitingStatus): + event.defer() + logger.debug( + "Sync-ing configurations needs more time, must wait before listing backups." + ) + return + if isinstance(pbm_status, BlockedStatus): + event.fail(f"Cannot list backups: {pbm_status.message}.") + return + + # TODO list backups + + def _on_restore_action(self, event) -> None: + if self.model.get_relation(S3_RELATION) is None: + event.fail("Relation with s3-integrator charm missing, cannot restore from a backup.") + return + + backup_id = event.params.get("backup-id") + if not backup_id: + event.fail("Missing backup-id to restore") + return + + # only leader can restore backups. This prevents multiple restores from being attempted at + # once. + if not self.charm.unit.is_leader(): + event.fail("The action can be run only on leader unit.") + return + + # cannot restore backup if pbm is not ready. This could be due to: resyncing, incompatible, + # options, incorrect credentials, creating a backup, or already performing a restore. + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + if isinstance(pbm_status, MaintenanceStatus): + event.fail("Please wait for current backup/restore to finish.") + return + if isinstance(pbm_status, WaitingStatus): + event.defer() + logger.debug("Sync-ing configurations needs more time, must wait before restoring.") + return + if isinstance(pbm_status, BlockedStatus): + event.fail(f"Cannot restore backup {pbm_status.message}.") + return + + # TODO restore backup + + + # BEGIN: helper functions + def _set_config_options(self, pbm_configs, pbm_service): + """Applying given configurations with pbm.""" + # Setting empty pbm + container = self.charm.get_container() + container.push( + PBM_CONFIG_FILE_PATH, + "# this file is to be left empty. Changes in this file will be ignored.", + make_dirs=True, + permissions=0o400, + ) + container.add_layer(Config.Backup.SERVICE_NAME, self.charm.backup_layer, combine=True) + container.replan( + # subprocess.check_output( + # f"charmed-mongodb.pbm config --file {MONGODB_SNAP_DATA_DIR}/etc/pbm/pbm_config.yaml", + # shell=True, + # ) + + # the pbm tool can only set one configuration at a time. + for pbm_key, pbm_value in pbm_configs.items(): + self._pbm_set_config(pbm_key, pbm_value) + + def _pbm_set_config(self, key: str, value: str) -> None: + """Runs the charmed-mongodb.pbm config command for the provided key and value.""" + config_cmd = f'pbm config --set {key}="{value}"' + #subprocess.check_output(config_cmd, shell=True) + + def _get_pbm_configs(self) -> Dict: + """Returns a dictionary of desired PBM configurations.""" + pbm_configs = {"storage.type": "s3"} + credentials = self.s3_client.get_s3_connection_info() + for s3_option, s3_value in credentials.items(): + if s3_option not in S3_PBM_OPTION_MAP: + continue + + pbm_configs[S3_PBM_OPTION_MAP[s3_option]] = s3_value + return pbm_configs \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index fcf53f87d..b41d31ff6 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -38,6 +38,8 @@ requires: interface: loki_push_api limit: 1 optional: true + s3-credentials: + interface: s3 containers: mongod: resource: mongodb-image @@ -53,3 +55,4 @@ storage: mongodb: type: filesystem location: /var/lib/mongodb + \ No newline at end of file diff --git a/src/charm.py b/src/charm.py index 31e3a90b7..4e0d08f19 100755 --- a/src/charm.py +++ b/src/charm.py @@ -9,6 +9,7 @@ from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer from charms.mongodb.v0.helpers import ( + build_unit_status, generate_keyfile, generate_password, get_create_user_cmd, @@ -19,6 +20,7 @@ MongoDBConnection, NotReadyError, ) +from charms.mongodb.v0.mongodb_backups import S3_RELATION, MongoDBBackups from charms.mongodb.v0.mongodb_provider import MongoDBProvider from charms.mongodb.v0.mongodb_tls import MongoDBTLS from charms.mongodb.v0.users import ( @@ -35,6 +37,7 @@ RelationDepartedEvent, StartEvent, StorageDetachingEvent, + UpdateStatusEvent, ) from ops.main import main from ops.model import ( @@ -45,7 +48,7 @@ Unit, WaitingStatus, ) -from ops.pebble import ExecError, Layer, PathError, ProtocolError +from ops.pebble import ExecError, Layer, PathError, ProtocolError, ServiceInfo from pymongo.errors import PyMongoError from tenacity import Retrying, before_log, retry, stop_after_attempt, wait_fixed @@ -63,7 +66,7 @@ def __init__(self, *args): self.framework.observe(self.on.mongod_pebble_ready, self._on_mongod_pebble_ready) self.framework.observe(self.on.start, self._on_start) - + self.framework.observe(self.on.update_status, self._on_update_status) self.framework.observe( self.on[Config.Relations.PEERS].relation_joined, self._relation_changes_handler ) @@ -85,6 +88,7 @@ def __init__(self, *args): self.client_relations = MongoDBProvider(self) self.tls = MongoDBTLS(self, Config.Relations.PEERS, Config.SUBSTRATE) + self.backups = MongoDBBackups(self, substrate=Config.SUBSTRATE) self.metrics_endpoint = MetricsEndpointProvider( self, refresh_event=self.on.start, jobs=Config.Monitoring.JOBS @@ -144,7 +148,28 @@ def monitor_config(self) -> MongoDBConfiguration: def backup_config(self) -> MongoDBConfiguration: """Generates a MongoDBConfiguration object for backup.""" self._check_or_set_user_password(BackupUser) - return self._get_mongodb_config_for_user(BackupUser, BackupUser.get_hosts()) + return self._get_mongodb_config_for_user( + BackupUser, [self.get_hostname_for_unit(self.unit)] + ) + + @property + def _mongod_layer(self) -> Layer: + """Returns a Pebble configuration layer for mongod.""" + layer_config = { + "summary": "mongod layer", + "description": "Pebble config layer for replicated mongod", + "services": { + "mongod": { + "override": "replace", + "summary": "mongod", + "command": "mongod " + get_mongod_args(self.mongodb_config), + "startup": "enabled", + "user": Config.UNIX_USER, + "group": Config.UNIX_GROUP, + } + }, + } + return Layer(layer_config) # type: ignore @property def _monitor_layer(self) -> Layer: @@ -167,19 +192,19 @@ def _monitor_layer(self) -> Layer: return Layer(layer_config) # type: ignore @property - def _mongod_layer(self) -> Layer: - """Returns a Pebble configuration layer for mongod.""" - layer_config = { - "summary": "mongod layer", - "description": "Pebble config layer for replicated mongod", + def backup_layer(self) -> Layer: + layer_config ={ + "summary": "pbm layer", + "description": "Pebble config layer for pbm", "services": { - "mongod": { + Config.Backup.SERVICE_NAME: { "override": "replace", - "summary": "mongod", - "command": "mongod " + get_mongod_args(self.mongodb_config), + "summary": "pbm", + "command": "pbm-agent", "startup": "enabled", "user": Config.UNIX_USER, "group": Config.UNIX_GROUP, + "environment": {"PBM_MONGODB_URI": self.backup_config.uri}, } }, } @@ -204,11 +229,13 @@ def app_peer_data(self) -> RelationDataContent: return relation.data[self.app] @property - def _db_initialised(self) -> bool: + def db_initialised(self) -> bool: + """Check if MongoDB is initialised.""" return "db_initialised" in self.app_peer_data - @_db_initialised.setter - def _db_initialised(self, value): + @db_initialised.setter + def db_initialised(self, value): + """Set the db_initialised flag.""" if isinstance(value, bool): self.app_peer_data["db_initialised"] = str(value) else: @@ -248,6 +275,7 @@ def _on_mongod_pebble_ready(self, event) -> None: # when a network cuts and the pod restarts - reconnect to the exporter self._connect_mongodb_exporter() + self._connect_pbm_agent() def _on_start(self, event) -> None: """Initialise MongoDB. @@ -285,6 +313,7 @@ def _on_start(self, event) -> None: return self._connect_mongodb_exporter() + self._connect_pbm_agent() self._initialise_replica_set(event) @@ -294,7 +323,7 @@ def _on_start(self, event) -> None: def _relation_changes_handler(self, event) -> None: """Handles different relation events and updates MongoDB replica set.""" self._connect_mongodb_exporter() - + self._connect_pbm_agent() if not self.unit.is_leader(): return @@ -302,7 +331,7 @@ def _relation_changes_handler(self, event) -> None: # This code runs on leader_elected event before mongod_pebble_ready self._generate_secrets() - if not self._db_initialised: + if not self.db_initialised: return with MongoDBConnection(self.mongodb_config) as mongo: @@ -369,6 +398,40 @@ def _on_storage_detaching(self, event: StorageDetachingEvent) -> None: except PyMongoError as e: logger.error("Failed to remove %s from replica set, error=%r", self.unit.name, e) + def _on_update_status(self, event: UpdateStatusEvent): + # no need to report on replica set status until initialised + if not self.db_initialised: + return + + # Cannot check more advanced MongoDB statuses if mongod hasn't started. + with MongoDBConnection(self.mongodb_config, "localhost", direct=True) as direct_mongo: + if not direct_mongo.is_ready: + self.unit.status = WaitingStatus("Waiting for MongoDB to start") + return + + # leader should periodically handle configuring the replica set. Incidents such as network + # cuts can lead to new IP addresses and therefore will require a reconfigure. Especially + # in the case that the leader a change in IP address it will not receive a relation event. + if self.unit.is_leader(): + self._relation_changes_handler(event) + + # update the units status based on it's replica set config and backup status. An error in + # the status of MongoDB takes precedence over pbm status. + mongodb_status = build_unit_status( + self.mongodb_config, self.get_hostname_for_unit(self.unit) + ) + pbm_status = self.backups._get_pbm_status() + if ( + not isinstance(mongodb_status, ActiveStatus) + or not self.model.get_relation( + S3_RELATION + ) # if s3 relation doesn't exist only report MongoDB status + or isinstance(pbm_status, ActiveStatus) # pbm is ready then report the MongoDB status + ): + self.unit.status = mongodb_status + else: + self.unit.status = pbm_status + # END: charm events # BEGIN: actions @@ -418,6 +481,9 @@ def _on_set_password(self, event: ActionEvent) -> None: self.set_secret("app", MongoDBUser.get_password_key_name_for_user(username), new_password) + if username == BackupUser.get_username(): + self._connect_pbm_agent() + if username == MonitorUser.get_username(): self._connect_mongodb_exporter() @@ -587,7 +653,7 @@ def _update_app_relation_data(self, database_users: Set[str]) -> None: def _initialise_replica_set(self, event: StartEvent) -> None: """Initialise replica set and create users.""" - if self._db_initialised: + if self.db_initialised: # The replica set should be initialised only once. Check should be # external (e.g., check initialisation inside peer relation). We # shouldn't rely on MongoDB response because the data directory @@ -619,7 +685,7 @@ def _initialise_replica_set(self, event: StartEvent) -> None: event.defer() return - self._db_initialised = True + self.db_initialised = True def _add_units_from_replica_set( self, event, mongo: MongoDBConnection, units_to_add: Set[str] @@ -681,6 +747,7 @@ def restart_mongod_service(self): container.replan() self._connect_mongodb_exporter() + self._connect_pbm_agent() def _push_keyfile_to_workload(self, container: Container) -> None: """Upload the keyFile to a workload container.""" @@ -770,6 +837,9 @@ def _connect_mongodb_exporter(self) -> None: if not container.can_connect(): return + if not self.db_initialised: + return + # must wait for leader to set URI before connecting if not self.get_secret("app", MonitorUser.get_password_key_name()): return @@ -779,9 +849,33 @@ def _connect_mongodb_exporter(self) -> None: # Restart changed services and start startup-enabled services. container.replan() + def _connect_pbm_agent(self) -> None: + """Updates URI for pbm-agent.""" + container = self.unit.get_container(Config.CONTAINER_NAME) + + if not container.can_connect(): + return + + if not self.db_initialised: + return + + # must wait for leader to set URI before any attempts to update are made + if not self.get_secret("app", BackupUser.get_password_key_name()): + return + + container.add_layer(Config.Backup.SERVICE_NAME, self.backup_layer, combine=True) + container.replan() + def _socket_exists(self, container) -> bool: return container.exists(Config.SOCKET_PATH) + def get_backup_service(self) -> ServiceInfo: + container = self.get_container() + return container.get_service(Config.Backup.SERVICE_NAME) + + def get_container(self) -> Container: + return self.unit.get_container(Config.CONTAINER_NAME) + # END: helper functions # BEGIN: static methods diff --git a/tests/integration/backup_tests/__init__.py b/tests/integration/backup_tests/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/backup_tests/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/backup_tests/helpers.py b/tests/integration/backup_tests/helpers.py new file mode 100644 index 000000000..b4cb47682 --- /dev/null +++ b/tests/integration/backup_tests/helpers.py @@ -0,0 +1,149 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import os + +import ops +from pymongo import MongoClient +from pytest_operator.plugin import OpsTest +from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed + +from ..ha_tests import helpers as ha_helpers + +S3_APP_NAME = "s3-integrator" +TIMEOUT = 10 * 60 + + +async def destroy_cluster(ops_test: OpsTest, cluster_name: str) -> None: + """Destroy the cluster and wait for its removal.""" + units = ops_test.model.applications[cluster_name].units + # best practice to scale down before removing the entire cluster. Wait for cluster to settle + # removing the next + for i in range(0, len(units[:-1])): + await units[i].remove() + await ops_test.model.block_until( + lambda: len(ops_test.model.applications[cluster_name].units) == len(units) - i - 1, + timeout=TIMEOUT, + ) + ops_test.model.wait_for_idle(apps=[cluster_name], status="active") + + # now that the cluster only has one unit left we can remove the application from Juju + await ops_test.model.applications[cluster_name].destroy() + + # verify there are no more units. + await ops_test.model.block_until( + lambda: cluster_name not in ops_test.model.applications, + timeout=TIMEOUT, + ) + + +async def create_and_verify_backup(ops_test: OpsTest) -> None: + """Creates and verifies that a backup was successfully created.""" + db_unit = await get_leader_unit(ops_test) + prev_backups = await count_logical_backups(db_unit) + action = await db_unit.run_action(action_name="create-backup") + backup = await action.wait() + assert backup.status == "completed", "Backup not started." + + # verify that backup was made on the bucket + try: + for attempt in Retrying(stop=stop_after_attempt(4), wait=wait_fixed(5)): + with attempt: + backups = await count_logical_backups(db_unit) + assert backups == prev_backups + 1, "Backup not created." + except RetryError: + assert backups == prev_backups + 1, "Backup not created." + + +async def get_leader_unit(ops_test: OpsTest, db_app_name=None) -> ops.model.Unit: + """Returns the leader unit of the database charm.""" + db_app_name = db_app_name or await app_name(ops_test) + for unit in ops_test.model.applications[db_app_name].units: + if await unit.is_leader_from_status(): + return unit + + +async def app_name(ops_test: OpsTest) -> str: + """Returns the name of the cluster running MongoDB. + + This is important since not all deployments of the MongoDB charm have the application name + "mongodb". + + Note: if multiple clusters are running MongoDB this will return the one first found. + """ + status = await ops_test.model.get_status() + for app in ops_test.model.applications: + # note that format of the charm field is not exactly "mongodb" but instead takes the form + # of `local:focal/mongodb-6` + if "mongodb" in status["applications"][app]["charm"]: + return app + + return None + + +async def count_logical_backups(db_unit: ops.model.Unit) -> int: + """Count the number of logical backups.""" + action = await db_unit.run_action(action_name="list-backups") + list_result = await action.wait() + list_result = list_result.results["backups"] + list_result = list_result.split("\n") + backups = 0 + for res in list_result: + backups += 1 if "logical" in res else 0 + + return backups + + +async def count_failed_backups(db_unit: ops.model.Unit) -> int: + """Count the number of failed backups.""" + action = await db_unit.run_action(action_name="list-backups") + list_result = await action.wait() + list_result = list_result.results["backups"] + list_result = list_result.split("\n") + failed_backups = 0 + for res in list_result: + failed_backups += 1 if "failed" in res else 0 + + return failed_backups + + +async def set_credentials(ops_test: OpsTest, cloud: str) -> None: + """Sets the s3 crednetials for the provided cloud, valid options are AWS or GCP.""" + # set access key and secret keys + access_key = os.environ.get(f"{cloud}_ACCESS_KEY", False) + secret_key = os.environ.get(f"{cloud}_SECRET_KEY", False) + assert access_key and secret_key, f"{cloud} access key and secret key not provided." + + s3_integrator_unit = ops_test.model.applications[S3_APP_NAME].units[0] + parameters = {"access-key": access_key, "secret-key": secret_key} + action = await s3_integrator_unit.run_action(action_name="sync-s3-credentials", **parameters) + await action.wait() + + +def is_relation_joined(ops_test: OpsTest, endpoint_one: str, endpoint_two: str) -> bool: + """Check if a relation is joined. + + Args: + ops_test: The ops test object passed into every test case + endpoint_one: The first endpoint of the relation + endpoint_two: The second endpoint of the relation + """ + for rel in ops_test.model.relations: + endpoints = [endpoint.name for endpoint in rel.endpoints] + if endpoint_one in endpoints and endpoint_two in endpoints: + return True + return False + + +async def insert_unwanted_data(ops_test: OpsTest) -> None: + """Inserts the data into the MongoDB cluster via primary replica.""" + app = await app_name(ops_test) + ip_addresses = [unit.public_address for unit in ops_test.model.applications[app].units] + primary = (await ha_helpers.replica_set_primary(ip_addresses, ops_test)).public_address + password = await ha_helpers.get_password(ops_test, app) + client = MongoClient(ha_helpers.unit_uri(primary, password, app), directConnection=True) + db = client["new-db"] + test_collection = db["test_collection"] + test_collection.insert_one({"unwanted_data": "bad data 1"}) + test_collection.insert_one({"unwanted_data": "bad data 2"}) + test_collection.insert_one({"unwanted_data": "bad data 3"}) + client.close() diff --git a/tests/integration/backup_tests/test_backups.py b/tests/integration/backup_tests/test_backups.py new file mode 100644 index 000000000..1436b047e --- /dev/null +++ b/tests/integration/backup_tests/test_backups.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import asyncio +import secrets +import string +import time + +import pytest +from pytest_operator.plugin import OpsTest +from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed + +from ..ha_tests import helpers as ha_helpers +from . import helpers + +S3_APP_NAME = "s3-integrator" +TIMEOUT = 15 * 60 +ENDPOINT = "s3-credentials" +NEW_CLUSTER = "new-mongodb" + + +@pytest.fixture() +async def continuous_writes_to_db(ops_test: OpsTest): + """Continuously writes to DB for the duration of the test.""" + await ha_helpers.start_continous_writes(ops_test, 1) + yield + await ha_helpers.stop_continous_writes(ops_test) + await ha_helpers.clear_db_writes(ops_test) + + +@pytest.fixture() +async def add_writes_to_db(ops_test: OpsTest): + """Adds writes to DB before test starts and clears writes at the end of the test.""" + await ha_helpers.start_continous_writes(ops_test, 1) + time.sleep(20) + await ha_helpers.stop_continous_writes(ops_test) + yield + await ha_helpers.clear_db_writes(ops_test) + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest) -> None: + """Build and deploy one unit of MongoDB.""" + # it is possible for users to provide their own cluster for testing. Hence check if there + # is a pre-existing cluster. + if not await helpers.app_name(ops_test): + db_charm = await ops_test.build_charm(".") + await ops_test.model.deploy(db_charm, num_units=3) + + # deploy the s3 integrator charm + await ops_test.model.deploy(S3_APP_NAME, channel="edge") + + await ops_test.model.wait_for_idle() + + +@pytest.mark.abort_on_fail +async def test_blocked_incorrect_creds(ops_test: OpsTest) -> None: + """Verifies that the charm goes into blocked status when s3 creds are incorrect.""" + db_app_name = await helpers.app_name(ops_test) + + # set incorrect s3 credentials + s3_integrator_unit = ops_test.model.applications[S3_APP_NAME].units[0] + parameters = {"access-key": "user", "secret-key": "doesnt-exist"} + action = await s3_integrator_unit.run_action(action_name="sync-s3-credentials", **parameters) + await action.wait() + + # relate after s3 becomes active add and wait for relation + await ops_test.model.wait_for_idle(apps=[S3_APP_NAME], status="active") + await ops_test.model.add_relation(S3_APP_NAME, db_app_name) + await ops_test.model.block_until( + lambda: helpers.is_relation_joined(ops_test, ENDPOINT, ENDPOINT) is True, + timeout=TIMEOUT, + ) + + # verify that Charmed MongoDB is blocked and reports incorrect credentials + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[S3_APP_NAME], status="active"), + ops_test.model.wait_for_idle(apps=[db_app_name], status="blocked", idle_period=20), + ) + db_unit = ops_test.model.applications[db_app_name].units[0] + + assert db_unit.workload_status_message == "s3 credentials are incorrect." + + +@pytest.mark.abort_on_fail +async def test_blocked_incorrect_conf(ops_test: OpsTest) -> None: + """Verifies that the charm goes into blocked status when s3 config options are incorrect.""" + db_app_name = await helpers.app_name(ops_test) + + # set correct AWS credentials for s3 storage but incorrect configs + await helpers.set_credentials(ops_test, cloud="AWS") + + # wait for both applications to be idle with the correct statuses + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[S3_APP_NAME], status="active"), + ops_test.model.wait_for_idle(apps=[db_app_name], status="blocked", idle_period=20), + ) + db_unit = ops_test.model.applications[db_app_name].units[0] + assert db_unit.workload_status_message == "s3 configurations are incompatible." + + +@pytest.mark.abort_on_fail +async def test_ready_correct_conf(ops_test: OpsTest) -> None: + """Verifies charm goes into active status when s3 config and creds options are correct.""" + db_app_name = await helpers.app_name(ops_test) + choices = string.ascii_letters + string.digits + unique_path = "".join([secrets.choice(choices) for _ in range(4)]) + configuration_parameters = { + "bucket": "data-charms-testing", + "path": f"mongodb-vm/test-{unique_path}", + "endpoint": "https://s3.amazonaws.com", + "region": "us-east-1", + } + + # apply new configuration options + await ops_test.model.applications[S3_APP_NAME].set_config(configuration_parameters) + + # after applying correct config options and creds the applications should both be active + await ops_test.model.wait_for_idle(apps=[S3_APP_NAME], status="active", timeout=TIMEOUT) + await ops_test.model.wait_for_idle( + apps=[db_app_name], status="active", timeout=TIMEOUT, idle_period=60 + ) + + +@pytest.mark.abort_on_fail +async def test_create_and_list_backups(ops_test: OpsTest) -> None: + db_unit = await helpers.get_leader_unit(ops_test) + + # verify backup list works + action = await db_unit.run_action(action_name="list-backups") + list_result = await action.wait() + backups = list_result.results["backups"] + assert backups, "backups not outputted" + + # verify backup is started + action = await db_unit.run_action(action_name="create-backup") + backup_result = await action.wait() + assert backup_result.results["backup-status"] == "backup started", "backup didn't start" + + # verify backup is present in the list of backups + # the action `create-backup` only confirms that the command was sent to the `pbm`. Creating a + # backup can take a lot of time so this function returns once the command was successfully + # sent to pbm. Therefore we should retry listing the backup several times + try: + for attempt in Retrying(stop=stop_after_delay(20), wait=wait_fixed(3)): + with attempt: + backups = await helpers.count_logical_backups(db_unit) + assert backups == 1 + except RetryError: + assert backups == 1, "Backup not created." + + +@pytest.mark.abort_on_fail +async def test_multi_backup(ops_test: OpsTest, continuous_writes_to_db) -> None: + """With writes in the DB test creating a backup while another one is running. + + Note that before creating the second backup we change the bucket and change the s3 storage + from AWS to GCP. This test verifies that the first backup in AWS is made, the second backup + in GCP is made, and that before the second backup is made that pbm correctly resyncs. + """ + db_app_name = await helpers.app_name(ops_test) + db_unit = await helpers.get_leader_unit(ops_test) + + # create first backup once ready + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + action = await db_unit.run_action(action_name="create-backup") + first_backup = await action.wait() + assert first_backup.status == "completed", "First backup not started." + + # while first backup is running change access key, secret keys, and bucket name + # for GCP + await helpers.set_credentials(ops_test, cloud="GCP") + + # change to GCP configs and wait for PBM to resync + configuration_parameters = { + "bucket": "data-charms-testing", + "endpoint": "https://storage.googleapis.com", + "region": "", + } + await ops_test.model.applications[S3_APP_NAME].set_config(configuration_parameters) + + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # create a backup as soon as possible. might not be immediately possible since only one backup + # can happen at a time. + try: + for attempt in Retrying(stop=stop_after_delay(40), wait=wait_fixed(5)): + with attempt: + action = await db_unit.run_action(action_name="create-backup") + second_backup = await action.wait() + assert second_backup.status == "completed" + except RetryError: + assert second_backup.status == "completed", "Second backup not started." + + # the action `create-backup` only confirms that the command was sent to the `pbm`. Creating a + # backup can take a lot of time so this function returns once the command was successfully + # sent to pbm. Therefore before checking, wait for Charmed MongoDB to finish creating the + # backup + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # verify that backups was made in GCP bucket + try: + for attempt in Retrying(stop=stop_after_delay(4), wait=wait_fixed(5)): + with attempt: + backups = await helpers.count_logical_backups(db_unit) + assert backups == 1, "Backup not created in bucket on GCP." + except RetryError: + assert backups == 1, "Backup not created in first bucket on GCP." + + # set AWS credentials, set configs for s3 storage, and wait to resync + await helpers.set_credentials(ops_test, cloud="AWS") + configuration_parameters = { + "bucket": "data-charms-testing", + "region": "us-east-1", + "endpoint": "https://s3.amazonaws.com", + } + await ops_test.model.applications[S3_APP_NAME].set_config(configuration_parameters) + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # verify that backups was made on the AWS bucket + try: + for attempt in Retrying(stop=stop_after_delay(4), wait=wait_fixed(5)): + with attempt: + backups = await helpers.count_logical_backups(db_unit) + assert backups == 2, "Backup not created in bucket on AWS." + except RetryError: + assert backups == 2, "Backup not created in bucket on AWS." + + +@pytest.mark.abort_on_fail +async def test_restore(ops_test: OpsTest, add_writes_to_db) -> None: + """Simple backup tests that verifies that writes are correctly restored.""" + # count total writes + number_writes = await ha_helpers.count_writes(ops_test) + assert number_writes > 0, "no writes to backup" + + # create a backup in the AWS bucket + db_app_name = await helpers.app_name(ops_test) + db_unit = await helpers.get_leader_unit(ops_test) + prev_backups = await helpers.count_logical_backups(db_unit) + action = await db_unit.run_action(action_name="create-backup") + first_backup = await action.wait() + assert first_backup.status == "completed", "First backup not started." + + # verify that backup was made on the bucket + try: + for attempt in Retrying(stop=stop_after_delay(4), wait=wait_fixed(5)): + with attempt: + backups = await helpers.count_logical_backups(db_unit) + assert backups == prev_backups + 1, "Backup not created." + except RetryError: + assert backups == prev_backups + 1, "Backup not created." + + # add writes to be cleared after restoring the backup. Note these are written to the same + # collection that was backed up. + await helpers.insert_unwanted_data(ops_test) + new_number_of_writes = await ha_helpers.count_writes(ops_test) + assert new_number_of_writes > number_writes, "No writes to be cleared after restoring." + + # find most recent backup id and restore + action = await db_unit.run_action(action_name="list-backups") + list_result = await action.wait() + list_result = list_result.results["backups"] + most_recent_backup = list_result.split("\n")[-1] + backup_id = most_recent_backup.split()[0] + action = await db_unit.run_action(action_name="restore", **{"backup-id": backup_id}) + restore = await action.wait() + assert restore.results["restore-status"] == "restore started", "restore not successful" + + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # verify all writes are present + try: + for attempt in Retrying(stop=stop_after_delay(4), wait=wait_fixed(20)): + with attempt: + number_writes_restored = await ha_helpers.count_writes(ops_test) + assert number_writes == number_writes_restored, "writes not correctly restored" + except RetryError: + assert number_writes == number_writes_restored, "writes not correctly restored" + + +@pytest.mark.parametrize("cloud_provider", ["AWS", "GCP"]) +async def test_restore_new_cluster(ops_test: OpsTest, add_writes_to_db, cloud_provider): + # configure test for the cloud provider + db_app_name = await helpers.app_name(ops_test) + await helpers.set_credentials(ops_test, cloud=cloud_provider) + if cloud_provider == "AWS": + configuration_parameters = { + "bucket": "data-charms-testing", + "region": "us-east-1", + "endpoint": "https://s3.amazonaws.com", + } + else: + configuration_parameters = { + "bucket": "data-charms-testing", + "endpoint": "https://storage.googleapis.com", + "region": "", + } + + await ops_test.model.applications[S3_APP_NAME].set_config(configuration_parameters) + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[S3_APP_NAME], status="active"), + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # create a backup + writes_in_old_cluster = await ha_helpers.count_writes(ops_test, db_app_name) + assert writes_in_old_cluster > 0, "old cluster has no writes." + await helpers.create_and_verify_backup(ops_test) + + # save old password, since after restoring we will need this password to authenticate. + old_password = await ha_helpers.get_password(ops_test, db_app_name) + + # deploy a new cluster with a different name + db_charm = await ops_test.build_charm(".") + await ops_test.model.deploy(db_charm, num_units=3, application_name=NEW_CLUSTER) + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[NEW_CLUSTER], status="active", idle_period=20), + ) + + db_unit = await helpers.get_leader_unit(ops_test, db_app_name=NEW_CLUSTER) + action = await db_unit.run_action("set-password", **{"password": old_password}) + action = await action.wait() + assert action.status == "completed" + + # relate to s3 - s3 has the necessary configurations + await ops_test.model.add_relation(S3_APP_NAME, NEW_CLUSTER) + await ops_test.model.block_until( + lambda: helpers.is_relation_joined(ops_test, ENDPOINT, ENDPOINT) is True, + timeout=TIMEOUT, + ) + + # wait for new cluster to sync + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[NEW_CLUSTER], status="active", idle_period=20), + ) + + # verify that the listed backups from the old cluster are not listed as failed. + assert ( + await helpers.count_failed_backups(db_unit) == 0 + ), "Backups from old cluster are listed as failed" + + # find most recent backup id and restore + action = await db_unit.run_action(action_name="list-backups") + list_result = await action.wait() + list_result = list_result.results["backups"] + most_recent_backup = list_result.split("\n")[-1] + backup_id = most_recent_backup.split()[0] + action = await db_unit.run_action(action_name="restore", **{"backup-id": backup_id}) + restore = await action.wait() + assert restore.results["restore-status"] == "restore started", "restore not successful" + + # verify all writes are present + try: + for attempt in Retrying(stop=stop_after_delay(4), wait=wait_fixed(20)): + with attempt: + writes_in_new_cluster = await ha_helpers.count_writes(ops_test, NEW_CLUSTER) + assert ( + writes_in_new_cluster == writes_in_old_cluster + ), "new cluster writes do not match old cluster writes after restore" + except RetryError: + assert ( + writes_in_new_cluster == writes_in_old_cluster + ), "new cluster writes do not match old cluster writes after restore" + + await helpers.destroy_cluster(ops_test, cluster_name=NEW_CLUSTER) + + +@pytest.mark.abort_on_fail +async def test_update_backup_password(ops_test: OpsTest) -> None: + """Verifies that after changing the backup password the pbm tool is updated and functional.""" + db_app_name = await helpers.app_name(ops_test) + db_unit = await helpers.get_leader_unit(ops_test) + + # wait for charm to be idle before setting password + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + parameters = {"username": "backup"} + action = await db_unit.run_action("set-password", **parameters) + action = await action.wait() + assert action.status == "completed", "failed to set backup password" + + # wait for charm to be idle after setting password + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[db_app_name], status="active", idle_period=20), + ) + + # verify we still have connection to pbm via creating a backup + action = await db_unit.run_action(action_name="create-backup") + backup_result = await action.wait() + assert backup_result.results["backup-status"] == "backup started", "backup didn't start" diff --git a/tests/unit/test_mongodb_backups.py b/tests/unit/test_mongodb_backups.py new file mode 100644 index 000000000..b54f59c83 --- /dev/null +++ b/tests/unit/test_mongodb_backups.py @@ -0,0 +1,729 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import unittest +from subprocess import CalledProcessError +from unittest import mock +from unittest.mock import patch + +import tenacity +from charms.mongodb.v0.mongodb_backups import ( + PBMBusyError, + ResyncError, + SetPBMConfigError, + stop_after_attempt, + wait_fixed, +) +from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus +from ops.testing import Harness + +from charm import MongodbOperatorCharm + +from .helpers import patch_network_get + +RELATION_NAME = "s3-credentials" + + +class TestMongoBackups(unittest.TestCase): + @patch_network_get(private_address="1.1.1.1") + def setUp(self): + self.harness = Harness(MongodbOperatorCharm) + self.harness.begin() + self.harness.add_relation("database-peers", "database-peers") + self.harness.set_leader(True) + self.charm = self.harness.charm + self.addCleanup(self.harness.cleanup) + + def test_current_pbm_op(self): + """Test if _current_pbm_op can identify the operation pbm is running.""" + action = self.harness.charm.backups._current_pbm_op( + "nothing\nCurrently running:\n====\nexpected action" + ) + self.assertEqual(action, "expected action") + + no_action = self.harness.charm.backups._current_pbm_op("pbm not started") + self.assertEqual(no_action, "") + + @patch("charm.snap.SnapCache") + def test_get_pbm_status_snap_not_present(self, snap): + """Tests that when the snap is not present pbm is in blocked state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = False + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_get_pbm_status_resync(self, snap, output): + """Tests that when pbm is resyncing that pbm is in waiting state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + output.return_value = b"Currently running:\n====\nResync op" + self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), WaitingStatus)) + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_get_pbm_status_running(self, snap, output): + """Tests that when pbm not running an op that pbm is in active state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + output.return_value = b"Currently running:\n====\n(none)" + self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), ActiveStatus)) + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_get_pbm_status_backup(self, snap, output): + """Tests that when pbm running a backup that pbm is in maintenance state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + output.return_value = b"Currently running:\n====\nSnapshot backup" + self.assertTrue( + isinstance(self.harness.charm.backups._get_pbm_status(), MaintenanceStatus) + ) + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_get_pbm_status_incorrect_cred(self, snap, output): + """Tests that when pbm has incorrect credentials that pbm is in blocked state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_get_pbm_status_incorrect_conf(self, snap, output): + """Tests that when pbm has incorrect configs that pbm is in blocked state.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=42, output=b"" + ) + self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) + + @patch("charm.subprocess.check_output") + @patch("charm.MongoDBBackups._get_pbm_status") + def test_verify_resync_config_error(self, _get_pbm_status, check_output): + """Tests that when pbm cannot perform the resync command it raises an error.""" + mock_snap = mock.Mock() + check_output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=42 + ) + + with self.assertRaises(CalledProcessError): + self.harness.charm.backups._resync_config_options(mock_snap) + + @patch("charm.subprocess.check_output") + def test_verify_resync_cred_error(self, check_output): + """Tests that when pbm cannot resync due to creds that it raises an error.""" + mock_snap = mock.Mock() + check_output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + with self.assertRaises(CalledProcessError): + self.harness.charm.backups._resync_config_options(mock_snap) + + @patch("charm.subprocess.check_output") + @patch("charm.MongoDBBackups._get_pbm_status") + def test_verify_resync_syncing(self, _get_pbm_status, check_output): + """Tests that when pbm needs more time to resync that it raises an error.""" + mock_snap = mock.Mock() + check_output.return_value = b"Currently running:\n====\nResync op" + + # disable retry + self.harness.charm.backups._wait_pbm_status.retry.retry = tenacity.retry_if_not_result( + lambda x: True + ) + + with self.assertRaises(ResyncError): + self.harness.charm.backups._resync_config_options(mock_snap) + + @patch("charms.mongodb.v0.mongodb_backups.wait_fixed") + @patch("charms.mongodb.v0.mongodb_backups.stop_after_attempt") + @patch("charm.MongoDBBackups._get_pbm_status") + def test_resync_config_options_failure(self, pbm_status, retry_stop, retry_wait): + """Verifies _resync_config_options raises an error when a resync cannot be performed.""" + pbm_status.return_value = MaintenanceStatus() + mock_snap = mock.Mock() + with self.assertRaises(PBMBusyError): + self.harness.charm.backups._resync_config_options(mock_snap) + + @patch("charm.subprocess.check_output") + @patch("charms.mongodb.v0.mongodb_backups.wait_fixed") + @patch("charms.mongodb.v0.mongodb_backups.stop_after_attempt") + @patch("charm.MongoDBBackups._get_pbm_status") + def test_resync_config_restart(self, pbm_status, retry_stop, retry_wait, check_output): + """Verifies _resync_config_options restarts that snap if alreaady resyncing.""" + retry_stop.return_value = stop_after_attempt(1) + retry_stop.return_value = wait_fixed(1) + pbm_status.return_value = WaitingStatus() + mock_snap = mock.Mock() + + with self.assertRaises(PBMBusyError): + self.harness.charm.backups._resync_config_options(mock_snap) + + mock_snap.restart.assert_called() + + @patch("charm.subprocess.check_output") + def test_set_config_options(self, check_output): + """Verifies _set_config_options failure raises SetPBMConfigError.""" + # the first check_output should succesd + check_output.side_effect = [ + None, + CalledProcessError( + cmd="charmed-mongodb.pbm config --set this_key=doesnt_exist", returncode=42 + ), + ] + + with self.assertRaises(SetPBMConfigError): + self.harness.charm.backups._set_config_options({"this_key": "doesnt_exist"}) + + def test_backup_without_rel(self): + """Verifies no backups are attempted without s3 relation.""" + action_event = mock.Mock() + action_event.params = {} + + self.harness.charm.backups._on_create_backup_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_backup_syncing(self, snap, output): + """Verifies backup is deferred if more time is needed to resync.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + output.return_value = b"Currently running:\n====\nResync op" + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_create_backup_action(action_event) + + action_event.defer.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_backup_running_backup(self, snap, output): + """Verifies backup is fails if another backup is already running.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + output.return_value = b"Currently running:\n====\nSnapshot backup" + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_create_backup_action(action_event) + + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_backup_wrong_cred(self, snap, output): + """Verifies backup is fails if the credentials are incorrect.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_create_backup_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.MongoDBBackups._get_pbm_status") + @patch("charm.snap.SnapCache") + def test_backup_failed(self, snap, pbm_status, output): + """Verifies backup is fails if the pbm command failed.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + pbm_status.return_value = ActiveStatus("") + + output.side_effect = CalledProcessError(cmd="charmed-mongodb.pbm backup", returncode=42) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_create_backup_action(action_event) + + action_event.fail.assert_called() + + def test_backup_list_without_rel(self): + """Verifies no backup lists are attempted without s3 relation.""" + action_event = mock.Mock() + action_event.params = {} + + self.harness.charm.backups._on_list_backups_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_backup_list_syncing(self, snap, output): + """Verifies backup list is deferred if more time is needed to resync.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + output.return_value = b"Currently running:\n====\nResync op" + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_list_backups_action(action_event) + + action_event.defer.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_backup_list_wrong_cred(self, snap, output): + """Verifies backup list fails with wrong credentials.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_list_backups_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.MongoDBBackups._get_pbm_status") + @patch("charm.snap.SnapCache") + def test_backup_list_failed(self, snap, pbm_status, output): + """Verifies backup list fails if the pbm command fails.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {} + pbm_status.return_value = ActiveStatus("") + + output.side_effect = CalledProcessError(cmd="charmed-mongodb.pbm list", returncode=42) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_list_backups_action(action_event) + + action_event.fail.assert_called() + + @patch("ops.framework.EventBase.defer") + def test_s3_credentials_no_db(self, defer): + """Verifies that when there is no DB that setting credentials is deferred.""" + del self.harness.charm.app_peer_data["db_initialised"] + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_called() + + @patch("ops.framework.EventBase.defer") + @patch("charm.snap.SnapCache") + def test_s3_credentials_no_snap(self, snap, defer): + """Verifies that when there is no DB that setting credentials is deferred.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = False + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_called() + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + def test_s3_credentials_set_pbm_failure(self, _set_config_options, snap): + """Test charm goes into blocked state when setting pbm configs fail.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + _set_config_options.side_effect = SetPBMConfigError + self.harness.charm.app_peer_data["db_initialised"] = "True" + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + @patch("charm.MongoDBBackups._resync_config_options") + @patch("ops.framework.EventBase.defer") + def test_s3_credentials_config_error(self, defer, resync, _set_config_options, snap): + """Test charm defers when more time is needed to sync pbm.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + resync.side_effect = SetPBMConfigError + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + @patch("charm.MongoDBBackups._resync_config_options") + @patch("ops.framework.EventBase.defer") + def test_s3_credentials_syncing(self, defer, resync, _set_config_options, snap): + """Test charm defers when more time is needed to sync pbm credentials.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + resync.side_effect = ResyncError + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_called() + self.assertTrue(isinstance(self.harness.charm.unit.status, WaitingStatus)) + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + @patch("charm.MongoDBBackups._resync_config_options") + @patch("ops.framework.EventBase.defer") + def test_s3_credentials_pbm_busy(self, defer, resync, _set_config_options, snap): + """Test charm defers when more time is needed to sync pbm.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + resync.side_effect = PBMBusyError + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_called() + self.assertTrue(isinstance(self.harness.charm.unit.status, WaitingStatus)) + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + @patch("charm.MongoDBBackups._resync_config_options") + @patch("ops.framework.EventBase.defer") + def test_s3_credentials_snap_start_error(self, defer, resync, _set_config_options, snap): + """Test charm defers when more time is needed to sync pbm.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + resync.side_effect = snap.SnapError + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_not_called() + self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) + + @patch_network_get(private_address="1.1.1.1") + @patch("charm.snap.SnapCache") + @patch("charm.MongoDBBackups._set_config_options") + @patch("charm.MongoDBBackups._resync_config_options") + @patch("ops.framework.EventBase.defer") + @patch("charm.subprocess.check_output") + def test_s3_credentials_pbm_error(self, output, defer, resync, _set_config_options, snap): + """Test charm defers when more time is needed to sync pbm.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + mock_pbm_snap.set = mock.Mock() + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + self.harness.charm.app_peer_data["db_initialised"] = "True" + resync.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + + # triggering s3 event with correct fields + mock_s3_info = mock.Mock() + mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} + self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info + relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.add_relation_unit(relation_id, "s3-integrator/0") + self.harness.update_relation_data( + relation_id, + "s3-integrator/0", + {"bucket": "hat"}, + ) + + defer.assert_not_called() + self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) + + @patch("charm.subprocess.check_output") + def test_generate_backup_list_output(self, check_output): + """Tests correct formation of backup list output. + + Specifically the spacing of the backups, the header, the backup order, and the backup + contents. + """ + # case 1: running backup is listed in error state + with open("tests/unit/data/pbm_status_duplicate_running.txt") as f: + output_contents = f.readlines() + output_contents = "".join(output_contents) + + check_output.return_value = output_contents.encode("utf-8") + formatted_output = self.harness.charm.backups._generate_backup_list_output() + formatted_output = formatted_output.split("\n") + header = formatted_output[0] + self.assertEqual(header, "backup-id | backup-type | backup-status") + divider = formatted_output[1] + self.assertEqual(divider, "-" * len(header)) + eariest_backup = formatted_output[2] + self.assertEqual( + eariest_backup, + "1900-02-14T13:59:14Z | physical | failed: internet not invented yet", + ) + failed_backup = formatted_output[3] + self.assertEqual(failed_backup, "2000-02-14T14:09:43Z | logical | finished") + inprogress_backup = formatted_output[4] + self.assertEqual(inprogress_backup, "2023-02-14T17:06:38Z | logical | in progress") + + # case 2: running backup is not listed in error state + with open("tests/unit/data/pbm_status.txt") as f: + output_contents = f.readlines() + output_contents = "".join(output_contents) + + check_output.return_value = output_contents.encode("utf-8") + formatted_output = self.harness.charm.backups._generate_backup_list_output() + formatted_output = formatted_output.split("\n") + header = formatted_output[0] + self.assertEqual(header, "backup-id | backup-type | backup-status") + divider = formatted_output[1] + self.assertEqual( + divider, "-" * len("backup-id | backup-type | backup-status") + ) + eariest_backup = formatted_output[2] + self.assertEqual( + eariest_backup, + "1900-02-14T13:59:14Z | physical | failed: internet not invented yet", + ) + failed_backup = formatted_output[3] + self.assertEqual(failed_backup, "2000-02-14T14:09:43Z | logical | finished") + inprogress_backup = formatted_output[4] + self.assertEqual(inprogress_backup, "2023-02-14T17:06:38Z | logical | in progress") + + def test_restore_without_rel(self): + """Verifies no restores are attempted without s3 relation.""" + action_event = mock.Mock() + action_event.params = {"backup-id": "back-me-up"} + + self.harness.charm.backups._on_restore_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_restore_syncing(self, snap, output): + """Verifies restore is deferred if more time is needed to resync.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {"backup-id": "back-me-up"} + output.return_value = b"Currently running:\n====\nResync op" + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_restore_action(action_event) + + action_event.defer.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_restore_running_backup(self, snap, output): + """Verifies restore is fails if another backup is already running.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {"backup-id": "back-me-up"} + output.return_value = b"Currently running:\n====\nSnapshot backup" + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_restore_action(action_event) + + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.snap.SnapCache") + def test_restore_wrong_cred(self, snap, output): + """Verifies restore is fails if the credentials are incorrect.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {"backup-id": "back-me-up"} + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" + ) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_restore_action(action_event) + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + @patch("charm.MongoDBBackups._get_pbm_status") + @patch("charm.snap.SnapCache") + def test_restore_failed(self, snap, pbm_status, output): + """Verifies restore is fails if the pbm command failed.""" + mock_pbm_snap = mock.Mock() + mock_pbm_snap.present = True + snap.return_value = {"charmed-mongodb": mock_pbm_snap} + + action_event = mock.Mock() + action_event.params = {"backup-id": "back-me-up"} + pbm_status.return_value = ActiveStatus("") + + output.side_effect = CalledProcessError( + cmd="charmed-mongodb.pbm backup", returncode=42, output=b"failed" + ) + + self.harness.add_relation(RELATION_NAME, "s3-integrator") + self.harness.charm.backups._on_restore_action(action_event) + + action_event.fail.assert_called() + + @patch("charm.subprocess.check_output") + def test_remap_replicaset_no_backup(self, check_output): + """Test verifies that no remapping is given if the backup_id doesn't exist.""" + with open("tests/unit/data/pbm_status.txt") as f: + output_contents = f.readlines() + output_contents = "".join(output_contents) + + check_output.return_value = output_contents.encode("utf-8") + remap = self.harness.charm.backups._remap_replicaset("this-id-doesnt-exist") + self.assertEqual(remap, "") + + @patch("charm.subprocess.check_output") + def test_remap_replicaset_no_remap_necessary(self, check_output): + """Test verifies that no remapping is given if no remapping is necessary.""" + with open("tests/unit/data/pbm_status_error_remap.txt") as f: + output_contents = f.readlines() + output_contents = "".join(output_contents) + + check_output.return_value = output_contents.encode("utf-8") + + # first case is that the backup is not in the error state + remap = self.harness.charm.backups._remap_replicaset("2000-02-14T14:09:43Z") + self.assertEqual(remap, "") + + # second case is that the backup has an error not related to remapping + remap = self.harness.charm.backups._remap_replicaset("1900-02-14T13:59:14Z") + self.assertEqual(remap, "") + + # third case is that the backup has two errors one related to remapping and another + # related to something else + remap = self.harness.charm.backups._remap_replicaset("2001-02-14T13:59:14Z") + self.assertEqual(remap, "") + + @patch("charm.subprocess.check_output") + def test_remap_replicaset_remap_necessary(self, check_output): + """Test verifies that remapping is provided and correct when necessary.""" + with open("tests/unit/data/pbm_status_error_remap.txt") as f: + output_contents = f.readlines() + output_contents = "".join(output_contents) + + check_output.return_value = output_contents.encode("utf-8") + self.harness.charm.app.name = "current-app-name" + + # first case is that the backup is not in the error state + remap = self.harness.charm.backups._remap_replicaset("2002-02-14T13:59:14Z") + self.assertEqual(remap, "--replset-remapping current-app-name=old-cluster-name")