From f471ebf76306872879b3ef544349c844479636ab Mon Sep 17 00:00:00 2001 From: Jozef Kralik Date: Mon, 17 Jun 2024 15:28:03 +0000 Subject: [PATCH] WIP --- .../disaster-recovery-replica-set.md | 420 ++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 content/en/docs/tutorials/disaster-recovery-replica-set.md diff --git a/content/en/docs/tutorials/disaster-recovery-replica-set.md b/content/en/docs/tutorials/disaster-recovery-replica-set.md new file mode 100644 index 00000000..2f0d10b0 --- /dev/null +++ b/content/en/docs/tutorials/disaster-recovery-replica-set.md @@ -0,0 +1,420 @@ +--- +title: 'Disaster Recovery via Replica Set' +description: 'How to Disaster Recovery with Replica Set?' +date: '2024-06-17' +categories: [architecture, d2c, provisioning, disaster-recovery] +keywords: [architecture, d2c, provisioning, disaster-recovery] +weight: 15 +--- + +plgd-hub helm charts support disaster recovery via Mongodb replica set, because the source of truth is stored in the Mongodb database. In tutorial we have two clusters priamry and standby. Each of them use 3 root CA certificates: + +- `external CA certificate pair` used for public API (coap, https, grpc) and it is same for both clusters. +- `internal CA certificate pair` used for plgd services to communicate with each other, the MongoDB and NATs. Each cluster has its own internal CA certificate. +- `storage CA certificate pair` used for MongoDB. Each cluster has its own storage CA certificate. + +Then we `authorization CA certificate` used to communicate with OAuth2 authorization server. In this tutorial `mock-oauth-server` and his certificate is signed by `external CA certificate pair`. So we have only one `authorization CA certificate` for both clusters which is `external CA certificate`. + +The target is that the only mongo DBs from primary and standby clusters are able to communicate with each other. But plgd services are able connect to only mongoDB in the same cluster. All APIs is will available on root domain `primary.plgd.cloud` for primary cluster and `standby.plgd.cloud` for standby cluster. Also mongo members are exposed via LoadBalancer service type and each member need to have own DNS name. + +| For primary cluster we have | For standby cluster we have | +| `mongodb-0.primary.plgd.cloud` | `mongodb-0.standby.plgd.cloud` | +| `mongodb-1.primary.plgd.cloud` | `mongodb-1.standby.plgd.cloud` | +| `mongodb-2.primary.plgd.cloud` | `mongodb-2.standby.plgd.cloud` | +| `mongodb.primary.plgd.cloud` | `----` | + +The `mongodb.primary.plgd.cloud` is used for external access to the MongoDB replica set for the standby cluster. This DNS record is alias to all members of the primary cluster. + +This DNS needs to be resolved to the external IP address of the LoadBalancer. The external IP address of the LoadBalancer is used to connect to the MongoDB replica set from the other cluster. For that you can [external-dns](https://github.com/kubernetes-sigs/external-dns/) tool to create DNS records. + +{{< note >}} + +It is also recommended to setup firewall between clusters with filtering source IP addresses to the MongoDB to mitigate DDOS attack. The default port for MongoDB is 27017. Or use VPN to interconnect clusters. + +{{< /note >}} + +## Creating certificates + +To create certificates you can use `cert-tool` docker image. The `cert-tool` is a tool for generating root CA certificates for plgd services. + +1. Create external CA certificate pair same for both clusters. + + ```bash + mkdir -p .tmp/certs/external + docker run \ + --rm -v $(pwd)/.tmp/certs/external:/certs \ + --user $(id -u):$(id -g) \ + ghcr.io/plgd-dev/hub/cert-tool:vnext \ + --cmd.generateRootCA --outCert=/certs/tls.crt --outKey=/certs/tls.key \ + --cert.subject.cn=external.root.ca --cert.validFor=876000h + ``` + +2. Create internal CA certificate pair for primary cluster. + + ```bash + mkdir -p .tmp/primary/certs/internal + docker run \ + --rm -v $(pwd)/.tmp/primary/certs/internal:/certs \ + --user $(id -u):$(id -g) \ + ghcr.io/plgd-dev/hub/cert-tool:vnext \ + --cmd.generateRootCA --outCert=/certs/tls.crt --outKey=/certs/tls.key \ + --cert.subject.cn=primary.internal.root.ca --cert.validFor=876000h + ``` + +3. Create storage CA certificate pair for primary cluster. + + ```bash + mkdir -p .tmp/primary/certs/storage + docker run \ + --rm -v $(pwd)/.tmp/primary/certs/storage:/certs \ + --user $(id -u):$(id -g) \ + ghcr.io/plgd-dev/hub/cert-tool:vnext \ + --cmd.generateRootCA --outCert=/certs/tls.crt --outKey=/certs/tls.key \ + --cert.subject.cn=primary.storage.root.ca --cert.validFor=876000h + ``` + +4. Create internal CA certificate pair for standby cluster. + + ```bash + mkdir -p .tmp/standby/certs/internal + docker run \ + --rm -v $(pwd)/.tmp/standby/certs/internal:/certs \ + --user $(id -u):$(id -g) \ + ghcr.io/plgd-dev/hub/cert-tool:vnext \ + --cmd.generateRootCA --outCert=/certs/tls.crt --outKey=/certs/tls.key \ + --cert.subject.cn=standby.internal.root.ca --cert.validFor=876000h + ``` + +5. Create storage CA certificate pair for standby cluster. + + ```bash + mkdir -p .tmp/standby/certs/storage + docker run \ + --rm -v $(pwd)/.tmp/standby/certs/storage:/certs \ + --user $(id -u):$(id -g) \ + ghcr.io/plgd-dev/hub/cert-tool:vnext \ + --cmd.generateRootCA --outCert=/certs/tls.crt --outKey=/certs/tls.key \ + --cert.subject.cn=standby.storage.root.ca --cert.validFor=876000h + ``` + +## Setup cert-manager on primary + +It is expected that you have installed cert-manager. + +1. Create external tls secret for issuing certificates. + + ```bash + kubectl -n cert-manager create secret tls external-plgd-ca-tls \ + --cert=.tmp/certs/external/tls.crt \ + --key=.tmp/certs/external/tls.key \ + ``` + +2. Create cluster issuer which points to external-plgd-ca-tls. + + ```bash + cat < values.yaml + +global: + domain: "primary.plgd.cloud" + hubId: "d03a1bb4-0a77-428c-b78c-1c46efe6a38e" + ownerClaim: "https://plgd.dev/owner" + standby: false + extraCAPool: + authorization: | + -----BEGIN CERTIFICATE----- + MIIBeTCCAR+gAwIBAgIRANdOwU9UTbmxq1+MWw08vQ8wCgYIKoZIzj0EAwIwGzEZ + MBcGA1UEAxMQZXh0ZXJuYWwucm9vdC5jYTAgFw0yNDA2MTAxMjQ2NDBaGA8yMTIz + MDEwMzEyNDY0MFowGzEZMBcGA1UEAxMQZXh0ZXJuYWwucm9vdC5jYTBZMBMGByqG + SM49AgEGCCqGSM49AwEHA0IABNUSjdul7Gs4C6wTvRO3x6ttNh000XRqfpInWNt4 + 6jRGEolYu0VWOKG0tqZRgamMjPXPn6lELMkkoFzYgkhOYJyjQjBAMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTJgVg8N9U4QihghYQn + 0BCdF81MFDAKBggqhkjOPQQDAgNIADBFAiEA8CxcZ1rnxYtrHoYxDPvyxZjJyJ0T + w6dVsDd8XGOOH2ICIGLeQnaLgm8w2pM5TCAOlLqzbJmcUX0CQXDh9WxhjVAt + -----END CERTIFICATE----- + internal: | + -----BEGIN CERTIFICATE----- + MIIBijCCAS+gAwIBAgIRAJelljcC2UT8f6WhBaGpKiAwCgYIKoZIzj0EAwIwIzEh + MB8GA1UEAxMYcHJpbWFyeS5pbnRlcm5hbC5yb290LmNhMCAXDTI0MDYxMDEyNDYy + N1oYDzIxMjMwMTAzMTI0NjI3WjAjMSEwHwYDVQQDExhwcmltYXJ5LmludGVybmFs + LnJvb3QuY2EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATYtu+gktujsTmsEuTL + f9GZCXKFy5cpp5QhdYG9HEs4+oGLF2Gvpa5NnaYev177iDlEqJ9i05xl61DnPUmb + 7YHEo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E + FgQUPC3RWYDFJpJk8J1aWJlkh2AnfAowCgYIKoZIzj0EAwIDSQAwRgIhAOjTwAGj + ePfuE4UUuY1LQpzIb+c9fpke0bqcI805FdS3AiEAgsihXQpPZbqU78JCU9RKYTIz + wXkHs5ZLpOdIdpgWPMU= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBhjCCAS2gAwIBAgIRALS5d3qDbhwYR33p49rCBZ4wCgYIKoZIzj0EAwIwIjEg + MB4GA1UEAxMXcHJpbWFyeS5zdG9yYWdlLnJvb3QuY2EwIBcNMjQwNjEwMTI0NzAx + WhgPMjEyMzAxMDMxMjQ3MDFaMCIxIDAeBgNVBAMTF3ByaW1hcnkuc3RvcmFnZS5y + b290LmNhMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEH+1E/axN9h0lq0K3L/fW + eSlQ/5GS4525woUVMnvXIzkeWn7w2vrBpJr+zT/09gmFT5/suv8DW49zAesXZpvI + hKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE + FEfkvstDJQzFzXCTiRxy3FlI1pE5MAoGCCqGSM49BAMCA0cAMEQCIBFjdbTcYhTj + 9h34g/iH0ZTX5AajN487cZBqC6z9+4V3AiB4KhIfLGH9l1VgLm0oPdr/fv6tkXqx + sF1RLi26Eu5acg== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBeTCCAR+gAwIBAgIRANdOwU9UTbmxq1+MWw08vQ8wCgYIKoZIzj0EAwIwGzEZ + MBcGA1UEAxMQZXh0ZXJuYWwucm9vdC5jYTAgFw0yNDA2MTAxMjQ2NDBaGA8yMTIz + MDEwMzEyNDY0MFowGzEZMBcGA1UEAxMQZXh0ZXJuYWwucm9vdC5jYTBZMBMGByqG + SM49AgEGCCqGSM49AwEHA0IABNUSjdul7Gs4C6wTvRO3x6ttNh000XRqfpInWNt4 + 6jRGEolYu0VWOKG0tqZRgamMjPXPn6lELMkkoFzYgkhOYJyjQjBAMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTJgVg8N9U4QihghYQn + 0BCdF81MFDAKBggqhkjOPQQDAgNIADBFAiEA8CxcZ1rnxYtrHoYxDPvyxZjJyJ0T + w6dVsDd8XGOOH2ICIGLeQnaLgm8w2pM5TCAOlLqzbJmcUX0CQXDh9WxhjVAt + -----END CERTIFICATE----- + storage: | + -----BEGIN CERTIFICATE----- + MIIBhjCCAS2gAwIBAgIRALS5d3qDbhwYR33p49rCBZ4wCgYIKoZIzj0EAwIwIjEg + MB4GA1UEAxMXcHJpbWFyeS5zdG9yYWdlLnJvb3QuY2EwIBcNMjQwNjEwMTI0NzAx + WhgPMjEyMzAxMDMxMjQ3MDFaMCIxIDAeBgNVBAMTF3ByaW1hcnkuc3RvcmFnZS5y + b290LmNhMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEH+1E/axN9h0lq0K3L/fW + eSlQ/5GS4525woUVMnvXIzkeWn7w2vrBpJr+zT/09gmFT5/suv8DW49zAesXZpvI + hKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE + FEfkvstDJQzFzXCTiRxy3FlI1pE5MAoGCCqGSM49BAMCA0cAMEQCIBFjdbTcYhTj + 9h34g/iH0ZTX5AajN487cZBqC6z9+4V3AiB4KhIfLGH9l1VgLm0oPdr/fv6tkXqx + sF1RLi26Eu5acg== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBijCCAS+gAwIBAgIRAJelljcC2UT8f6WhBaGpKiAwCgYIKoZIzj0EAwIwIzEh + MB8GA1UEAxMYcHJpbWFyeS5pbnRlcm5hbC5yb290LmNhMCAXDTI0MDYxMDEyNDYy + N1oYDzIxMjMwMTAzMTI0NjI3WjAjMSEwHwYDVQQDExhwcmltYXJ5LmludGVybmFs + LnJvb3QuY2EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATYtu+gktujsTmsEuTL + f9GZCXKFy5cpp5QhdYG9HEs4+oGLF2Gvpa5NnaYev177iDlEqJ9i05xl61DnPUmb + 7YHEo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E + FgQUPC3RWYDFJpJk8J1aWJlkh2AnfAowCgYIKoZIzj0EAwIDSQAwRgIhAOjTwAGj + ePfuE4UUuY1LQpzIb+c9fpke0bqcI805FdS3AiEAgsihXQpPZbqU78JCU9RKYTIz + wXkHs5ZLpOdIdpgWPMU= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBhjCCAS2gAwIBAgIRANKZ/Gq5LVtgKp73wxV3/1wwCgYIKoZIzj0EAwIwIjEg + MB4GA1UEAxMXc3RhbmRieS5zdG9yYWdlLnJvb3QuY2EwIBcNMjQwNjEwMTMwNTQ4 + WhgPMjEyMzAxMDMxMzA1NDhaMCIxIDAeBgNVBAMTF3N0YW5kYnkuc3RvcmFnZS5y + b290LmNhMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9aK0VKj8uC1OiC6T4ruc + Qt0sxOCyF47x2ZQq7fLiytFKIcCXHvlcyIifNwcAf94bOc7J9s1PSO9gAGBHE9cH + WaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE + FH6/kyJT6cUuUlUH9ITBtsy4U9bzMAoGCCqGSM49BAMCA0cAMEQCIClqseeCh8kF + eMq/FZ0krpYsA5qwNu1paN+aKSVok8QLAiBGSeuAW/TDXo1zb6JU/MRIfIGu9nwf + bJez1WkWlc8bqw== + -----END CERTIFICATE----- +mockoauthserver: + enabled: true + oauth: + - name: "plgd.dps" + clientID: "test" + clientSecret: "test" + grantType: "clientCredentials" + redirectURL: "https://primary.plgd.cloud/things" + scopes: ['openid'] + - name: "plgd.web" + clientID: "test" + clientSecret: "test" + redirectURL: "https://primary.plgd.cloud/things" + scopes: ['openid'] + useInUi: true +mongodb: + tls: + extraDnsNames: + - 'mongodb.primary.plgd.cloud' + externalAccess: + enabled: true + service: + type: LoadBalancer + publicNames: + - 'mongodb-0.primary.plgd.cloud' + - 'mongodb-1.primary.plgd.cloud' + - 'mongodb-2.primary.plgd.cloud' + +certmanager: + storage: + issuer: + kind: ClusterIssuer + name: storage-plgd-ca-issuer + internal: + issuer: + kind: ClusterIssuer + name: internal-plgd-ca-issuer + default: + ca: + issuerRef: + kind: ClusterIssuer + name: external-plgd-ca-issuer +httpgateway: + apiDomain: "primary.plgd.cloud" +grpcgateway: + domain: "primary.plgd.cloud" +certificateauthority: + domain: "primary.plgd.cloud" +coapgateway: + service: + type: NodePort + nodePort: 15684 +resourcedirectory: + publicConfiguration: + coapGateway: "coaps+tcp://primary.plgd.cloud:15684" +deviceProvisioningService: + apiDomain: primary.plgd.cloud + service: + type: NodePort + image: + dockerConfigSecret: | + { + "auths": { + "ghcr.io": { + "auth": "" + } + } + } + enrollmentGroups: + - id: "5db6ccde-05e1-480b-a522-c1591ad7dfd2" + owner: "1" + attestationMechanism: + x509: + certificateChain: |- + + hub: + coapGateway: "primary.plgd.cloud:15684" + certificateAuthority: + grpc: + address: "primary.plgd.cloud:443" + authorization: + provider: + name: "plgd.dps" + clientId: "test" + clientSecret: "test" + audience: "https://primary.plgd.cloud" + scopes: ["openid"] +EOF +helm upgrade -i -n plgd --create-namespace -f values.yaml hub plgd/plgd-hub +``` \ No newline at end of file