Upgrade kube-prometheus-stack helm chart (#2472)

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/monitoring/main.tf

@@ -3,16 +3,28 @@ resource "random_password" "grafana_admin_password" {
   special = false
 }
 
+resource "kubernetes_secret" "grafana_oauth_secret" {
+  metadata {
+    name      = "grafana-oauth-secret"
+    namespace = var.namespace
+  }
+
+  data = {
+    "grafana-oauth-client-id"     = module.grafana-client-id.config.client_id
+    "grafana-oauth-client-secret" = module.grafana-client-id.config.client_secret
+  }
+}
+
 resource "helm_release" "prometheus-grafana" {
   name       = "nebari"
   namespace  = var.namespace
   repository = "https://prometheus-community.github.io/helm-charts"
   chart      = "kube-prometheus-stack"
-  version    = "30.1.0"
+  version    = "58.4.0"
 
   values = concat([
     file("${path.module}/values.yaml"),
-    # https://github.com/prometheus-community/helm-charts/blob/kube-prometheus-stack-30.1.0/charts/kube-prometheus-stack/values.yaml
+    # https://github.com/prometheus-community/helm-charts/blob/kube-prometheus-stack-58.4.0/charts/kube-prometheus-stack/values.yaml
     jsonencode({
       alertmanager = {
         alertmanagerSpec = {
@@ -206,6 +218,8 @@ resource "helm_release" "prometheus-grafana" {
           }
         }
 
+        envFromSecret = kubernetes_secret.grafana_oauth_secret.metadata[0].name
+
         "grafana.ini" : {
           server = {
             protocol            = "http"
@@ -222,8 +236,8 @@ resource "helm_release" "prometheus-grafana" {
             enabled                  = "true"
             name                     = "Login Keycloak"
             allow_sign_up            = "true"
-            client_id                = module.grafana-client-id.config.client_id
-            client_secret            = module.grafana-client-id.config.client_secret
+            client_id                = "$__env{grafana-oauth-client-id}"
+            client_secret            = "$__env{grafana-oauth-client-secret}"
             scopes                   = "profile"
             auth_url                 = module.grafana-client-id.config.authentication_url
             token_url                = module.grafana-client-id.config.token_url

src/_nebari/upgrade.py

@@ -791,6 +791,56 @@ class Upgrade_2024_6_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
+        # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled
+        if config.get("monitoring", {}).get("enabled", True):
+            rich.print(
+                "\n ⚠️  Warning ⚠️"
+                "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]"
+                "\n-> [red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]"
+            )
+
+            # We're upgrading from version 30.1.0 to 58.4.0. This is a major upgrade and requires manual intervention.
+            # See https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/README.md#upgrading-chart
+            # for more information on why the following commands are necessary.
+            commands = textwrap.dedent(
+                f"""
+                [cyan bold]
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+                kubectl delete daemonset -l app=prometheus-node-exporter --namespace {config['namespace']}
+                [/cyan bold]
+            """
+            )
+
+            # By default, rich wraps lines by splitting them into multiple lines. This is
+            # far from ideal, as users copy-pasting the commands will get errors when running them.
+            # To avoid this, we use a rich console with a larger width to print the entire commands
+            # and let the terminal wrap them if needed.
+            Prompt.ask("Hit enter to show the commands")
+            console = rich.console.Console(width=220)
+            console.print(commands)
+
+            Prompt.ask("Hit enter to continue")
+            continue_ = Prompt.ask(
+                "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?",
+                choices=["y", "N"],
+                default="N",
+            )
+            if not continue_ == "y":
+                rich.print(
+                    f"[red bold]You must back up your custom dashboards (if necessary), delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]"
+                )
+                exit()
+
+        # Prompt users to upgrade to the new default node groups for GCP
         if (provider := config.get("provider", "")) == ProviderEnum.gcp.value:
             provider_full_name = provider_enum_name_map[provider]
             if not config.get(provider_full_name, {}).get("node_groups", {}):

tests/tests_e2e/cypress.config.js

@@ -0,0 +1,12 @@
+const { defineConfig } = require('cypress')
+
+module.exports = defineConfig({
+  video: true,
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.js')(on, config)
+    },
+  },
+})

tests/tests_e2e/cypress/integration/main.js → tests/tests_e2e/cypress/e2e/main.cy.js

@@ -87,7 +87,7 @@ describe('First Test', () => {
 
       cy.visit('/monitoring/dashboards');
 
-      cy.get('div.page-header h1', { timeout: 20000 }).should('contain', 'Dashboards');
+      cy.get('div#pageContent h1', { timeout: 20000 }).should('contain', 'Dashboards');
 
       // Visit Keycloak User Profile