Merge branch 'main' into feature/cdb-schemas

AMRC-FactoryPlus · Oct 18, 2024 · 0a57a97 · 0a57a97
2 parents 4cfc23e + 6ea1269
commit 0a57a97
Show file tree

Hide file tree

Showing 58 changed files with 7,502 additions and 65 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -165,7 +165,9 @@ jobs:
           - acs-service-setup
           - acs-visualiser
           - edge-helm-charts
-          - influxdb-sparkplug-ingester
+          - historian-sparkplug
+          - historian-uns
+          - uns-ingester-sparkplug
     permissions:
       contents: read
       packages: write

diff --git a/ReleaseNotes.md b/ReleaseNotes.md
@@ -8,6 +8,24 @@ chronological order.
 These changes have not been released yet, but are likely to appear in
 the next release.
 
+## v3.4.0
+
+### Unified Namespace & Historian
+This release of ACS enables a true Unified Namespace (UNS). The UNS is a
+single point of truth for all data collected by ACS in human-readable
+format. The UNS is "fed" by ingesters, which take channels of data (in
+this case, Sparkplug), and publishes the human-readable content to
+`UNS/v1`. In the future additional ingesters may be added to ACS.
+
+In addition to the Sparkplug ingester, this release features a UNS 
+historian, which persists the UNS data to the same InfluxDB 
+database used by the legacy Sparkplug historian. **By default, the 
+UNS historian is disabled** in an effort to minimise the impact of
+this change on existing installations. To enable the UNS historian,
+set the `historians.uns.enabled` environment variable to `true`. If 
+you only want to exclusively persist UNS data (and not legacy 
+Sparkplug data) then set `historians.sparkplug.enabled` to `false`.
+
 ## v3.1.0
 
 ### Administration interface

diff --git a/acs-manager/get-keytab.sh b/acs-manager/get-keytab.sh
@@ -4,4 +4,4 @@
 #
 
 # bin/bash
-kubectl --kubeconfig ./k3s.yaml get -n fpd-v3 secret manager-keytab -o jsonpath="{.data.client-keytab}" | base64 -d >"./keytab"
+kubectl get secret manager-keytab -o jsonpath="{.data.client-keytab}" | base64 -d >"./keytab"
diff --git a/acs-service-setup/dumps/admin.yaml b/acs-service-setup/dumps/admin.yaml
@@ -49,3 +49,6 @@ aces:
   - principal: !u ACS.Group.GlobalDebuggers
     permission: !u UUIDs.Permission.CmdEsc.Rebirth
     target: !u UUIDs.Special.Null
+  - principal: !u ACS.Group.GlobalDebuggers
+    permission: !u ACS.Perm.MQTT.ReadEntireUNS
+    target: !u UUIDs.Special.Null
diff --git a/acs-service-setup/dumps/uns.yaml b/acs-service-setup/dumps/uns.yaml
@@ -0,0 +1,42 @@
+# This dump creates a service account for the Sparkplug Ingester service
+# and grants it the necessary permissions to read and write to the UNS.
+# It also adds the ReadEntireUNS permission to the Warehouse role so
+# that the UNS historian can use it and has permission to read the UNS.
+---
+service: !u UUIDs.Service.ConfigDB
+version: 1
+overwrite: true
+objects:
+  !u ACS.Class.Permission:
+    - !u ACS.Perm.MQTT.WriteToEntireUNS
+    - !u ACS.Perm.MQTT.ReadEntireUNS
+  !u ACS.Class.UserGroup:
+    - !u ACS.Group.SparkplugIngesters
+    - !u ACS.Group.HistorianUNS
+configs:
+  !u UUIDs.App.Info:
+    !u ACS.Perm.MQTT.WriteToEntireUNS: { name: "MQTT: Write to entire UNS" }
+    !u ACS.Perm.MQTT.ReadEntireUNS: { name: "MQTT: Read entire UNS" }
+    !u ACS.Group.SparkplugIngesters: { name: "Sparkplug Ingesters" }
+    !u ACS.Group.HistorianUNS: { name: "Historian UNS" }
+  !u UUIDs.App.MQTTPermissionTemplate:
+    !u ACS.Perm.MQTT.WriteToEntireUNS:
+      UNS/v1/#: w
+    !u ACS.Perm.MQTT.ReadEntireUNS:
+      UNS/v1/#: rs
+---
+service: !u UUIDs.Service.Authentication
+version: 1
+groups:
+  !u ACS.Group.GlobalDebuggers:
+    - !u ACS.Group.SparkplugIngesters
+  !u ACS.PermGroup.MQTT:
+    - !u ACS.Perm.MQTT.WriteToEntireUNS
+    - !u ACS.Perm.MQTT.ReadEntireUNS
+aces:
+  - principal: !u ACS.Group.HistorianUNS
+    permission: !u ACS.Perm.MQTT.ReadEntireUNS
+    target: !u UUIDs.Special.Null
+  - principal: !u ACS.Group.SparkplugIngesters
+    permission: !u ACS.Perm.MQTT.WriteToEntireUNS
+    target: !u UUIDs.Special.Self
diff --git a/acs-service-setup/lib/uuids.js b/acs-service-setup/lib/uuids.js
@@ -8,6 +8,7 @@ export const ACS = {
         ClientRole:         "1c567e3c-5519-4418-8682-6086f22fbc13",
         EdgeAccount:        "97756c9a-38e6-4238-b78c-3df6f227a6c9",
         ServiceAccount:     "e463b4ae-a322-46cc-8976-4ba76838e908",
+        Permission:         "8ae784bb-c4b5-4995-9bf6-799b3c7f21ad",
         UserAccount:        "8b3e8f35-78e5-4f93-bf21-7238bcb2ba9d",
         UserGroup:          "f1fabdd1-de90-4399-b3da-ccf6c2b2c08b",
     },
@@ -22,6 +23,8 @@ export const ACS = {
         EdgeGroups:         "9ba0de4b-056f-4b5e-b966-2d5d85d07767",
         EdgePermissions:    "7594cd71-e5b9-4467-88c0-b11a66d47fec",
         CentralMonitor:     "1bc3dbca-68fe-48d2-9590-3a528c111827",
+        SparkplugIngesters: "e414d355-b991-429b-8f5d-97e823ff71f5",
+        HistorianUNS:       "03f5f08a-f61e-4134-8f66-b2951e3bbb69",
     },
     Perm: {
         MQTT: {
@@ -30,6 +33,8 @@ export const ACS = {
             ReadAllStates:      "8790cf3d-b793-423c-b373-8cfcf9f63529",
             ReadNode:           "046d6603-fa62-4208-9400-65d61f8b1ec4",
             ReadWholeNamespace: "81833dbb-1150-4078-b1db-978c646ba73e",
+            WriteToEntireUNS:   "9fa6ff20-9d2a-4444-960c-40ebcf56f5b4",
+            ReadEntireUNS:      "ffa40b36-3a61-4545-832a-2d1e8b860d63",
         },
     },
     PermGroup: {
@@ -55,6 +60,7 @@ export const ACS = {
     Role: {
         EdgeNodeConsumer:       "17a64293-b82d-4db4-af4d-63359bb62934",
         GlobalDebugger:         "4473fe9c-05b0-42cc-ad8c-8e05f6d0ca86",
+        Warehouse:              "6958c812-fbe2-4e6c-b997-6f850b89f679",
     },
     /* XXX This should not be fixed. Currently this matches the fixed
      * UUID deployed by the dumps in the ACS Helm chart. This needs

diff --git a/acs-service-setup/package.json b/acs-service-setup/package.json
@@ -12,7 +12,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@amrc-factoryplus/utilities": "^1.3.1",
+    "@amrc-factoryplus/utilities": "^1.3.4",
     "eslint": "^8.55.0",
     "yaml": "^2.3.4"
   }

diff --git a/deploy/templates/auth/principals/service-clients.yaml b/deploy/templates/auth/principals/service-clients.yaml
@@ -1,18 +1,18 @@
 ## Service client principals.
-  ##
-  ## These are the principals that services use when they are acting as a client of another
-  ## service. These are the principals that should be granted rights.
-  ##
-  ## Random type principals get a random key and a Kerberos keytab. They are the most
-  ## secure option but can only be used for services that can act as a Kerberos client.
-  ##
-  ## Password type principals get a random password generated by the system. This is next
-  ## most secure.
-  ##
-  ## PresetPassword type principals use a password set by the administrator in the
-  ## krb5-presets secret. This option should only be used for services that can't pull
-  ## their password from a k8s secret, like Canary.
-  #
+##
+## These are the principals that services use when they are acting as a client of another
+## service. These are the principals that should be granted rights.
+##
+## Random type principals get a random key and a Kerberos keytab. They are the most
+## secure option but can only be used for services that can act as a Kerberos client.
+##
+## Password type principals get a random password generated by the system. This is next
+## most secure.
+##
+## PresetPassword type principals use a password set by the administrator in the
+## krb5-presets secret. This option should only be used for services that can't pull
+## their password from a k8s secret, like Canary.
+#
 {{- if .Values.auth.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey
@@ -80,7 +80,7 @@ spec:
   secret: manager-keytab/client-keytab
 {{- end }}
 ---
-{{- if .Values.warehouse.ingester.enabled }}
+{{- if .Values.historians.sparkplug.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey
 metadata:
@@ -122,3 +122,41 @@ spec:
       group: {{ .Values.acs.organisation }}-Service-Core
       node: Monitor
 {{- end }}
+---
+{{- if .Values.unsIngesters.sparkplug.enabled }}
+apiVersion: factoryplus.app.amrc.co.uk/v1
+kind: KerberosKey
+metadata:
+  name: sv1sparkplugingester
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: Random
+  principal: sv1sparkplugingester@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
+  secret: uns-ingester-sparkplug-keytabs/client
+  account:
+    class: e463b4ae-a322-46cc-8976-4ba76838e908
+    name: Sparkplug Ingester
+    groups:
+      - e414d355-b991-429b-8f5d-97e823ff71f5
+
+{{- end }}
+---
+# In the future if we have any more ingesters then this should be
+# enabled if any of the ingesters are enabled.
+{{- if .Values.historians.uns.enabled }}
+apiVersion: factoryplus.app.amrc.co.uk/v1
+kind: KerberosKey
+metadata:
+  name: sv1historianuns
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: Random
+  principal: sv1historianuns@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
+  secret: historian-uns-keytabs/client
+  account:
+    class: e463b4ae-a322-46cc-8976-4ba76838e908
+    name: Historian UNS
+    groups:
+      - 03f5f08a-f61e-4134-8f66-b2951e3bbb69
+
+{{- end }}
diff --git a/deploy/templates/auth/principals/services.yaml b/deploy/templates/auth/principals/services.yaml
@@ -100,19 +100,6 @@ spec:
   principal: postgres/postgres.{{ .Release.Namespace }}.svc.cluster.local@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
 {{- end }}
 ---
-{{- if .Values.warehouse.enabled }}
-apiVersion: factoryplus.app.amrc.co.uk/v1
-kind: KerberosKey
-metadata:
-  name: http.warehouse
-  namespace: {{ .Release.Namespace }}
-spec:
-  type: Random
-  principal: HTTP/warehouse.{{ .Release.Namespace }}.svc.cluster.local@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
-  additionalPrincipals:
-    - HTTP/warehouse.{{.Values.acs.baseUrl | required "values.acs.baseUrl is required"}}@{{ .Values.identity.realm | required "values.identity.realm is required!" }}
-{{- end }}
----
 {{- if .Values.git.enabled }}
 apiVersion: factoryplus.app.amrc.co.uk/v1
 kind: KerberosKey

diff --git a/.../grafana/grafana-datasource-influxdb.yaml → ...templates/grafana/grafana-datasource.yaml b/.../grafana/grafana-datasource-influxdb.yaml → ...templates/grafana/grafana-datasource.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: grafana-datasource-influxdb
+  name: grafana-datasource
   namespace: {{ .Release.Namespace }}
   labels:
     grafana_datasource: "1"
@@ -13,6 +13,7 @@ data:
     # list of datasources to insert/update depending
     # what's available in the database
     datasources:
+      {{ if .Values.historians.sparkplug.enabled }}
       # <string, required> name of the datasource. Required
       - name: InfluxDB (Flux)
         # <string, required> datasource type. Required
@@ -55,3 +56,48 @@ data:
           httpHeaderValue1: Token ${admin-token}
         version: 1
         editable: true
+        {{- end -}}
+        {{ if .Values.historians.uns.enabled }}
+        # <string, required> name of the datasource. Required
+      - name: UNS (Flux)
+        # <string, required> datasource type. Required
+        type: influxdb
+        # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+        access: proxy
+        # <int> org id. will default to orgId 1 if not specified
+        orgId: 1
+        # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+        uid: influxdb-flux-uns
+        # <string> url
+        url: http://acs-influxdb2
+        # <bool> mark as default datasource. Max one per org
+        isDefault: false
+        jsonData:
+          organization: default
+          defaultBucket: uns
+          version: Flux
+        # <string> json object of data that will be encrypted.
+        secureJsonData:
+          token: ${admin-token}
+        version: 1
+        editable: true
+      - name: UNS (InfluxQL)
+        type: influxdb
+        # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+        access: proxy
+        # <int> org id. will default to orgId 1 if not specified
+        orgId: 1
+        # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+        uid: influxdb-uns-influxql
+        # <string> url
+        url: http://acs-influxdb2
+        # <bool> mark as default datasource. Max one per org
+        isDefault: false
+        jsonData:
+          dbName: uns
+          httpHeaderName1: Authorization
+        secureJsonData:
+          httpHeaderValue1: Token ${admin-token}
+        version: 1
+        editable: true
+        {{- end -}}
diff --git a/...emplates/warehouse/influxdb-ingester.yaml → deploy/templates/historians/sparkplug.yaml b/...emplates/warehouse/influxdb-ingester.yaml → deploy/templates/historians/sparkplug.yaml
@@ -1,21 +1,21 @@
-{{ if .Values.warehouse.ingester.enabled }}
+{{ if .Values.historians.sparkplug.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: influxdb-ingester
+  name: historian-sparkplug
   namespace: {{ .Release.Namespace }}
   labels:
-    component: influxdb-ingester
+    component: historian-sparkplug
 spec:
   replicas: 1
   selector:
     matchLabels:
-      component: influxdb-ingester
+      component: historian-sparkplug
   template:
     metadata:
       labels:
-        component: influxdb-ingester
-        factory-plus.service: influxdb-ingester
+        component: historian-sparkplug
+        factory-plus.service: historian-sparkplug
     spec:
       {{- with .Values.acs.imagePullSecrets }}
       imagePullSecrets:
@@ -33,8 +33,8 @@ spec:
                 path: client
 
       containers:
-        - name: influxdb-ingester
-          image: "{{ include "amrc-connectivity-stack.image-name" (list . .Values.warehouse.ingester) }}"
+        - name: historian-sparkplug
+          image: "{{ include "amrc-connectivity-stack.image-name" (list . .Values.historians.sparkplug) }}"
           command: [ "/usr/bin/k5start", "-Uf", "/keytabs/client" ]
           args: [ "node", "--es-module-specifier-resolution=node", "bin/ingester.js" ]
           imagePullPolicy: Always
@@ -44,11 +44,11 @@ spec:
             - name: CLIENT_KEYTAB
               value: /keytabs/client
             - name: LOG_LEVEL
-              value: {{ .Values.warehouse.ingester.logLevel | required "values.warehouse.ingester.logLevel" }}
+              value: {{ .Values.historians.sparkplug.logLevel | required "values.historians.sparkplug.logLevel" }}
             - name: BATCH_SIZE
-              value: {{ .Values.warehouse.ingester.batchSize | quote | required "values.warehouse.ingester.batchSize" }}
+              value: {{ .Values.historians.sparkplug.batchSize | quote | required "values.historians.sparkplug.batchSize" }}
             - name: FLUSH_INTERVAL
-              value: {{ .Values.warehouse.ingester.flushInterval | quote | required "values.warehouse.ingester.flushInterval" }}
+              value: {{ .Values.historians.sparkplug.flushInterval | quote | required "values.historians.sparkplug.flushInterval" }}
             - name: DIRECTORY_URL
               value: http://directory.{{ .Release.Namespace }}.svc.cluster.local
             - name: INFLUX_URL