Merge pull request #8402 from mandy-chessell/oak2024

Additions to secrets connectors
odpi · Sep 27, 2024 · 78e776a · 78e776a
2 parents e2dcac8 + 53d6904
commit 78e776a
Show file tree

Hide file tree

Showing 96 changed files with 2,157 additions and 903 deletions.
diff --git a/EgeriaContentPacksGUIDMap.json b/EgeriaContentPacksGUIDMap.json
diff --git a/content-packs/APIsContentPack.omarchive b/content-packs/APIsContentPack.omarchive
diff --git a/content-packs/ApacheAtlasContentPack.omarchive b/content-packs/ApacheAtlasContentPack.omarchive
diff --git a/content-packs/ApacheKafkaContentPack.omarchive b/content-packs/ApacheKafkaContentPack.omarchive
diff --git a/content-packs/CocoBusinessSystemsArchive.omarchive b/content-packs/CocoBusinessSystemsArchive.omarchive
diff --git a/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive b/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive
diff --git a/content-packs/CocoComboArchive.omarchive b/content-packs/CocoComboArchive.omarchive
diff --git a/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive b/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive
diff --git a/content-packs/CocoGovernanceProgramArchive.omarchive b/content-packs/CocoGovernanceProgramArchive.omarchive
diff --git a/content-packs/CocoOrganizationArchive.omarchive b/content-packs/CocoOrganizationArchive.omarchive
diff --git a/content-packs/CocoSustainabilityArchive.omarchive b/content-packs/CocoSustainabilityArchive.omarchive
diff --git a/content-packs/CocoTypesArchive.omarchive b/content-packs/CocoTypesArchive.omarchive
diff --git a/content-packs/CoreContentPack.omarchive b/content-packs/CoreContentPack.omarchive
diff --git a/content-packs/ObservabilityContentPack.omarchive b/content-packs/ObservabilityContentPack.omarchive
diff --git a/content-packs/PostgresContentPack.omarchive b/content-packs/PostgresContentPack.omarchive
diff --git a/content-packs/UnityCatalogContentPack.omarchive b/content-packs/UnityCatalogContentPack.omarchive
diff --git a/...mance-suite-client/src/main/java/org/odpi/openmetadata/conformance/client/RESTClient.java b/...mance-suite-client/src/main/java/org/odpi/openmetadata/conformance/client/RESTClient.java
@@ -80,6 +80,7 @@ class RESTClient
                                                            serverPlatformURLRoot,
                                                            userId,
                                                            password,
+                                                           null,
                                                            null);
 
         try

diff --git a/...n/java/org/odpi/openmetadata/accessservices/assetowner/client/SurveyAssetStoreClient.java b/...n/java/org/odpi/openmetadata/accessservices/assetowner/client/SurveyAssetStoreClient.java
@@ -4,6 +4,7 @@
 package org.odpi.openmetadata.accessservices.assetowner.client;
 
 import org.odpi.openmetadata.adminservices.configuration.registration.AccessServiceDescription;
+import org.odpi.openmetadata.frameworks.auditlog.AuditLog;
 import org.odpi.openmetadata.frameworks.connectors.Connector;
 import org.odpi.openmetadata.frameworks.connectors.ffdc.*;
 import org.odpi.openmetadata.frameworks.connectors.properties.AssetUniverse;
@@ -22,6 +23,7 @@ public class SurveyAssetStoreClient extends SurveyAssetStore
     private final ConnectedAssetClient connectedAssetClient;
     private final FileSystemAssetOwner fileSystemAssetOwner;
     private final CSVFileAssetOwner    csvFileAssetOwner;
+    private final AuditLog             auditLog;
 
     /**
      * Constructor.
@@ -37,13 +39,15 @@ public SurveyAssetStoreClient(String               assetGUID,
                                   String               userId,
                                   ConnectedAssetClient connectedAssetClient,
                                   FileSystemAssetOwner fileSystemAssetOwner,
-                                  CSVFileAssetOwner    csvFileAssetOwner)
+                                  CSVFileAssetOwner    csvFileAssetOwner,
+                                  AuditLog             auditLog)
     {
         super(assetGUID, userId);
 
         this.connectedAssetClient = connectedAssetClient;
         this.fileSystemAssetOwner = fileSystemAssetOwner;
         this.csvFileAssetOwner    = csvFileAssetOwner;
+        this.auditLog             = auditLog;
     }
 
 
@@ -112,7 +116,7 @@ public void logAssetAuditMessage(String surveyService, String message) throws In
     @Override
     public Connector getConnectorToAsset() throws InvalidParameterException, ConnectionCheckedException, ConnectorCheckedException, UserNotAuthorizedException, PropertyServerException
     {
-        return connectedAssetClient.getConnectorForAsset(userId, assetGUID);
+        return connectedAssetClient.getConnectorForAsset(userId, assetGUID, auditLog);
     }
 
 

diff --git a/...pi/openmetadata/accessservices/governanceprogram/server/GovernanceDomainRESTServices.java b/...pi/openmetadata/accessservices/governanceprogram/server/GovernanceDomainRESTServices.java
@@ -857,8 +857,8 @@ public GovernanceDomainsResponse getGovernanceDomains(String serverName,
                                                                                                                         methodName);
 
             response.setElements(domainHandler.getBeansByType(userId,
-                                                              OpenMetadataType.GOVERNANCE_DOMAIN_TYPE_GUID,
-                                                              OpenMetadataType.GOVERNANCE_DOMAIN_TYPE_NAME,
+                                                              OpenMetadataType.GOVERNANCE_DOMAIN_DESCRIPTION_TYPE_GUID,
+                                                              OpenMetadataType.GOVERNANCE_DOMAIN_DESCRIPTION_TYPE_NAME,
                                                               null,
                                                               startFrom,
                                                               pageSize,
@@ -974,7 +974,7 @@ public GovernanceDomainSetsResponse getSetsForGovernanceDomain(String serverName
             List<GovernanceDomainSetElement> sets = handler.getAttachedElements(userId,
                                                                                 governanceDomainGUID,
                                                                                 guidParameterName,
-                                                                                OpenMetadataType.GOVERNANCE_DOMAIN_TYPE_NAME,
+                                                                                OpenMetadataType.GOVERNANCE_DOMAIN_DESCRIPTION_TYPE_NAME,
                                                                                 OpenMetadataType.COLLECTION_MEMBERSHIP_RELATIONSHIP.typeGUID,
                                                                                 OpenMetadataType.COLLECTION_MEMBERSHIP_RELATIONSHIP.typeName,
                                                                                 OpenMetadataType.COLLECTION.typeName,

diff --git a/...openmetadata/adapters/connectors/postgres/catalog/PostgresServerIntegrationConnector.java b/...openmetadata/adapters/connectors/postgres/catalog/PostgresServerIntegrationConnector.java
@@ -136,7 +136,7 @@ public void integrateCatalogTarget(RequestedCatalogTarget requestedCatalogTarget
             String databaseManagerGUID = this.getDatabaseManagerGUID(databaseServerGUID, requestedCatalogTarget.getCatalogTargetElement().getUniqueName());
             try
             {
-                Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(databaseServerGUID);
+                Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(databaseServerGUID, auditLog);
 
                 JDBCResourceConnector assetConnector = (JDBCResourceConnector)connector;
 

diff --git a/.../odpi/openmetadata/adapters/connectors/postgres/controls/PostgresPlaceholderProperty.java b/.../odpi/openmetadata/adapters/connectors/postgres/controls/PostgresPlaceholderProperty.java
@@ -159,8 +159,8 @@ public static List<PlaceholderPropertyType> getPostgresServerPlaceholderProperty
         placeholderPropertyTypes.add(PlaceholderProperty.PORT_NUMBER.getPlaceholderType());
         placeholderPropertyTypes.add(PlaceholderProperty.SERVER_NAME.getPlaceholderType());
         placeholderPropertyTypes.add(PlaceholderProperty.DESCRIPTION.getPlaceholderType());
-        placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
-        placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.VERSION_IDENTIFIER.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.SECRETS_STORE.getPlaceholderType());
 
         return placeholderPropertyTypes;
     }

diff --git a/...rs/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http b/...rs/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http
@@ -68,7 +68,31 @@ Content-Type: application/json
 
 ###
 
-@server1GUID=281c6358-ee6f-4009-9de9-4ef92a46f393
+
+###
+# @name createElementFromTemplate (with token)
+# Create a new element from a template.
+# Further Information: https://egeria-project.org/features/templated-cataloguing/overview/
+POST {{baseURL}}/servers/{{viewServer}}/api/open-metadata/automated-curation/catalog-templates/new-element
+Authorization: Bearer {{token}}
+Content-Type: application/json
+
+{
+  "templateGUID" : "3f7f62f6-4abc-424e-9f92-523306e7d5d5",
+  "isOwnAnchor" : true,
+  "placeholderPropertyValues" : {
+    "serverName" : "Databricks UC",
+    "hostURL" : "http://localhost",
+    "portNumber" : "8080",
+    "secretsStorePathName" : "loading-bay/secrets/default.omsecrets",
+    "description" : "Local Unity Catalog",
+    "versionIdentifier" : "V1.0"
+  }
+}
+
+###
+
+@server1GUID=5d133ab0-efaf-4c39-aece-0a3f63bb8e63
 
 
 ###
@@ -83,6 +107,19 @@ Content-Type: application/json
 }
 
 
+
+###
+# @name getTechnologyTypeElements (UC Server)
+# Retrieve the elements for the requested deployed implementation type. There are no wildcards allowed in the name.
+POST {{baseURL}}/servers/{{viewServer}}/api/open-metadata/automated-curation/technology-types/elements?startFrom=0&pageSize=10&getTemplates=false
+Authorization: Bearer {{token}}
+Content-Type: application/json
+
+{
+  "filter" : "Databricks Unity Catalog Server"
+}
+
+
 ###
 # @name=getAssetGraph (UC Server 1)
 # Return all the elements that are anchored to an asset plus relationships between these elements and to other elements.

diff --git a/...ata/adapters/connectors/unitycatalog/controls/UnityCatalogDeployedImplementationType.java b/...ata/adapters/connectors/unitycatalog/controls/UnityCatalogDeployedImplementationType.java
@@ -78,57 +78,6 @@ public enum UnityCatalogDeployedImplementationType implements DeployedImplementa
                              "The OSS Unity Catalog (UC) Server is an operational data platform 'catalog of catalogs' that supports controlled access to data managed through a related data platforms.",
                              "https://github.com/unitycatalog/unitycatalog"),
 
-    /**
-     * A collection of related data files within the Unity Catalog (UC) 'catalog of catalogs'.
-     */
-    DB_UC_VOLUME("Databricks Unity Catalog Volume",
-                  DeployedImplementationType.DATA_FOLDER,
-                  OpenMetadataType.DATA_FOLDER.typeName,
-                  null,
-                  "A collection of related data files within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
-                  "https://github.com/unitycatalog/unitycatalog"),
-
-
-    /**
-     * A function found in Unity Catalog (UC) that is working with data..
-     */
-    DB_UC_FUNCTION("Databricks Unity Catalog Function",
-                    OSS_UC_FUNCTION,
-                    OpenMetadataType.DEPLOYED_API.typeName,
-                    null,
-                    "A function found in Databricks Unity Catalog (UC) that is working with data.",
-                    "https://github.com/unitycatalog/unitycatalog"),
-
-    /**
-     * A relational table within the Unity Catalog (UC) 'catalog of catalogs'.
-     */
-    DB_UC_TABLE("Databricks Unity Catalog Table",
-                 OSS_UC_TABLE,
-                 OpenMetadataType.VIRTUAL_RELATIONAL_TABLE.typeName,
-                 null,
-                 "A relational table within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
-                 "https://github.com/unitycatalog/unitycatalog"),
-
-
-    /**
-     * A schema that organizes data assets for an operational data platform catalog within the Unity Catalog (UC) 'catalog of catalogs'.
-     */
-    DB_UC_SCHEMA("Databricks Unity Catalog Schema",
-                  OSS_UC_SCHEMA,
-                  OpenMetadataType.DEPLOYED_DATABASE_SCHEMA.typeName,
-                  null,
-                  "A schema that organizes data assets for an operational data platform catalog within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
-                  "https://github.com/unitycatalog/unitycatalog"),
-
-    /**
-     * An operational data platform catalog within the Unity Catalog (UC) 'catalog of catalogs'.
-     */
-    DB_UC_CATALOG("Databricks Unity Catalog Catalog",
-                   OSS_UC_CATALOG,
-                   OpenMetadataType.CATALOG.typeName,
-                   OpenMetadataType.ASSET_MANAGER.typeName,
-                   "An operational data platform catalog within the Databricks Unity Catalog (UC) 'catalog of catalogs'.",
-                   "https://github.com/unitycatalog/unitycatalog"),
 
     /**
      * An operational metadata catalog that supports controlled access to data managed through a data platform.

diff --git a/...enmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogPlaceholderProperty.java b/...enmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogPlaceholderProperty.java
@@ -162,6 +162,26 @@ public static List<PlaceholderPropertyType> getServerPlaceholderPropertyTypes()
         return placeholderPropertyTypes;
     }
 
+
+    /**
+     * Retrieve all the defined placeholder properties
+     *
+     * @return list of placeholder property types
+     */
+    public static List<PlaceholderPropertyType> getSecureServerPlaceholderPropertyTypes()
+    {
+        List<PlaceholderPropertyType> placeholderPropertyTypes = new ArrayList<>();
+
+        placeholderPropertyTypes.add(PlaceholderProperty.HOST_URL.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.PORT_NUMBER.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.SERVER_NAME.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.DESCRIPTION.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.VERSION_IDENTIFIER.getPlaceholderType());
+        placeholderPropertyTypes.add(PlaceholderProperty.SECRETS_STORE.getPlaceholderType());
+
+        return placeholderPropertyTypes;
+    }
+
     /**
      * Retrieve all the defined placeholder properties
      *

diff --git a/...odpi/openmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogTemplateType.java b/...odpi/openmetadata/adapters/connectors/unitycatalog/controls/UnityCatalogTemplateType.java
@@ -35,69 +35,34 @@ public enum UnityCatalogTemplateType
                             "5ee006aa-a6d6-411b-9b8d-5f720c079cae",
                             null),
 
-    DB_UC_CATALOG_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getDeployedImplementationType(),
-                            "Create a " + UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getAssociatedTypeName() + " SoftwareCapability.",
-                            UnityCatalogDeployedImplementationType.DB_UC_CATALOG.getAssociatedTypeName(),
-                            false,
-                            "32a82fe1-b1f5-4d86-b54a-b19d0be87e6b",
-                            null),
-
     OSS_UC_SCHEMA_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getDeployedImplementationType(),
                            "Create a DeployedDatabaseSchema to represent a schema from the Unity Catalog (UC).",
                            UnityCatalogDeployedImplementationType.OSS_UC_SCHEMA.getAssociatedTypeName(),
                            false,
                            "5bf92b0f-3970-41ea-b0a3-aacfbf6fd92e",
                            null),
 
-    DB_UC_SCHEMA_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_SCHEMA.getDeployedImplementationType(),
-                           "Create a DeployedDatabaseSchema to represent a schema from the Unity Catalog (UC) running on a Databricks managed platform.",
-                           UnityCatalogDeployedImplementationType.DB_UC_SCHEMA.getAssociatedTypeName(),
-                           false,
-                           "844631dc-dc8a-4e8e-b285-56165917de09",
-                           null),
-
     OSS_UC_VOLUME_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getDeployedImplementationType(),
                            "Create a DataFolder asset to represent a volume from the Unity Catalog (UC).",
                            UnityCatalogDeployedImplementationType.OSS_UC_VOLUME.getAssociatedTypeName(),
                            false,
                            "92d2d2dc-0798-41f0-9512-b10548d312b7",
                            null),
 
-    DB_UC_VOLUME_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_VOLUME.getDeployedImplementationType(),
-                           "Create a DataFolder asset to represent a volume from the Unity Catalog (UC) running on a Databricks managed platform.",
-                           UnityCatalogDeployedImplementationType.DB_UC_VOLUME.getAssociatedTypeName(),
-                           false,
-                           "a639212a-5c2d-464c-933f-aa612b9a4706",
-                           null),
-
     OSS_UC_TABLE_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getDeployedImplementationType(),
                           "Create a VirtualTableAsset asset to represent a table from the Unity Catalog (UC).",
                           UnityCatalogDeployedImplementationType.OSS_UC_TABLE.getAssociatedTypeName(),
                           false,
                           "6cc1e5f5-4c1e-4290-a80e-e06643ffb13d",
                           null),
 
-    DB_UC_TABLE_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_TABLE.getDeployedImplementationType(),
-                          "Create a VirtualTableAsset asset to represent a table from the Unity Catalog (UC) running on a Databricks managed platform.",
-                          UnityCatalogDeployedImplementationType.DB_UC_TABLE.getAssociatedTypeName(),
-                          false,
-                          "a1f8a374-3f78-4d06-8d8c-cc3322fc8b53",
-                          null),
-
     OSS_UC_FUNCTION_TEMPLATE(UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getDeployedImplementationType(),
                              "Create a DeployedSoftwareComponent asset to represent a function from the Unity Catalog (UC).",
                              UnityCatalogDeployedImplementationType.OSS_UC_FUNCTION.getAssociatedTypeName(),
                              false,
                              "a490ba65-6104-4213-9be9-524e16fed8aa",
                              null),
 
-    DB_UC_FUNCTION_TEMPLATE(UnityCatalogDeployedImplementationType.DB_UC_FUNCTION.getDeployedImplementationType(),
-                             "Create a DeployedSoftwareComponent asset to represent a function from the Unity Catalog (UC) running on a Databricks managed platform.",
-                             UnityCatalogDeployedImplementationType.DB_UC_FUNCTION.getAssociatedTypeName(),
-                             false,
-                             "d01bb98a-a6a1-490a-b935-31bae5974f6e",
-                             null),
-
     ;
 
 

diff --git a/...nmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java b/...nmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java
@@ -118,7 +118,8 @@ public void start() throws ConnectorCheckedException
                                                               targetRootURL,
                                                               connectionProperties.getUserId(),
                                                               connectionProperties.getClearPassword(),
-                                                              secretsStoreConnectorMap);
+                                                              secretsStoreConnectorMap,
+                                                              auditLog);
 
             this.clientConnector = factory.getClientConnector();
 

diff --git a/...data/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncConnector.java b/...data/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncConnector.java
@@ -240,7 +240,8 @@ public void processEvent(AssetManagerOutTopicEvent event)
 
                                 if (propertyHelper.isTypeOf(catalogTarget.getCatalogTargetElement(), OpenMetadataType.ASSET.typeName))
                                 {
-                                    requestedCatalogTarget.setCatalogTargetConnector(integrationContext.getConnectedAssetContext().getConnectorToAsset(catalogTarget.getCatalogTargetElement().getGUID()));
+                                    requestedCatalogTarget.setCatalogTargetConnector(integrationContext.getConnectedAssetContext().getConnectorToAsset(catalogTarget.getCatalogTargetElement().getGUID(),
+                                                                                                                                                       auditLog));
                                 }
 
                                 auditLog.logMessage(methodName,
@@ -291,7 +292,7 @@ public void integrateCatalogTarget(RequestedCatalogTarget requestedCatalogTarget
                 {
                     String catalogName = requestedCatalogTarget.getConfigurationProperties().get(UnityCatalogPlaceholderProperty.CATALOG_NAME.getName()).toString();
 
-                    Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(requestedCatalogTarget.getCatalogTargetElement().getGUID());
+                    Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(requestedCatalogTarget.getCatalogTargetElement().getGUID(), auditLog);
 
                     OSSUnityCatalogResourceConnector assetConnector = (OSSUnityCatalogResourceConnector) connector;
 

diff --git a/...penmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java b/...penmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java
@@ -154,7 +154,7 @@ public void integrateCatalogTarget(RequestedCatalogTarget requestedCatalogTarget
             String ucServerGUID = requestedCatalogTarget.getCatalogTargetElement().getGUID();
             try
             {
-                Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(ucServerGUID);
+                Connector connector = getContext().getConnectedAssetContext().getConnectorToAsset(ucServerGUID, auditLog);
 
                 OSSUnityCatalogResourceConnector assetConnector = (OSSUnityCatalogResourceConnector) connector;
 

diff --git a/...va/org/odpi/openmetadata/adapters/connectors/datastore/datafolder/DataFolderProvider.java b/...va/org/odpi/openmetadata/adapters/connectors/datastore/datafolder/DataFolderProvider.java
@@ -30,7 +30,7 @@ public class DataFolderProvider extends ConnectorProviderBase
     private static final String  connectorQualifiedName = "Egeria:ResourceConnector:DataFolder";
     private static final String  connectorTypeName = "Data Folder Connector";
     private static final String  connectorTypeDescription = "Connector supports reading of data files grouped under a single folder.";
-    private static final String connectorWikiPage = "https://egeria-project.org/connectors/resource/data-folder-resource-connector/";
+    private static final String  connectorWikiPage = "https://egeria-project.org/connectors/resource/data-folder-resource-connector/";
 
 
     private static final String  connectorClass = "org.odpi.openmetadata.adapters.connectors.datastore.datafolder.DataFolderConnector";