Merge pull request #8337 from mandy-chessell/oak2024

Fixes to Coco Data Quality Sample

open-metadata-implementation/access-services/asset-consumer/asset-consumer-server/src/main/java/org/odpi/openmetadata/accessservices/assetconsumer/server/AssetConsumerRESTServices.java

@@ -2,6 +2,7 @@
 /* Copyright Contributors to the ODPi Egeria project. */
 package org.odpi.openmetadata.accessservices.assetconsumer.server;
 
+import org.odpi.openmetadata.frameworks.connectors.ffdc.InvalidParameterException;
 import org.odpi.openmetadata.frameworks.openmetadata.metadataelements.*;
 import org.odpi.openmetadata.accessservices.assetconsumer.handlers.LoggingHandler;
 import org.odpi.openmetadata.frameworks.openmetadata.metadataelements.AssetGraph;
@@ -509,15 +510,24 @@ public AssetSearchMatchesListResponse findAssetsInDomain(String
 
                     for (String assetGUID : organizedEntities.keySet())
                     {
-                        AssetElement asset = assetHandler.getBeanFromRepository(userId,
-                                                                                assetGUID,
-                                                                                parameterName,
-                                                                                OpenMetadataType.ASSET.typeName,
-                                                                                methodName);
+                        AssetElement asset;
+
+                        try
+                        {
+                            asset = assetHandler.getBeanFromRepository(userId,
+                                                                       assetGUID,
+                                                                       parameterName,
+                                                                       OpenMetadataType.ASSET.typeName,
+                                                                       methodName);
+                        }
+                        catch (InvalidParameterException notVisible)
+                        {
+                            asset = null;
+                        }
 
                         if (asset != null)
                         {
-                            AssetSearchMatches         assetSearchMatches = new AssetSearchMatches(asset);
+                            AssetSearchMatches           assetSearchMatches = new AssetSearchMatches(asset);
                             Map<String, EntityDetail>    assetEntityMap   = organizedEntities.get(assetGUID);
                             List<MetadataElementSummary> anchoredElements = new ArrayList<>();
 

open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-onboarding-clinical-trial.http

@@ -35,6 +35,8 @@ Content-Type: application/json
 DELETE {{ucURL}}/api/2.1/unity-catalog/schemas/clinical_trials.teddy_bear_drop_foot
 Content-Type: application/json
 
+###
+# Remember to delete files in data lake
 ###
 # =====================================================================================================================
 # These commands add the CocoComboArchive to active-metadata-store and a new governance engine for clinical trials
@@ -383,7 +385,7 @@ Content-Type: application/json
     {
       "class" : "NewActionTarget",
       "actionTargetName": "dataLakeCatalog",
-      "actionTargetGUID": "c861d5e8-1cda-4063-9c89-d79e0e505d05"
+      "actionTargetGUID": "7dc0f1ad-c831-4e53-8d2b-dd503042e259"
     }],
     "requestParameters" : {
        "dataLakeSchemaName" : "teddy_bear_drop_foot",

open-metadata-implementation/adapters/open-connectors/data-store-connectors/file-connectors/csv-file-connector/src/main/java/org/odpi/openmetadata/adapters/connectors/datastore/csvfile/CSVFileStore.java

@@ -50,5 +50,5 @@ public interface CSVFileStore extends BasicFileStore
      * @throws FileException there is a problem accessing the file
      * @throws FileReadException unable to find, open or read the file, or the file does not include the requested record.
      */
-    List<String>      readRecord(int  rowNumber) throws FileException, FileReadException;
+    List<String>      readRecord(long  rowNumber) throws FileException, FileReadException;
 }

open-metadata-implementation/adapters/open-connectors/data-store-connectors/file-connectors/csv-file-connector/src/main/java/org/odpi/openmetadata/adapters/connectors/datastore/csvfile/CSVFileStoreConnector.java

@@ -105,6 +105,7 @@ public void initialize(String connectorInstanceId, ConnectionProperties connecti
      * @throws FileException problem accessing the file
      * @throws FileReadException unable to find, open or scan the file.
      */
+    @Override
     public long     getRecordCount() throws FileException, FileReadException
     {
         final String  methodName = "getRecordCount";
@@ -149,6 +150,7 @@ public long     getRecordCount() throws FileException, FileReadException
      * @throws FileException problem accessing the file
      * @throws FileReadException unable to retrieve the column names
      */
+    @Override
     public List<String>      getColumnNames() throws FileException,
                                                      FileReadException
     {
@@ -176,7 +178,7 @@ public List<String>      getColumnNames() throws FileException,
      * @throws FileException problem accessing the file
      * @throws FileReadException unable to find, open or read the file, or the file does not include the requested record.
      */
-    public List<String>      readRecord(int  dataRecordNumber) throws FileException, FileReadException
+    public List<String>      readRecord(long  dataRecordNumber) throws FileException, FileReadException
     {
         final String  methodName = "readRecord";
 
@@ -200,7 +202,7 @@ public List<String>      readRecord(int  dataRecordNumber) throws FileException,
      * @throws FileException problem accessing the file
      * @throws FileReadException unable to find, open or read the file, or the file does not include the requested record.
      */
-    private List<String>      readRow(int     recordLocation,
+    private List<String>      readRow(long    recordLocation,
                                       String  methodName) throws FileException, FileReadException
     {
 
@@ -224,7 +226,7 @@ private List<String>      readRow(int     recordLocation,
             }
 
             throw new FileReadException(CSVFileConnectorErrorCode.FILE_TOO_SHORT.getMessageDefinition(fileStoreName,
-                                                                                                      Integer.toString(recordLocation)),
+                                                                                                      Long.toString(recordLocation)),
                                         this.getClass().getName(),
                                         methodName,
                                         fileStoreName);

open-metadata-implementation/adapters/open-connectors/data-store-connectors/file-connectors/csv-file-connector/src/test/java/org/odpi/openmetadata/adapters/connectors/datastore/csvfile/CSVFileStoreConnectorTest.java

@@ -95,15 +95,15 @@ private void scanFile(CSVFileStoreConnector connector) throws FileException,
         assertTrue(columnNames != null);
         assertFalse(columnNames.isEmpty());
 
-        for (int i=0; i<connector.getRecordCount(); i++)
+        for (long i=0; i<connector.getRecordCount(); i++)
         {
             List<String>  columns = connector.readRecord(i);
             assertTrue(columnNames.size() == columns.size());
         }
 
         try
         {
-            connector.readRecord(10000);
+            connector.readRecord(10000L);
             assertTrue(false);
 
         }

open-metadata-implementation/adapters/open-connectors/file-survey-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/surveyaction/surveycsv/CSVSurveyService.java

@@ -291,7 +291,7 @@ public void start() throws ConnectorCheckedException
                     }
                 }
 
-                for (int recordNumber=0; recordNumber < recordCount ; recordNumber++)
+                for (long recordNumber=0; recordNumber < recordCount ; recordNumber++)
                 {
                     List<String>  recordValues = assetConnector.readRecord(recordNumber);
 

open-metadata-implementation/adapters/open-connectors/governance-action-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/governanceactions/stewardship/CatalogServerGovernanceActionConnector.java

@@ -44,7 +44,7 @@ public CatalogServerGovernanceActionConnector()
      * @throws ConnectorCheckedException there is a problem within the governance action service.
      */
     @Override
-    public void start() throws ConnectorCheckedException
+    public synchronized void start() throws ConnectorCheckedException
     {
         final String methodName = "start";
 
@@ -187,7 +187,7 @@ Map<String, Object> combineProperties()
             configurationProperties.putAll(governanceContext.getRequestParameters());
         }
 
-        if (configurationProperties.isEmpty())
+        if ((configurationProperties == null) || (configurationProperties.isEmpty()))
         {
             return null;
         }

open-metadata-implementation/adapters/open-connectors/integration-connectors/csv-lineage-import-integration-connector/src/main/java/org/odpi/openmetadata/adapters/connectors/integration/csvlineageimporter/CSVLineageImporterConnector.java

@@ -128,7 +128,7 @@ private void processFile() throws ConnectorCheckedException
 
             long numberOfRecords = csvFileStoreConnector.getRecordCount();
 
-            for (int i = 0; i < numberOfRecords; i++)
+            for (long i = 0; i < numberOfRecords; i++)
             {
                 List<String> columns = csvFileStoreConnector.readRecord(i);
 

open-metadata-implementation/adapters/open-connectors/integration-connectors/openlineage-integration-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/integration/openlineage/OpenLineageLogStoreConnectorBase.java

@@ -180,6 +180,7 @@ public synchronized void start() throws ConnectorCheckedException
      *
      * @throws ConnectorCheckedException there is a problem with the connector.  It is not able to refresh the catalog targets.
      */
+    @Override
     public synchronized void refresh() throws ConnectorCheckedException
     {
         final String methodName = "refresh";

open-metadata-implementation/adapters/open-connectors/system-connectors/egeria-system-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/egeriainfrastructure/platform/catalog/OMAGServerPlatformCatalogConnector.java

@@ -71,7 +71,7 @@ public String toString()
      * @throws ConnectorCheckedException there is a problem within the connector.
      */
     @Override
-    public void start() throws ConnectorCheckedException
+    public synchronized void start() throws ConnectorCheckedException
     {
         super.start();
 
@@ -190,6 +190,7 @@ public void refresh() throws ConnectorCheckedException
     /**
      * Called each time an event that is published by the IT Infrastructure OMAS, it is looking for Software Server Platforms to add to monitoredPlatforms.
      */
+    @Override
     public void processEvent(ITInfrastructureOutTopicEvent event)
     {
         if ((event.getElementProperties() != null) &&

open-metadata-implementation/engine-services/survey-action/survey-action-server/src/main/java/org/odpi/openmetadata/engineservices/surveyaction/handlers/SurveyActionEngineHandler.java

@@ -3,30 +3,37 @@
 package org.odpi.openmetadata.engineservices.surveyaction.handlers;
 
 import org.odpi.openmetadata.accessservices.assetowner.client.CSVFileAssetOwner;
+import org.odpi.openmetadata.accessservices.assetowner.client.ConnectedAssetClient;
 import org.odpi.openmetadata.accessservices.assetowner.client.FileSystemAssetOwner;
+import org.odpi.openmetadata.accessservices.assetowner.client.SurveyAssetStoreClient;
 import org.odpi.openmetadata.accessservices.governanceserver.client.GovernanceConfigurationClient;
 import org.odpi.openmetadata.accessservices.governanceserver.client.GovernanceContextClient;
-import org.odpi.openmetadata.accessservices.assetowner.client.ConnectedAssetClient;
-import org.odpi.openmetadata.accessservices.assetowner.client.SurveyAssetStoreClient;
 import org.odpi.openmetadata.adminservices.configuration.properties.EngineConfig;
 import org.odpi.openmetadata.adminservices.configuration.registration.EngineServiceDescription;
 import org.odpi.openmetadata.engineservices.surveyaction.ffdc.SurveyActionAuditCode;
 import org.odpi.openmetadata.engineservices.surveyaction.ffdc.SurveyActionErrorCode;
 import org.odpi.openmetadata.frameworks.auditlog.AuditLog;
-import org.odpi.openmetadata.frameworks.connectors.ffdc.*;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.InvalidParameterException;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.PropertyServerException;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.UserNotAuthorizedException;
 import org.odpi.openmetadata.frameworks.governanceaction.client.OpenMetadataClient;
-import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataType;
+import org.odpi.openmetadata.frameworks.governanceaction.controls.ActionTarget;
 import org.odpi.openmetadata.frameworks.governanceaction.properties.ActionTargetElement;
-import org.odpi.openmetadata.frameworks.openmetadata.enums.EngineActionStatus;
 import org.odpi.openmetadata.frameworks.governanceaction.properties.RequestSourceElement;
+import org.odpi.openmetadata.frameworks.governanceaction.search.PropertyHelper;
+import org.odpi.openmetadata.frameworks.openmetadata.enums.EngineActionStatus;
+import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataType;
 import org.odpi.openmetadata.frameworks.surveyaction.AnnotationStore;
 import org.odpi.openmetadata.frameworks.surveyaction.SurveyAssetStore;
 import org.odpi.openmetadata.frameworks.surveyaction.SurveyContext;
 import org.odpi.openmetadata.frameworks.surveyaction.SurveyOpenMetadataStore;
 import org.odpi.openmetadata.governanceservers.enginehostservices.admin.GovernanceEngineHandler;
 import org.odpi.openmetadata.governanceservers.enginehostservices.admin.GovernanceServiceCache;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
 
 /**
  * The SurveyActionEngineHandler is responsible for running survey action services on demand.  It is initialized
@@ -40,6 +47,8 @@ public class SurveyActionEngineHandler extends GovernanceEngineHandler
     private final CSVFileAssetOwner    csvFileAssetOwner;    /* Initialized in constructor */
     private final OpenMetadataClient   openMetadataClient;      /* Initialized in constructor */
 
+    private final PropertyHelper propertyHelper = new PropertyHelper();
+
     /**
      * Create a client-side object for calling a survey action engine.
      *
@@ -195,17 +204,34 @@ private String getAssetGUIDFromActionTargets(List<ActionTargetElement> actionTar
         String assetGUID = null;
         List<String> ignoredAssets = new ArrayList<>();
 
+        /*
+         * First pick out all the assets ...
+         */
+        List<ActionTargetElement> assetTargetElements = new ArrayList<>();
+
         for (ActionTargetElement actionTargetElement : actionTargetElements)
         {
-            if ((actionTargetElement != null)
-                    && (actionTargetElement.getTargetElement() != null)
-                    && (actionTargetElement.getTargetElement().getType() != null))
+            if (actionTargetElement != null)
             {
-                String       typeName       = actionTargetElement.getTargetElement().getType().getTypeName();
-                List<String> superTypeNames = actionTargetElement.getTargetElement().getType().getSuperTypeNames();
+                if (propertyHelper.isTypeOf(actionTargetElement.getTargetElement(), OpenMetadataType.ASSET.typeName))
+                {
+                    assetTargetElements.add(actionTargetElement);
+                }
+            }
+        }
 
-                if ((OpenMetadataType.ASSET.typeName.equals(typeName)) ||
-                        ((superTypeNames != null) && (superTypeNames.contains(OpenMetadataType.ASSET.typeName))))
+        if (assetTargetElements.size() == 1)
+        {
+            assetGUID = assetTargetElements.get(0).getTargetElement().getElementGUID();
+        }
+        else
+        {
+            /*
+             * Since there are multiple assets, only pick out the ones with an action target name of "newAsset".
+             */
+            for (ActionTargetElement actionTargetElement : assetTargetElements)
+            {
+                if (ActionTarget.NEW_ASSET.getName().equals(actionTargetElement.getActionTargetName()))
                 {
                     if (assetGUID == null)
                     {
@@ -223,18 +249,20 @@ private String getAssetGUIDFromActionTargets(List<ActionTargetElement> actionTar
                     }
                 }
             }
-        }
 
-        if (! ignoredAssets.isEmpty())
-        {
-            auditLog.logMessage(methodName,
-                                SurveyActionAuditCode.IGNORING_ASSETS.getMessageDefinition(governanceServiceName,
-                                                                                           governanceRequestType,
-                                                                                           engineActionGUID,
-                                                                                           assetGUID,
-                                                                                           ignoredAssets.toString()));
+            if (! ignoredAssets.isEmpty())
+            {
+                auditLog.logMessage(methodName,
+                                    SurveyActionAuditCode.IGNORING_ASSETS.getMessageDefinition(governanceServiceName,
+                                                                                               governanceRequestType,
+                                                                                               engineActionGUID,
+                                                                                               assetGUID,
+                                                                                               ignoredAssets.toString()));
+            }
         }
 
+
+
         return assetGUID;
     }
 

open-metadata-implementation/frameworks/open-metadata-framework/src/main/java/org/odpi/openmetadata/frameworks/openmetadata/enums/DataType.java

@@ -126,14 +126,15 @@ public enum DataType implements OpenMetadataEnum
      *
      * @return int ordinal
      */
+    @Override
     public int getOrdinal() { return ordinal; }
 
 
     /**
      * Return the default name of the enumeration.
      *
      * @return String name
-     */
+     */@Override
     public String getName() { return name; }
 
 
@@ -142,6 +143,7 @@ public enum DataType implements OpenMetadataEnum
      *
      * @return String description
      */
+    @Override
     public String getDescription() { return description; }
 
 
@@ -150,6 +152,7 @@ public enum DataType implements OpenMetadataEnum
      *
      * @return  guid
      */
+    @Override
     public  String getDescriptionGUID()
     {
         return descriptionGUID;
@@ -161,6 +164,7 @@ public  String getDescriptionGUID()
      *
      * @return boolean
      */
+    @Override
     public boolean isDefault()
     {
         return isDefault;

open-metadata-implementation/frameworks/survey-action-framework/src/main/java/org/odpi/openmetadata/frameworks/surveyaction/AnnotationStore.java

@@ -750,6 +750,8 @@ public String  addAnnotation(Annotation annotation,
                                                                    null,
                                                                    this.getEffectiveTime());
                 }
+
+                return annotationGUID;
             }
         }
 

open-metadata-implementation/frameworks/survey-action-framework/src/main/java/org/odpi/openmetadata/frameworks/surveyaction/SurveyContext.java

@@ -346,7 +346,6 @@ public String toString()
                 ", openMetadataStore=" + openMetadataStore +
                 ", surveyActionServiceName='" + surveyActionServiceName + '\'' +
                 ", requesterUserId='" + requesterUserId + '\'' +
-                ", auditLog=" + auditLog +
                 ", fileClassifier=" + fileClassifier +
                 ", isActive=" + isActive +
                 ", completionStatus=" + completionStatus +

open-metadata-implementation/governance-server-services/engine-host-services/engine-host-services-api/src/main/java/org/odpi/openmetadata/governanceservers/enginehostservices/ffdc/EngineHostServicesAuditCode.java

@@ -476,7 +476,7 @@ public enum EngineHostServicesAuditCode implements AuditLogMessageSet
      * ENGINE-HOST-SERVICES-2000 - {0} caught an exception {1} while processing governance action {2}; the error message was {3}
      */
     ACTION_PROCESSING_ERROR( "ENGINE-HOST-SERVICES-2000",
-                             AuditLogRecordSeverityLevel.ERROR,
+                             AuditLogRecordSeverityLevel.EXCEPTION,
                             "{0} caught an exception {1} while processing engine action {2}; the error message was {3}",
                             "The server is not able to start or complete the requested processing related to the governance service for this engine action.",
                             "Follow the instructions for the message associated with the exception."),

open-metadata-implementation/governance-server-services/integration-daemon-services/integration-daemon-services-server/src/main/java/org/odpi/openmetadata/governanceservers/integrationdaemonservices/threads/IntegrationConnectorRefreshThread.java

@@ -111,10 +111,11 @@ else if (minMinutedBetweenRefresh > 0)
                 }
                 catch (Exception error)
                 {
-                    auditLog.logMessage(actionDescription,
-                                        IntegrationDaemonServicesAuditCode.REFRESH_THREAD_CONNECTOR_ERROR.getMessageDefinition(connectorHandler.getIntegrationConnectorName(),
-                                                                                                                               error.getClass().getName(),
-                                                                                                                               error.getMessage()));
+                    auditLog.logException(actionDescription,
+                                          IntegrationDaemonServicesAuditCode.REFRESH_THREAD_CONNECTOR_ERROR.getMessageDefinition(connectorHandler.getIntegrationConnectorName(),
+                                                                                                                                 error.getClass().getName(),
+                                                                                                                                 error.getMessage()),
+                                          error);
                 }
             }