From c72425370fb9120a75406c481d1077f8f0ff11c9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 1 Jan 2025 21:12:54 +0100 Subject: [PATCH] Update readme --- .github/workflows/build.yml | 2 +- README.md | 28 +++++++++---------- pom.xml | 8 +++--- .../databricks/Application.java | 12 ++++---- .../databricks/DatabricksProperties.java | 4 +-- src/main/resources/application.properties | 4 +-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 449276e..2b302fa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 - tags: datameshmanager/datamesh-manager-agent-databricks:latest + tags: datameshmanager/datamesh-manager-connector-databricks:latest sbom: true cache-from: type=gha cache-to: type=gha,mode=max \ No newline at end of file diff --git a/README.md b/README.md index 5fa1af0..e88b40b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -Data Mesh Manager Agent for Databricks +Data Mesh Manager Connector for Databricks === -The agent for databricks is a Spring Boot application that uses the [datamesh-manager-sdk](https://github.com/datamesh-manager/datamesh-manager-sdk) internally, and is available as a ready-to-use Docker image [datameshmanager/datamesh-manager-agent-databricks](https://hub.docker.com/repository/docker/datameshmanager/datamesh-manager-agent-databricks) to be deployed in your environment. +The connector for databricks is a Spring Boot application that uses the [datamesh-manager-sdk](https://github.com/datamesh-manager/datamesh-manager-sdk) internally, and is available as a ready-to-use Docker image [datameshmanager/datamesh-manager-connector-databricks](https://hub.docker.com/repository/docker/datameshmanager/datamesh-manager-connector-databricks) to be deployed in your environment. ## Features @@ -10,7 +10,7 @@ The agent for databricks is a Spring Boot application that uses the [datamesh-ma ## Usage -Start the agent using Docker. You must pass the API keys as environment variables. +Start the connector using Docker. You must pass the API keys as environment variables. ``` docker run \ @@ -22,7 +22,7 @@ docker run \ -e DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_ACCOUNTID='your-account-id' \ -e DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_CLIENTID='your-account-client-id' \ -e DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_CLIENTSECRET='your-account-client-secret' \ - datameshmanager/datamesh-manager-agent-databricks:latest + datameshmanager/datamesh-manager-connector-databricks:latest ``` ## Configuration @@ -38,16 +38,16 @@ docker run \ | `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_ACCOUNTID` | | The databricks Account ID. | | `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_CLIENTID` | | The client ID of a an account service principal with Account admin role. | | `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCOUNT_CLIENTSECRET` | | The client secret of a an account service principal with Account admin role. | -| `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCESSMANAGEMENT_AGENTID` | `databricks-access-management` | Identifier for the Databricks access management agent. | +| `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCESSMANAGEMENT_CONNECTORID` | `databricks-access-management` | Identifier for the Databricks access management connector. | | `DATAMESHMANAGER_CLIENT_DATABRICKS_ACCESSMANAGEMENT_ENABLED` | `true` | Indicates whether Databricks access management is enabled. | -| `DATAMESHMANAGER_CLIENT_DATABRICKS_ASSETS_AGENTID` | `databricks-assets` | Identifier for the Databricks assets agent. | +| `DATAMESHMANAGER_CLIENT_DATABRICKS_ASSETS_CONNECTORID` | `databricks-assets` | Identifier for the Databricks assets connector. | | `DATAMESHMANAGER_CLIENT_DATABRICKS_ASSETS_ENABLED` | `true` | Indicates whether Databricks asset tracking is enabled. | | `DATAMESHMANAGER_CLIENT_DATABRICKS_ASSETS_POLLINTERVAL` | `PT10M` | Polling interval for Databricks asset updates, in ISO 8601 duration format. | ## Access Management Flow -When an Access Request has been approved by the data product owner, and the start date is reached, Data Mesh Manager will publish an `AccessActivatedEvent`. When an end date is defined and reached, Data Mesh Manager will publish an `AccessDeactivatedEvent`. The agent listens for these events and grants access to the data consumer in Databricks. +When an Access Request has been approved by the data product owner, and the start date is reached, Data Mesh Manager will publish an `AccessActivatedEvent`. When an end date is defined and reached, Data Mesh Manager will publish an `AccessDeactivatedEvent`. The connector listens for these events and grants access to the data consumer in Databricks. ### Consumer Type: Data Product @@ -58,7 +58,7 @@ Example: - Consumer is a data product with ID `c-300`. - Access ID is `a-100`. -Agent Actions on `AccessActivatedEvent`: +Connector Actions on `AccessActivatedEvent`: - Create a new service principal `dataproduct-c-300`, if it does not exist. (if a custom field `databricksServicePrincipal` is defined in the data product, the value will be used as the service principal name instead of the ID) - Create a new group `access-a-100` for this access. @@ -68,7 +68,7 @@ Agent Actions on `AccessActivatedEvent`: - Add the group `team-t-300` to the group `access-a-101`. - Grant permissions `USE SCHEMA` and `SELECT` on the schema `my_catalog.schema_220` to group `access-a-100` -Agent Actions on `AccessDeactivatedEvent`: +Connector Actions on `AccessDeactivatedEvent`: - Delete the group `access-a-100` @@ -82,7 +82,7 @@ Example: - Consumer is a team with ID `t-400`. - Access ID is `a-101`. -Agent Actions on `AccessActivatedEvent`: +Connector Actions on `AccessActivatedEvent`: - Create a new group `team-t-400`, if it does not exist. (if a custom field `databricksGroupName` is defined in the team, the value will be used as the group name instead of the ID) - Add all members of the team `t-400` to the group `team-t-400`. @@ -90,7 +90,7 @@ Agent Actions on `AccessActivatedEvent`: - Add the group `team-t-400` to the group `access-a-101`. - Grant permissions `USE SCHEMA` and `SELECT` on the schema `my_catalog.schema_220` to group `access-a-101` -Agent Actions on `AccessDeactivatedEvent`: +Connector Actions on `AccessDeactivatedEvent`: - Delete the group `access-a-101` @@ -104,13 +104,13 @@ Example: - Consumer is an individual user with username `alice@example.com`. - Access ID is `a-102`. -Agent Actions on `AccessActivatedEvent`: +Connector Actions on `AccessActivatedEvent`: - Create a new group `access-a-102` for this access. -- Add the user `alice@example.com` to the group `access-a-102` (the agent currently assumes that the username in Data Mesh Manager and Databricks are equal). +- Add the user `alice@example.com` to the group `access-a-102` (the connector currently assumes that the username in Data Mesh Manager and Databricks are equal). - Grant permissions `USE SCHEMA` and `SELECT` on the schema `my_catalog.schema_220` to group `access-a-102` -Agent Actions on `AccessDeactivatedEvent`: +Connector Actions on `AccessDeactivatedEvent`: - Delete the group `access-a-102` diff --git a/pom.xml b/pom.xml index d1d6c49..a71a325 100644 --- a/pom.xml +++ b/pom.xml @@ -9,10 +9,10 @@ com.datamesh-manager - datamesh-manager-agent-databricks + datamesh-manager-connector-databricks 0.0.1-SNAPSHOT - datamesh-manager-agent-databricks - Data Mesh Manager Agent for Databricks Integration + datamesh-manager-connector-databricks + Data Mesh Manager Connector for Databricks Integration @@ -27,7 +27,7 @@ com.datamesh-manager datamesh-manager-sdk - 0.0.2 + 0.0.3 com.databricks diff --git a/src/main/java/datameshmanager/databricks/Application.java b/src/main/java/datameshmanager/databricks/Application.java index bf99fcd..c54476b 100644 --- a/src/main/java/datameshmanager/databricks/Application.java +++ b/src/main/java/datameshmanager/databricks/Application.java @@ -60,10 +60,10 @@ public DataMeshManagerEventListener dataMeshManagerEventListener( WorkspaceClient workspaceClient, AccountClient accountClient, TaskExecutor taskExecutor) { - var agentid = databricksProperties.accessmanagement().agentid(); + var connectorid = databricksProperties.accessmanagement().connectorid(); var eventHandler = new DatabricksAccessManagementHandler(client, workspaceClient, accountClient); - var stateRepository = new DataMeshManagerStateRepositoryRemote(agentid, client); - var dataMeshManagerEventListener = new DataMeshManagerEventListener(agentid, client, eventHandler, stateRepository); + var stateRepository = new DataMeshManagerStateRepositoryRemote(connectorid, client); + var dataMeshManagerEventListener = new DataMeshManagerEventListener(connectorid, "accessmanagement", client, eventHandler, stateRepository); taskExecutor.execute(dataMeshManagerEventListener::start); return dataMeshManagerEventListener; } @@ -75,10 +75,10 @@ public DataMeshManagerAssetsSynchronizer dataMeshManagerAssetsSynchronizer( DataMeshManagerClient client, WorkspaceClient workspaceClient, TaskExecutor taskExecutor) { - var agentid = databricksProperties.assets().agentid(); - var stateRepository = new DataMeshManagerStateRepositoryRemote(agentid, client); + var connectorid = databricksProperties.assets().connectorid(); + var stateRepository = new DataMeshManagerStateRepositoryRemote(connectorid, client); var assetsSupplier = new DatabricksAssetsSupplier(workspaceClient, stateRepository, databricksProperties); - var dataMeshManagerAssetsSynchronizer = new DataMeshManagerAssetsSynchronizer(agentid, client, assetsSupplier); + var dataMeshManagerAssetsSynchronizer = new DataMeshManagerAssetsSynchronizer(connectorid, client, assetsSupplier); if (databricksProperties.assets().pollinterval() != null) { dataMeshManagerAssetsSynchronizer.setDelay(databricksProperties.assets().pollinterval()); } diff --git a/src/main/java/datameshmanager/databricks/DatabricksProperties.java b/src/main/java/datameshmanager/databricks/DatabricksProperties.java index fe561ed..bfa1b2c 100644 --- a/src/main/java/datameshmanager/databricks/DatabricksProperties.java +++ b/src/main/java/datameshmanager/databricks/DatabricksProperties.java @@ -30,7 +30,7 @@ public record AccountProperties( public record AssetsProperties( Boolean enabled, - String agentid, + String connectorid, Duration pollinterval ) { @@ -38,7 +38,7 @@ public record AssetsProperties( public record AccessmanagementProperties( Boolean enabled, - String agentid + String connectorid ) { } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 6e923b6..f19d983 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -10,10 +10,10 @@ datameshmanager.client.databricks.account.account-id=your-account-id datameshmanager.client.databricks.account.client-id=your-account-service-principal-client-id datameshmanager.client.databricks.account.client-secret=your-account-service-principal-client-secret -datameshmanager.client.databricks.accessmanagement.agentid=databricks-access-management +datameshmanager.client.databricks.accessmanagement.connectorid=databricks-access-management datameshmanager.client.databricks.accessmanagement.enabled=true -datameshmanager.client.databricks.assets.agentid=databricks-assets +datameshmanager.client.databricks.assets.connectorid=databricks-assets datameshmanager.client.databricks.assets.enabled=true datameshmanager.client.databricks.assets.pollinterval=PT10M datameshmanager.client.databricks.assets.tables.include=*