Skip to content

Commit

Permalink
Merge pull request #8248 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history
New Unity Catalog resource connector
  • Loading branch information
mandy-chessell authored Jun 21, 2024
2 parents 7404422 + b507af2 commit 675c624
Show file tree
Hide file tree
Showing 74 changed files with 6,296 additions and 49 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/merge-v5.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
uses: docker/build-push-action@v5
with:
push: true
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria-platform:${{ env.VERSION }}, quay.io/odpi/egeria-platform:latest
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria:${{ env.VERSION }}, quay.io/odpi/egeria:latest
context: ./open-metadata-distribution/omag-server-platform/build/unpacked/egeria-platform-${{ env.VERSION }}-distribution.tar.gz
platforms: linux/amd64,linux/arm64
- name: Build and push(egeria-platform) to quay.io and docker.io (no tag latest)
Expand Down
2 changes: 1 addition & 1 deletion CoreContentPackGUIDMap.json

Large diffs are not rendered by default.

31 changes: 16 additions & 15 deletions MAINTAINERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,22 @@ new pull requests into main.
The current Egeria maintainers are:


| Name | Email | GitHub id
| --------------------- | ------------------------------ | --------------
| Mandy Chessell | mandy.e.chessell@gmail.com | mandy-chessell
| Nigel Jones | jonesn@uk.ibm.com | planetf1
| David Radley | david_radley@uk.ibm.com | davidradl
| Maryna Strelchuk | maryna.strelchuk@gmail.com | mstrelchuk
| Bogdan Sava | bogdan.sava@gmail.com | bogdan-sava
| Chris Grote | chris@thegrotes.net | cmgrote
| Raluca Popa | raluca.popa2@ing.com | popa-raluca
| Chris Replogle | Chris.Replogle@sas.com | CDARip2U
| Ljupcho Palashevski | ljupcho.palashevski@ing.com | lpalashevski
| Alexandra Bucur | alexandra-nona.bucur@ing.com | alexandra-bucur
| Cezar Sirbu | sirbunicolaecezar@gmail.com | sarbull
| Liviu Constantin Popa | liviu-constantin.popa@ing.com | lcpopa
| Cristian-Mihai Iliescu | cristianmihaiiliescu@gmail.com | MihaiIliescu
| Name | Email | GitHub id
|------------------------|--------------------------------| --------------
| Mandy Chessell | mandy.e.chessell@gmail.com | mandy-chessell
| Dan Wolfson | dan.wolfson@pdr-associates.com | dwolfson
| Nigel Jones | jonesn@uk.ibm.com | planetf1
| David Radley | david_radley@uk.ibm.com | davidradl
| Maryna Strelchuk | maryna.strelchuk@gmail.com | mstrelchuk
| Bogdan Sava | bogdan.sava@gmail.com | bogdan-sava
| Chris Grote | chris@thegrotes.net | cmgrote
| Raluca Popa | raluca.popa2@ing.com | popa-raluca
| Chris Replogle | Chris.Replogle@sas.com | CDARip2U
| Ljupcho Palashevski | ljupcho.palashevski@ing.com | lpalashevski
| Alexandra Bucur | alexandra-nona.bucur@ing.com | alexandra-bucur
| Cezar Sirbu | sirbunicolaecezar@gmail.com | sarbull
| Liviu Constantin Popa | liviu-constantin.popa@ing.com | lcpopa
| Cristian-Mihai Iliescu | cristianmihaiiliescu@gmail.com | MihaiIliescu

To understand how to become an Egeria maintainer
see the [Egeria Operations Guide](https://egeria-project.org/guides/project-operations/).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ public enum PostgresPlaceholderProperty
"string",
"myDatabase"),

/**
* The name of the database being catalogued.
*/
DATABASE_DESCRIPTION ("databaseDescription",
"The description of the database being catalogued.",
"string",
null),

/**
* The userId to store in the userId attribute of the connection.
*/
Expand All @@ -52,13 +60,23 @@ public enum PostgresPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the database server being catalogued.", "string", "myServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

/**
* The name of the schema being catalogued.
*/
SCHEMA_NAME ("schemaName", "The name of the database schema being catalogued.", "string", "MyServer.schema"),

/**
* The description of the schema being catalogued.
*/
SCHEMA_DESCRIPTION ("schemaDescription", "The description of the database schema being catalogued.", "string", null),

/**
* The name of the database table being catalogued.
*/
Expand Down Expand Up @@ -158,6 +176,7 @@ public static List<PlaceholderPropertyType> getPostgresServerPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.HOST_IDENTIFIER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand All @@ -178,6 +197,7 @@ public static List<PlaceholderPropertyType> getPostgresDatabasePlaceholderProper
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand All @@ -199,6 +219,7 @@ public static List<PlaceholderPropertyType> getPostgresSchemaPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@

@ucURL=http://localhost:8080

###
# @name list catalogs
GET {{ucURL}}/api/2.1/unity-catalog/catalogs?max_results=1

###
# @name create catalog
POST {{ucURL}}/api/2.1/unity-catalog/catalogs
Content-Type: application/json

{
"name": "test3",
"comment": "Test catalog3",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get catalog
GET {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json


###
# @name patch catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test4
Content-Type: application/json

{
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name rename catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test3_temp
Content-Type: application/json

{
"new_name" : "test4",
"comment" : "This is a new comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete catalog
DELETE {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list functions
GET {{ucURL}}/api/2.1/unity-catalog/functions?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create function
POST {{ucURL}}/api/2.1/unity-catalog/functions
Content-Type: application/json

{
"function_info" :
{
"name": "createWisdom",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Test catalog4 function 1",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get function
GET {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete function
DELETE {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@

@ucURL=http://localhost:8080

###
# @name list schemas
GET {{ucURL}}/api/2.1/unity-catalog/schemas?catalog_name=unity&max_results=10

###
# @name create schema
POST {{ucURL}}/api/2.1/unity-catalog/schemas
Content-Type: application/json

{
"name": "test4schema1",
"catalog_name" : "test",
"comment": "Test catalog4 schema 1",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get schema
GET {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json


###
# @name patch schema
PATCH {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema2
Content-Type: application/json

{
"new_name" : "test4schema1",
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete schema
DELETE {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list tables
GET {{ucURL}}/api/2.1/unity-catalog/tables?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create table
POST {{ucURL}}/api/2.1/unity-catalog/tables
Content-Type: application/json

{
"table_info" :
{
"name": "bookOfKnowledge",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Useful data",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get table
GET {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete table
DELETE {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@

@ucURL=http://localhost:8080

###
# @name list volumes
GET {{ucURL}}/api/2.1/unity-catalog/volumes?catalog_name=unity&schema_name=default&max_results=10

###
# @name create volume
POST {{ucURL}}/api/2.1/unity-catalog/volumes
Content-Type: application/json

{
"name": "test4volume1",
"catalog_name" : "test",
"schema_name": "test4schema1",
"comment": "Test catalog4 volume 1",
"volume_type": "EXTERNAL",
"storage_location": "file:///Users/me/Code/uc/unitycatalog/etc/data/external/unity/default/volumes/json_files/"
}

###
# @name get volume
GET {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.volume1
Content-Type: application/json


###
# @name patch volume
PATCH {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

{
"new_name" : "volume1",
"comment" : "This is an updated comment"
}

###
# @name delete volume
DELETE {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ dependencies {
compileOnly project(':open-metadata-implementation:access-services:asset-manager:asset-manager-api')
compileOnly project(':open-metadata-implementation:integration-services:catalog-integrator:catalog-integrator-api')
compileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-connectors-api')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-factory')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:spring-rest-client-connector')
testCompileOnly project(':open-metadata-implementation:frameworks:audit-log-framework')
testCompileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
testImplementation project(':open-metadata-test:open-metadata-ut')
testImplementation 'org.testng:testng'
compileOnly 'org.springframework:spring-core'
compileOnly 'com.fasterxml.jackson.core:jackson-databind'
compileOnly 'com.fasterxml.jackson.core:jackson-annotations'
}

shadowJar {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@ public enum UnityCatalogPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myOMAGServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

;

public final String name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ public enum UCAuditCode implements AuditLogMessageSet
"The integration connector continues, looking for another database.",
"This is an information message showing that the integration connector is working, but does not need to do any processing on this database."),



/**
* UNITY-CATALOG-CONNECTOR-0005 - A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR( "UNITY-CATALOG-CONNECTOR-0005",
AuditLogRecordSeverityLevel.EXCEPTION,
"A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}",
"The server has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final String logMessageId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,15 @@ public enum UCErrorCode implements ExceptionMessageSet
"The connector is unable to process the current request.",
"Use the details from the error message to determine the cause of the error and retry the request once it is resolved."),


/**
* UNITY-CATALOG-CONNECTOR-503-001 - A client-side exception {0} was received from API call {1} to URL {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR(503, "UNITY-CATALOG-CONNECTOR-503-001",
"A client-side exception of {0} was received from API call {1} to URL {2}. The error message was {3}",
"The connector has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final int httpErrorCode;
Expand Down
Loading

0 comments on commit 675c624

Please sign in to comment.