Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

zts/syncer dynamodb migration from aws v1 to v2 #2688

Merged
merged 1 commit into from
Aug 18, 2024
Merged

zts/syncer dynamodb migration from aws v1 to v2 #2688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 2 additions & 23 deletions libs/java/dynamodb_client_factory/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,9 @@
<name>athenz-dynamodb-client-factory</name>
<description>Generate dynamodb client based on AWS temporary credentials</description>
<packaging>jar</packaging>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>${aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws2.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<properties>
<code.coverage.min>0.9230</code.coverage.min>
<code.coverage.min>0.9417</code.coverage.min>
</properties>

<dependencies>
Expand All @@ -60,13 +42,10 @@
<artifactId>athenz-zts-java-client</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>dynamodb</artifactId>
<version>${aws2.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down
41 changes: 27 additions & 14 deletions ...ient_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientAndCredentials.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,30 +18,43 @@

package com.yahoo.athenz.db.dynamodb;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;

public class DynamoDBClientAndCredentials {
private final AmazonDynamoDB amazonDynamoDB;
private final DynamoDbAsyncClient amazonDynamoAsyncDB;
private final AWSCredentialsProviderImpl awsCredentialsProvider;

public DynamoDBClientAndCredentials(AmazonDynamoDB amazonDynamoDB, DynamoDbAsyncClient amazonDynamoAsyncDB, AWSCredentialsProviderImpl awsCredentialsProvider) {
this.amazonDynamoDB = amazonDynamoDB;
this.amazonDynamoAsyncDB = amazonDynamoAsyncDB;
private final DynamoDbClient dynamoDbClient;
private final DynamoDbAsyncClient dynamoDbAsyncClient;
private final AWSCredentialsProviderImplV2 awsCredentialsProvider;

public DynamoDBClientAndCredentials(DynamoDbClient dynamoDbClient, DynamoDbAsyncClient dynamoDbAsyncClient,
AWSCredentialsProviderImplV2 awsCredentialsProvider) {
this.dynamoDbClient = dynamoDbClient;
this.dynamoDbAsyncClient = dynamoDbAsyncClient;
this.awsCredentialsProvider = awsCredentialsProvider;
}

public AmazonDynamoDB getAmazonDynamoDB() {
return amazonDynamoDB;
public DynamoDbClient getDynamoDbClient() {
return dynamoDbClient;
}

public DynamoDbAsyncClient getAmazonDynamoAsyncDB() {
return amazonDynamoAsyncDB;
public DynamoDbAsyncClient getDynamoDbAsyncClient() {
return dynamoDbAsyncClient;
}

public AWSCredentialsProviderImpl getAwsCredentialsProvider() {
return awsCredentialsProvider;
public void close() {
if (dynamoDbClient != null) {
dynamoDbClient.close();
}
if (dynamoDbAsyncClient != null) {
dynamoDbAsyncClient.close();
}
if (awsCredentialsProvider != null) {
try {
awsCredentialsProvider.close();
} catch (Exception ignored) {
}
}
}
}
8 changes: 5 additions & 3 deletions ...modb_client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
public interface DynamoDBClientFetcher {

/**
* Returns a DynamoDBClient and the AWS credential provider used for authentication.
* The credentialProvider should be closed after DynamoDBClient is no longer needed.
* Returns a DynamoDBClient wrapper object that includes both regular
* and async clients along with the AWS credentials provider.
* The clients should be closed after DynamoDBClient is no
* longer needed which would close the associated AWS credential provider.
* (GC might not run for a long period of time)
* @param ztsClientNotificationSender notification sender object
* @param dynamoDBClientSettings contains private key store and client settings
* @return DynamoDBClientAndCredentials which contains both a DynamoDB client and the credentialProvider used
* @return DynamoDBClientAndCredentials which contains both DynamoDB clients and the credentialProvider used
*/
DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSender ztsClientNotificationSender,
DynamoDBClientSettings dynamoDBClientSettings);
Expand Down
90 changes: 37 additions & 53 deletions ..._client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientFetcherImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,27 @@

package com.yahoo.athenz.db.dynamodb;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.util.EC2MetadataUtils;
import com.oath.auth.KeyRefresher;
import com.oath.auth.Utils;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import com.yahoo.athenz.zts.ZTSClientNotificationSender;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.utils.StringUtils;

import javax.net.ssl.SSLContext;

public class DynamoDBClientFetcherImpl implements DynamoDBClientFetcher {
private static final Logger LOGGER = LoggerFactory.getLogger(DynamoDBClientFetcherImpl.class);
private String defaultAwsRegion;

public DynamoDBClientFetcherImpl() {
}

public DynamoDBClientFetcherImpl(String defaultAwsRegion) {
this.defaultAwsRegion = defaultAwsRegion;
}

@Override
public DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSender ztsClientNotificationSender,
DynamoDBClientSettings dynamoDBClientSettings) {
Expand All @@ -58,28 +52,29 @@ public DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSende
return getAuthenticatedDynamoDBClient(dynamoDBClientSettings, ztsClientNotificationSender);
} else {
LOGGER.info("DynamoDB client will use existing AWS authentication");
String region = getAWSRegion(dynamoDBClientSettings.getRegion());
AmazonDynamoDB client = AmazonDynamoDBClientBuilder
.standard()
.withRegion(region)
.build();

DynamoDbAsyncClient asyncClient = DynamoDbAsyncClient.builder()
.region(Region.of(region))
.build();
DynamoDbClient dynamoDbClient = null;
DynamoDbAsyncClient dynamoDbAsyncClient = null;
if (dynamoDBClientSettings.isAsyncClient()) {
dynamoDbAsyncClient = DynamoDbAsyncClient.builder()
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
} else {
dynamoDbClient = DynamoDbClient.builder()
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
}

return new DynamoDBClientAndCredentials(client, asyncClient, null);
return new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, null);
}
}

String getAWSRegion(final String settingRegion) {
if (StringUtils.isEmpty(settingRegion)) {
if (defaultAwsRegion == null) {
defaultAwsRegion = EC2MetadataUtils.getEC2InstanceRegion();
}
return defaultAwsRegion;
Region getAWSRegion(final String settingRegion) {
if (!StringUtils.isEmpty(settingRegion)) {
return Region.of(settingRegion);
} else {
return settingRegion;
DefaultAwsRegionProviderChain regionProvider = DefaultAwsRegionProviderChain.builder().build();
return regionProvider.getRegion();
}
}

Expand All @@ -102,10 +97,10 @@ private DynamoDBClientAndCredentials getAuthenticatedDynamoDBClient(DynamoDBClie
LOGGER.error("Failed to get AWS Temporary credentials", ex);
}

AWSCredentialsProviderImpl credentialsProvider = null;
String region = dynamoDBClientSettings.getRegion();
AWSCredentialsProviderImplV2 credentialsProvider = null;

try {
credentialsProvider = new AWSCredentialsProviderImpl(
credentialsProvider = new AWSCredentialsProviderImplV2(
dynamoDBClientSettings.getZtsURL(),
sslContext,
dynamoDBClientSettings.getDomainName(),
Expand All @@ -114,35 +109,24 @@ private DynamoDBClientAndCredentials getAuthenticatedDynamoDBClient(DynamoDBClie
dynamoDBClientSettings.getMinExpiryTime(),
dynamoDBClientSettings.getMaxExpiryTime(),
ztsClientNotificationSender);

} catch (Exception ex) {
LOGGER.error("Failed to generate AmazonDynamoDB client", ex);
LOGGER.error("Failed to generate DynamoDbClient client", ex);
}
AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
.withCredentials(credentialsProvider)
.withRegion(region)
.build();


AWSCredentialsProviderImplV2 credentialsProviderV2 = null;
try {
credentialsProviderV2 = new AWSCredentialsProviderImplV2(
dynamoDBClientSettings.getZtsURL(),
sslContext,
dynamoDBClientSettings.getDomainName(),
dynamoDBClientSettings.getRoleName(),
dynamoDBClientSettings.getExternalId(),
dynamoDBClientSettings.getMinExpiryTime(),
dynamoDBClientSettings.getMaxExpiryTime(),
ztsClientNotificationSender);
} catch (Exception ex) {
LOGGER.error("Failed to generate DynamoDbAsyncClient client", ex);
DynamoDbClient dynamoDbClient = null;
DynamoDbAsyncClient dynamoDbAsyncClient = null;
if (dynamoDBClientSettings.isAsyncClient()) {
dynamoDbAsyncClient = DynamoDbAsyncClient.builder()
.credentialsProvider(credentialsProvider)
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
} else {
dynamoDbClient = DynamoDbClient.builder()
.credentialsProvider(credentialsProvider)
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
}
DynamoDbAsyncClient asyncClient = DynamoDbAsyncClient.builder()
.credentialsProvider(credentialsProviderV2)
.region(Region.of(region))
.build();

return new DynamoDBClientAndCredentials(client, asyncClient, credentialsProvider);
return new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
}
}
33 changes: 15 additions & 18 deletions ...odb_client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,22 +34,14 @@ public class DynamoDBClientSettings {
private final String externalId;
private final Integer minExpiryTime;
private final Integer maxExpiryTime;
private final String keygroupName;

public DynamoDBClientSettings(String certPath,
String domainName,
String roleName,
String trustStore,
String trustStorePassword,
String ztsURL,
String region,
String keyPath,
String appName,
PrivateKeyStore keyStore,
String externalId,
Integer minExpiryTime,
Integer maxExpiryTime,
String keygroupName) {
private final String keyGroupName;
private final boolean isAsyncClient;

public DynamoDBClientSettings(String certPath, String domainName, String roleName, String trustStore,
String trustStorePassword, String ztsURL, String region, String keyPath, String appName,
PrivateKeyStore keyStore, String externalId, Integer minExpiryTime, Integer maxExpiryTime,
String keyGroupName, boolean isAsyncClient) {

this.certPath = certPath;
this.domainName = domainName;
this.roleName = roleName;
Expand All @@ -63,7 +55,8 @@ public DynamoDBClientSettings(String certPath,
this.externalId = externalId;
this.minExpiryTime = minExpiryTime;
this.maxExpiryTime = maxExpiryTime;
this.keygroupName = keygroupName;
this.keyGroupName = keyGroupName;
this.isAsyncClient = isAsyncClient;
}

public boolean areCredentialsProvided() {
Expand Down Expand Up @@ -123,6 +116,10 @@ public char[] getTrustStorePasswordChars() {
return null;
}

return keyStore.getSecret(appName, keygroupName, trustStorePassword);
return keyStore.getSecret(appName, keyGroupName, trustStorePassword);
}

public boolean isAsyncClient() {
return isAsyncClient;
}
}
55 changes: 44 additions & 11 deletions ..._factory/src/test/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientAndCredentialsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,61 @@

package com.yahoo.athenz.db.dynamodb;

import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import org.mockito.Mockito;
import org.testng.annotations.Test;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;

import java.io.IOException;

import static org.testng.Assert.assertEquals;

public class DynamoDBClientAndCredentialsTest {

@Test
public void testDynamoDBClientAndCredentials() {
public void testDynamoDBClientAndCredentials() throws IOException {

DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
AWSCredentialsProviderImplV2 credentialsProvider = Mockito.mock(AWSCredentialsProviderImplV2.class);
Mockito.doNothing().when(credentialsProvider).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}

AmazonDynamoDB amazonDynamoDB = Mockito.mock(AmazonDynamoDB.class);
DynamoDbAsyncClient amazonDynamoAsyncDB = Mockito.mock(DynamoDbAsyncClient.class);
AWSCredentialsProviderImpl awsCredentialsProvider = Mockito.mock(AWSCredentialsProviderImpl.class);
@Test
public void testDynamoDBClientAndCredentialsNullProvider() {

DynamoDBClientAndCredentials dynamoDBClientAndCredentials = new DynamoDBClientAndCredentials(
amazonDynamoDB, amazonDynamoAsyncDB, awsCredentialsProvider);
DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, null);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}

@Test
public void testDynamoDBClientAndCredentialsException() throws IOException {

assertEquals(amazonDynamoDB, dynamoDBClientAndCredentials.getAmazonDynamoDB());
assertEquals(amazonDynamoAsyncDB, dynamoDBClientAndCredentials.getAmazonDynamoAsyncDB());
assertEquals(awsCredentialsProvider, dynamoDBClientAndCredentials.getAwsCredentialsProvider());
DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
AWSCredentialsProviderImplV2 credentialsProvider = Mockito.mock(AWSCredentialsProviderImplV2.class);
Mockito.doThrow(new IllegalArgumentException()).when(credentialsProvider).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}
}
Loading
Loading