Skip to content

Commit

Permalink
zts/syncer dynamodb migration from aws v1 to v2 (#2688)
Browse files Browse the repository at this point in the history 
ddb update

Signed-off-by: Henry Avetisyan <hga@yahooinc.com>
  • Loading branch information
@havetisyan
havetisyan committed Aug 18, 2024
1 parent c0b66af commit 6c4e5f0
Show file tree
Hide file tree
Showing 47 changed files with 1,606 additions and 1,649 deletions.
25 changes: 2 additions & 23 deletions libs/java/dynamodb_client_factory/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,9 @@
<name>athenz-dynamodb-client-factory</name>
<description>Generate dynamodb client based on AWS temporary credentials</description>
<packaging>jar</packaging>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>${aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws2.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<properties>
<code.coverage.min>0.9230</code.coverage.min>
<code.coverage.min>0.9417</code.coverage.min>
</properties>

<dependencies>
Expand All @@ -60,13 +42,10 @@
<artifactId>athenz-zts-java-client</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>dynamodb</artifactId>
<version>${aws2.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down
41 changes: 27 additions & 14 deletions ...ient_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientAndCredentials.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,30 +18,43 @@

package com.yahoo.athenz.db.dynamodb;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;

public class DynamoDBClientAndCredentials {
private final AmazonDynamoDB amazonDynamoDB;
private final DynamoDbAsyncClient amazonDynamoAsyncDB;
private final AWSCredentialsProviderImpl awsCredentialsProvider;

public DynamoDBClientAndCredentials(AmazonDynamoDB amazonDynamoDB, DynamoDbAsyncClient amazonDynamoAsyncDB, AWSCredentialsProviderImpl awsCredentialsProvider) {
this.amazonDynamoDB = amazonDynamoDB;
this.amazonDynamoAsyncDB = amazonDynamoAsyncDB;
private final DynamoDbClient dynamoDbClient;
private final DynamoDbAsyncClient dynamoDbAsyncClient;
private final AWSCredentialsProviderImplV2 awsCredentialsProvider;

public DynamoDBClientAndCredentials(DynamoDbClient dynamoDbClient, DynamoDbAsyncClient dynamoDbAsyncClient,
AWSCredentialsProviderImplV2 awsCredentialsProvider) {
this.dynamoDbClient = dynamoDbClient;
this.dynamoDbAsyncClient = dynamoDbAsyncClient;
this.awsCredentialsProvider = awsCredentialsProvider;
}

public AmazonDynamoDB getAmazonDynamoDB() {
return amazonDynamoDB;
public DynamoDbClient getDynamoDbClient() {
return dynamoDbClient;
}

public DynamoDbAsyncClient getAmazonDynamoAsyncDB() {
return amazonDynamoAsyncDB;
public DynamoDbAsyncClient getDynamoDbAsyncClient() {
return dynamoDbAsyncClient;
}

public AWSCredentialsProviderImpl getAwsCredentialsProvider() {
return awsCredentialsProvider;
public void close() {
if (dynamoDbClient != null) {
dynamoDbClient.close();
}
if (dynamoDbAsyncClient != null) {
dynamoDbAsyncClient.close();
}
if (awsCredentialsProvider != null) {
try {
awsCredentialsProvider.close();
} catch (Exception ignored) {
}
}
}
}
8 changes: 5 additions & 3 deletions ...modb_client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientFetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
public interface DynamoDBClientFetcher {

/**
* Returns a DynamoDBClient and the AWS credential provider used for authentication.
* The credentialProvider should be closed after DynamoDBClient is no longer needed.
* Returns a DynamoDBClient wrapper object that includes both regular
* and async clients along with the AWS credentials provider.
* The clients should be closed after DynamoDBClient is no
* longer needed which would close the associated AWS credential provider.
* (GC might not run for a long period of time)
* @param ztsClientNotificationSender notification sender object
* @param dynamoDBClientSettings contains private key store and client settings
* @return DynamoDBClientAndCredentials which contains both a DynamoDB client and the credentialProvider used
* @return DynamoDBClientAndCredentials which contains both DynamoDB clients and the credentialProvider used
*/
DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSender ztsClientNotificationSender,
DynamoDBClientSettings dynamoDBClientSettings);
Expand Down
90 changes: 37 additions & 53 deletions ..._client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientFetcherImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,27 @@

package com.yahoo.athenz.db.dynamodb;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.util.EC2MetadataUtils;
import com.oath.auth.KeyRefresher;
import com.oath.auth.Utils;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import com.yahoo.athenz.zts.ZTSClientNotificationSender;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.utils.StringUtils;

import javax.net.ssl.SSLContext;

public class DynamoDBClientFetcherImpl implements DynamoDBClientFetcher {
private static final Logger LOGGER = LoggerFactory.getLogger(DynamoDBClientFetcherImpl.class);
private String defaultAwsRegion;

public DynamoDBClientFetcherImpl() {
}

public DynamoDBClientFetcherImpl(String defaultAwsRegion) {
this.defaultAwsRegion = defaultAwsRegion;
}

@Override
public DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSender ztsClientNotificationSender,
DynamoDBClientSettings dynamoDBClientSettings) {
Expand All @@ -58,28 +52,29 @@ public DynamoDBClientAndCredentials getDynamoDBClient(ZTSClientNotificationSende
return getAuthenticatedDynamoDBClient(dynamoDBClientSettings, ztsClientNotificationSender);
} else {
LOGGER.info("DynamoDB client will use existing AWS authentication");
String region = getAWSRegion(dynamoDBClientSettings.getRegion());
AmazonDynamoDB client = AmazonDynamoDBClientBuilder
.standard()
.withRegion(region)
.build();

DynamoDbAsyncClient asyncClient = DynamoDbAsyncClient.builder()
.region(Region.of(region))
.build();
DynamoDbClient dynamoDbClient = null;
DynamoDbAsyncClient dynamoDbAsyncClient = null;
if (dynamoDBClientSettings.isAsyncClient()) {
dynamoDbAsyncClient = DynamoDbAsyncClient.builder()
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
} else {
dynamoDbClient = DynamoDbClient.builder()
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
}

return new DynamoDBClientAndCredentials(client, asyncClient, null);
return new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, null);
}
}

String getAWSRegion(final String settingRegion) {
if (StringUtils.isEmpty(settingRegion)) {
if (defaultAwsRegion == null) {
defaultAwsRegion = EC2MetadataUtils.getEC2InstanceRegion();
}
return defaultAwsRegion;
Region getAWSRegion(final String settingRegion) {
if (!StringUtils.isEmpty(settingRegion)) {
return Region.of(settingRegion);
} else {
return settingRegion;
DefaultAwsRegionProviderChain regionProvider = DefaultAwsRegionProviderChain.builder().build();
return regionProvider.getRegion();
}
}

Expand All @@ -102,10 +97,10 @@ private DynamoDBClientAndCredentials getAuthenticatedDynamoDBClient(DynamoDBClie
LOGGER.error("Failed to get AWS Temporary credentials", ex);
}

AWSCredentialsProviderImpl credentialsProvider = null;
String region = dynamoDBClientSettings.getRegion();
AWSCredentialsProviderImplV2 credentialsProvider = null;

try {
credentialsProvider = new AWSCredentialsProviderImpl(
credentialsProvider = new AWSCredentialsProviderImplV2(
dynamoDBClientSettings.getZtsURL(),
sslContext,
dynamoDBClientSettings.getDomainName(),
Expand All @@ -114,35 +109,24 @@ private DynamoDBClientAndCredentials getAuthenticatedDynamoDBClient(DynamoDBClie
dynamoDBClientSettings.getMinExpiryTime(),
dynamoDBClientSettings.getMaxExpiryTime(),
ztsClientNotificationSender);

} catch (Exception ex) {
LOGGER.error("Failed to generate AmazonDynamoDB client", ex);
LOGGER.error("Failed to generate DynamoDbClient client", ex);
}
AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard()
.withCredentials(credentialsProvider)
.withRegion(region)
.build();


AWSCredentialsProviderImplV2 credentialsProviderV2 = null;
try {
credentialsProviderV2 = new AWSCredentialsProviderImplV2(
dynamoDBClientSettings.getZtsURL(),
sslContext,
dynamoDBClientSettings.getDomainName(),
dynamoDBClientSettings.getRoleName(),
dynamoDBClientSettings.getExternalId(),
dynamoDBClientSettings.getMinExpiryTime(),
dynamoDBClientSettings.getMaxExpiryTime(),
ztsClientNotificationSender);
} catch (Exception ex) {
LOGGER.error("Failed to generate DynamoDbAsyncClient client", ex);
DynamoDbClient dynamoDbClient = null;
DynamoDbAsyncClient dynamoDbAsyncClient = null;
if (dynamoDBClientSettings.isAsyncClient()) {
dynamoDbAsyncClient = DynamoDbAsyncClient.builder()
.credentialsProvider(credentialsProvider)
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
} else {
dynamoDbClient = DynamoDbClient.builder()
.credentialsProvider(credentialsProvider)
.region(getAWSRegion(dynamoDBClientSettings.getRegion()))
.build();
}
DynamoDbAsyncClient asyncClient = DynamoDbAsyncClient.builder()
.credentialsProvider(credentialsProviderV2)
.region(Region.of(region))
.build();

return new DynamoDBClientAndCredentials(client, asyncClient, credentialsProvider);
return new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
}
}
33 changes: 15 additions & 18 deletions ...odb_client_factory/src/main/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,22 +34,14 @@ public class DynamoDBClientSettings {
private final String externalId;
private final Integer minExpiryTime;
private final Integer maxExpiryTime;
private final String keygroupName;

public DynamoDBClientSettings(String certPath,
String domainName,
String roleName,
String trustStore,
String trustStorePassword,
String ztsURL,
String region,
String keyPath,
String appName,
PrivateKeyStore keyStore,
String externalId,
Integer minExpiryTime,
Integer maxExpiryTime,
String keygroupName) {
private final String keyGroupName;
private final boolean isAsyncClient;

public DynamoDBClientSettings(String certPath, String domainName, String roleName, String trustStore,
String trustStorePassword, String ztsURL, String region, String keyPath, String appName,
PrivateKeyStore keyStore, String externalId, Integer minExpiryTime, Integer maxExpiryTime,
String keyGroupName, boolean isAsyncClient) {

this.certPath = certPath;
this.domainName = domainName;
this.roleName = roleName;
Expand All @@ -63,7 +55,8 @@ public DynamoDBClientSettings(String certPath,
this.externalId = externalId;
this.minExpiryTime = minExpiryTime;
this.maxExpiryTime = maxExpiryTime;
this.keygroupName = keygroupName;
this.keyGroupName = keyGroupName;
this.isAsyncClient = isAsyncClient;
}

public boolean areCredentialsProvided() {
Expand Down Expand Up @@ -123,6 +116,10 @@ public char[] getTrustStorePasswordChars() {
return null;
}

return keyStore.getSecret(appName, keygroupName, trustStorePassword);
return keyStore.getSecret(appName, keyGroupName, trustStorePassword);
}

public boolean isAsyncClient() {
return isAsyncClient;
}
}
55 changes: 44 additions & 11 deletions ..._factory/src/test/java/com/yahoo/athenz/db/dynamodb/DynamoDBClientAndCredentialsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,61 @@

package com.yahoo.athenz.db.dynamodb;

import com.yahoo.athenz.zts.AWSCredentialsProviderImplV2;
import org.mockito.Mockito;
import org.testng.annotations.Test;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.yahoo.athenz.zts.AWSCredentialsProviderImpl;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;

import java.io.IOException;

import static org.testng.Assert.assertEquals;

public class DynamoDBClientAndCredentialsTest {

@Test
public void testDynamoDBClientAndCredentials() {
public void testDynamoDBClientAndCredentials() throws IOException {

DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
AWSCredentialsProviderImplV2 credentialsProvider = Mockito.mock(AWSCredentialsProviderImplV2.class);
Mockito.doNothing().when(credentialsProvider).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}

AmazonDynamoDB amazonDynamoDB = Mockito.mock(AmazonDynamoDB.class);
DynamoDbAsyncClient amazonDynamoAsyncDB = Mockito.mock(DynamoDbAsyncClient.class);
AWSCredentialsProviderImpl awsCredentialsProvider = Mockito.mock(AWSCredentialsProviderImpl.class);
@Test
public void testDynamoDBClientAndCredentialsNullProvider() {

DynamoDBClientAndCredentials dynamoDBClientAndCredentials = new DynamoDBClientAndCredentials(
amazonDynamoDB, amazonDynamoAsyncDB, awsCredentialsProvider);
DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, null);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}

@Test
public void testDynamoDBClientAndCredentialsException() throws IOException {

assertEquals(amazonDynamoDB, dynamoDBClientAndCredentials.getAmazonDynamoDB());
assertEquals(amazonDynamoAsyncDB, dynamoDBClientAndCredentials.getAmazonDynamoAsyncDB());
assertEquals(awsCredentialsProvider, dynamoDBClientAndCredentials.getAwsCredentialsProvider());
DynamoDbClient dynamoDbClient = Mockito.mock(DynamoDbClient.class);
Mockito.doNothing().when(dynamoDbClient).close();
DynamoDbAsyncClient dynamoDbAsyncClient = Mockito.mock(DynamoDbAsyncClient.class);
Mockito.doNothing().when(dynamoDbAsyncClient).close();
AWSCredentialsProviderImplV2 credentialsProvider = Mockito.mock(AWSCredentialsProviderImplV2.class);
Mockito.doThrow(new IllegalArgumentException()).when(credentialsProvider).close();
DynamoDBClientAndCredentials dynamoDBClientAndCredentials =
new DynamoDBClientAndCredentials(dynamoDbClient, dynamoDbAsyncClient, credentialsProvider);
assertEquals(dynamoDbClient, dynamoDBClientAndCredentials.getDynamoDbClient());
assertEquals(dynamoDbAsyncClient, dynamoDBClientAndCredentials.getDynamoDbAsyncClient());
dynamoDBClientAndCredentials.close();
}
}
Loading

0 comments on commit 6c4e5f0

Please sign in to comment.