Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PLUGIN-1783] Adding Retry with dev.failsafe #35

Merged
merged 1 commit into from
May 13, 2024
+363 −66
Merged

[PLUGIN-1783] Adding Retry with dev.failsafe #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<apache.olingo.v2>2.0.0</apache.olingo.v2>
<wiremock.version>2.27.2</wiremock.version>
<hydrator.version>2.7.0</hydrator.version>
<failsafe.version>3.3.2</failsafe.version>
<testSourceLocation>${project.basedir}/src/test/java/</testSourceLocation>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
Expand All @@ -61,7 +62,11 @@
</repositories>

<dependencies>

<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-api</artifactId>
Expand Down Expand Up @@ -365,12 +370,6 @@
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>2.0.0</version>
</dependency>

</dependencies>

<build>
Expand Down
10 changes: 7 additions & 3 deletions src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ List<String> listEntities() throws TransportException, IOException {
URL dataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsEntity
(dataURL, MediaType.APPLICATION_JSON, METADATA);
(dataURL, MediaType.APPLICATION_JSON);
try (InputStream inputStream = responseContainer.getResponseStream()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
String result = reader.lines().collect(Collectors.joining(""));
Expand Down Expand Up @@ -225,7 +225,11 @@ private InputStream callEntityData(long top, String entityName)
addQueryParameter(TOP_OPTION, String.valueOf(top)).addQueryParameter(SELECT_OPTION, selectFields.toString())
.build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL);
SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsWithRetry(
dataURL, MediaType.APPLICATION_JSON, SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS,
SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS,
SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER, SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT);

ExceptionParser.checkAndThrowException("", responseContainer);
return responseContainer.getResponseStream();
Expand Down Expand Up @@ -255,7 +259,7 @@ private InputStream getMetaDataStream(String entity) throws TransportException,
.addPathSegment(METADATACALL).build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient
.callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML, METADATA);
.callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML);
return responseContainer.getResponseStream();
}
}
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.cdap.cdap.api.annotation.Name;
import io.cdap.cdap.api.plugin.PluginConfig;
import io.cdap.cdap.etl.api.FailureCollector;
import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException;
import io.cdap.plugin.successfactors.common.exception.TransportException;
import io.cdap.plugin.successfactors.common.util.ResourceConstants;
import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil;
Expand Down Expand Up @@ -153,7 +152,7 @@ public void validateConnection(FailureCollector collector) {
SuccessFactorsResponseContainer responseContainer = null;
try {
responseContainer =
successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON, TEST);
successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON);
} catch (TransportException e) {
LOG.error("Unable to fetch the response", e);
collector.addFailure("Unable to call SuccessFactorsEntity",
Expand Down
120 changes: 118 additions & 2 deletions src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ public class SuccessFactorsPluginConfig extends PluginConfig {
private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey();
private static final Pattern PATTERN = Pattern.compile("\\(.*\\)");
private static final String SAP_SUCCESSFACTORS_ENTITY_NAME = "Entity Name";
private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration";
private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration";
private static final String NAME_RETRY_MULTIPLIER = "retryMultiplier";
private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount";
public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 2;
public static final int DEFAULT_RETRY_MULTIPLIER = 2;
public static final int DEFAULT_MAX_RETRY_COUNT = 3;
public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 10;

@Macro
@Name(ENTITY_NAME)
Expand Down Expand Up @@ -128,6 +136,30 @@ public class SuccessFactorsPluginConfig extends PluginConfig {
@Description("The existing connection to use.")
private SuccessFactorsConnectorConfig connection;

@Name(NAME_INITIAL_RETRY_DURATION)
@Description("Time taken for the first retry. Default is 2 seconds.")
@Nullable
@Macro
private Integer initialRetryDuration;

@Name(NAME_MAX_RETRY_DURATION)
@Description("Maximum time in seconds retries can take. Default is 300 seconds.")
@Nullable
@Macro
private Integer maxRetryDuration;

@Name(NAME_MAX_RETRY_COUNT)
@Description("Maximum number of retries allowed. Default is 3.")
@Nullable
@Macro
private Integer maxRetryCount;

@Name(NAME_RETRY_MULTIPLIER)
@Description("Multiplier for exponential backoff. Default is 2.")
@Nullable
@Macro
private Integer retryMultiplier;

@VisibleForTesting
public SuccessFactorsPluginConfig(String referenceName,
String baseURL,
Expand All @@ -142,7 +174,11 @@ public SuccessFactorsPluginConfig(String referenceName,
@Nullable String selectOption,
@Nullable String expandOption,
@Nullable String additionalQueryParameters,
String paginationType) {
String paginationType,
@Nullable Integer initialRetryDuration,
@Nullable Integer maxRetryDuration,
@Nullable Integer retryMultiplier,
@Nullable Integer maxRetryCount) {
this.connection = new SuccessFactorsConnectorConfig(username, password, baseURL, proxyUrl, proxyPassword,
proxyUsername);
this.referenceName = referenceName;
Expand All @@ -153,6 +189,10 @@ public SuccessFactorsPluginConfig(String referenceName,
this.expandOption = expandOption;
this.paginationType = paginationType;
this.additionalQueryParameters = additionalQueryParameters;
this.initialRetryDuration = initialRetryDuration;
this.maxRetryDuration = maxRetryDuration;
this.retryMultiplier = retryMultiplier;
this.maxRetryCount = maxRetryCount;
}
@Nullable
public SuccessFactorsConnectorConfig getConnection() {
Expand Down Expand Up @@ -210,6 +250,22 @@ public String getAdditionalQueryParameters() {
return this.additionalQueryParameters;
}

public int getInitialRetryDuration() {
return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration;
}

public int getMaxRetryDuration() {
return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration;
}

public int getRetryMultiplier() {
return retryMultiplier == null ? DEFAULT_RETRY_MULTIPLIER : retryMultiplier;
}

public int getMaxRetryCount() {
return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount;
}

/**
* Checks if the call to SuccessFactors service is required for metadata creation.
* condition parameters: ['host' | 'serviceName' | 'entityName' | 'username' | 'password']
Expand Down Expand Up @@ -243,6 +299,7 @@ public void validatePluginParameters(FailureCollector failureCollector) {
validateMandatoryParameters(failureCollector);
validateBasicCredentials(failureCollector);
validateEntityParameter(failureCollector);
validateRetryConfiguration(failureCollector);
failureCollector.getOrThrowException();
}

Expand Down Expand Up @@ -292,6 +349,43 @@ private void validateEntityParameter(FailureCollector failureCollector) {
}
}

/**
* Validates the retry configuration.
*
* @param failureCollector {@code FailureCollector}
*/
public void validateRetryConfiguration(FailureCollector failureCollector) {
if (containsMacro(NAME_INITIAL_RETRY_DURATION) || containsMacro(NAME_MAX_RETRY_DURATION) ||
containsMacro(NAME_MAX_RETRY_COUNT) || containsMacro(NAME_RETRY_MULTIPLIER)) {
return;
}
if (initialRetryDuration != null && initialRetryDuration <= 0) {
failureCollector.addFailure("Initial retry duration must be greater than 0.",
"Please specify a valid initial retry duration.")
.withConfigProperty(NAME_INITIAL_RETRY_DURATION);
}
if (maxRetryDuration != null && maxRetryDuration <= 0) {
failureCollector.addFailure("Max retry duration must be greater than 0.",
"Please specify a valid max retry duration.")
.withConfigProperty(NAME_MAX_RETRY_DURATION);
}
if (maxRetryCount != null && maxRetryCount <= 0) {
failureCollector.addFailure("Max retry count must be greater than 0.",
"Please specify a valid max retry count.")
.withConfigProperty(NAME_MAX_RETRY_COUNT);
}
if (retryMultiplier != null && retryMultiplier <= 1) {
failureCollector.addFailure("Retry multiplier must be strictly greater than 1.",
"Please specify a valid retry multiplier.")
.withConfigProperty(NAME_RETRY_MULTIPLIER);
}
if (maxRetryDuration != null && initialRetryDuration != null && maxRetryDuration <= initialRetryDuration) {
failureCollector.addFailure("Max retry duration must be greater than initial retry duration.",
"Please specify a valid max retry duration.")
.withConfigProperty(NAME_MAX_RETRY_DURATION);
}
}

/**
* Helper class to simplify {@link SuccessFactorsPluginConfig} class creation.
*/
Expand All @@ -310,6 +404,10 @@ public static class Builder {
private String proxyUrl;
private String proxyUsername;
private String proxyPassword;
private Integer initialRetryDuration;
private Integer maxRetryDuration;
private Integer retryMultiplier;
private Integer maxRetryCount;

public Builder referenceName(String referenceName) {
this.referenceName = referenceName;
Expand Down Expand Up @@ -379,11 +477,29 @@ public Builder additionalQueryParameters(@Nullable String additionalQueryParamet
return this;
}

public Builder setInitialRetryDuration(Integer initialRetryDuration) {
this.initialRetryDuration = initialRetryDuration;
return this;
}
public Builder setMaxRetryDuration(Integer maxRetryDuration) {
this.maxRetryDuration = maxRetryDuration;
return this;
}
public Builder setRetryMultiplier(Integer retryMultiplier) {
this.retryMultiplier = retryMultiplier;
return this;
}
public Builder setMaxRetryCount(Integer maxRetryCount) {
this.maxRetryCount = maxRetryCount;
return this;
}

public SuccessFactorsPluginConfig build() {
return new SuccessFactorsPluginConfig(referenceName, baseURL, entityName, associateEntityName, username,
password, proxyUrl, proxyUsername, proxyPassword,
filterOption, selectOption, expandOption, additionalQueryParameters,
paginationType);
paginationType, initialRetryDuration, maxRetryDuration,
retryMultiplier, maxRetryCount);
}
}
}
13 changes: 9 additions & 4 deletions src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public SuccessFactorsService(SuccessFactorsPluginConfig pluginConfig,
public void checkSuccessFactorsURL() throws TransportException, SuccessFactorsServiceException {

SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON, TEST);
successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON);

ExceptionParser.checkAndThrowException(ResourceConstants.ERR_FAILED_ENTITY_VALIDATION.getMsgForKey(),
responseContainer);
Expand Down Expand Up @@ -163,7 +163,9 @@ private SuccessFactorsEntityProvider fetchServiceMetadata(InputStream metadataSt
*/
private InputStream callEntityMetadata() throws TransportException {
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient
.callSuccessFactorsEntity(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, METADATA);
.callSuccessFactorsWithRetry(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, pluginConfig
.getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(), pluginConfig.getRetryMultiplier(),
pluginConfig.getMaxRetryCount());
return responseContainer.getResponseStream();
}

Expand Down Expand Up @@ -320,7 +322,10 @@ private InputStream callEntityData(@Nullable Long skip, @Nullable Long top)
} else {
dataURL = urlContainer.getDataFetchURL(skip, top);
}
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL);
SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsWithRetry(
dataURL, MediaType.APPLICATION_JSON, pluginConfig.getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(),
pluginConfig.getRetryMultiplier(), pluginConfig.getMaxRetryCount());

ExceptionParser.checkAndThrowException("", responseContainer);
return responseContainer.getResponseStream();
Expand All @@ -337,7 +342,7 @@ public List<String> getNonNavigationalProperties() throws TransportException, Su

/**
* Filter the data stream after removing the expanded entity data.
*
*
* Data stream after conversion to JSON has the following format:
* "d": {
* "results": [
Expand Down
Loading