diff --git a/pom.xml b/pom.xml index 691125d..6427907 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ 27.0.1-jre 2.6.0 1.10 + 3.3.2 @@ -60,6 +61,11 @@ + + dev.failsafe + failsafe + ${failsafe.version} + com.google.guava guava diff --git a/src/main/java/io/cdap/plugin/ariba/source/AribaBatchSource.java b/src/main/java/io/cdap/plugin/ariba/source/AribaBatchSource.java index d628ad2..b706c55 100644 --- a/src/main/java/io/cdap/plugin/ariba/source/AribaBatchSource.java +++ b/src/main/java/io/cdap/plugin/ariba/source/AribaBatchSource.java @@ -32,7 +32,6 @@ import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.connector.Connector; import io.cdap.plugin.ariba.source.config.AribaPluginConfig; -import io.cdap.plugin.ariba.source.connector.AribaConnector; import io.cdap.plugin.ariba.source.exception.AribaException; import io.cdap.plugin.ariba.source.util.AribaUtil; import io.cdap.plugin.ariba.source.util.ResourceConstants; @@ -69,7 +68,12 @@ public class AribaBatchSource extends BatchSource getSplits(JobContext jobContext) throws IOException { AribaPluginConfig pluginConfig = getPluginConfig(jobContext); - AribaServices aribaServices = new AribaServices(pluginConfig.getConnection()); + AribaServices aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), + true); boolean previewEnabled = Boolean.parseBoolean(jobContext.getConfiguration(). get(ResourceConstants.IS_PREVIEW_ENABLED)); @@ -66,7 +71,12 @@ public List getSplits(JobContext jobContext) throws IOException { public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { AribaPluginConfig pluginConfig = getPluginConfig(taskAttemptContext); - AribaServices aribaServices = new AribaServices(pluginConfig.getConnection()); + AribaServices aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), + true); Schema outputSchema = Schema.parseJson(taskAttemptContext.getConfiguration().get(ResourceConstants.OUTPUT_SCHEMA)); return new AribaRecordReader(aribaServices, outputSchema, pluginConfig); } diff --git a/src/main/java/io/cdap/plugin/ariba/source/AribaServices.java b/src/main/java/io/cdap/plugin/ariba/source/AribaServices.java index f089e7c..f649247 100644 --- a/src/main/java/io/cdap/plugin/ariba/source/AribaServices.java +++ b/src/main/java/io/cdap/plugin/ariba/source/AribaServices.java @@ -21,10 +21,14 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.ariba.source.config.AribaPluginConfig; import io.cdap.plugin.ariba.source.connector.AribaConnectorConfig; import io.cdap.plugin.ariba.source.exception.AribaException; +import io.cdap.plugin.ariba.source.exception.AribaRetryableException; import io.cdap.plugin.ariba.source.metadata.AribaColumnMetadata; import io.cdap.plugin.ariba.source.metadata.AribaResponseContainer; import io.cdap.plugin.ariba.source.metadata.AribaSchemaGenerator; @@ -57,8 +61,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.zip.ZipInputStream; import javax.annotation.Nullable; import javax.ws.rs.core.MediaType; @@ -100,18 +102,36 @@ public class AribaServices { private static final String TYPE = "type"; private static final String UTC = "UTC"; private static final Logger LOG = LoggerFactory.getLogger(AribaServices.class); - private static final int MAX_RETRIES = 5; private final AribaConnectorConfig pluginConfig; private final ObjectMapper objectMapper = new ObjectMapper(); private final Gson gson = new Gson(); private int availableLimit; - private boolean isDayLimitExhausted; - private boolean isHourLimitExhausted; - private boolean isMinuteLimitExhausted; - private boolean isSecondsLimitExhausted; + boolean isDayLimitExhausted; + boolean isHourLimitExhausted; + boolean isMinuteLimitExhausted; + boolean isSecondsLimitExhausted; - public AribaServices(AribaConnectorConfig pluginConfig) { + private final Integer initialRetryDuration; + private final Integer maxRetryDuration; + private final Integer maxRetryCount; + private final Integer retryMultiplier; + + /** + * Determines if retry is required for the service call. + * If true, then the service call will be retried based on the retry configuration. + * If false, then the service call will not be retried. + */ + private final boolean retryRequired; + + public AribaServices(AribaConnectorConfig pluginConfig, Integer maxRetryCount, + Integer initialRetryDuration, Integer maxRetryDuration, Integer retryMultiplier, + boolean retryRequired) { this.pluginConfig = pluginConfig; + this.maxRetryCount = maxRetryCount; + this.initialRetryDuration = initialRetryDuration; + this.maxRetryDuration = maxRetryDuration; + this.retryMultiplier = retryMultiplier; + this.retryRequired = retryRequired; } /** @@ -366,16 +386,7 @@ public List getMetadata(String accessToken, String template public JsonNode createJob(AribaPluginConfig aribaPluginConfig, @Nullable String pageToken, String templateName) throws AribaException, IOException, InterruptedException { Request req = buildJobRequest(jobBuilder(pageToken).build().url(), aribaPluginConfig, templateName); - int count = 0; - Response response; - do { - response = executeRequest(req); - if (response.code() == ResourceConstants.API_LIMIT_EXCEED) { - checkAndThrowException(response); - count++; - } - } while (response.code() == ResourceConstants.API_LIMIT_EXCEED && count <= MAX_RETRIES); - + Response response = executeRequest(req); AribaResponseContainer responseContainer = tokenResponse(response); InputStream responseStream = responseContainer.getResponseBody(); if (responseContainer.getHttpStatusCode() == HttpURLConnection.HTTP_OK) { @@ -394,17 +405,11 @@ public JsonNode createJob(AribaPluginConfig aribaPluginConfig, @Nullable String public JsonNode fetchJobStatus(String accessToken, String jobId) throws IOException, AribaException, InterruptedException { URL url = fetchDataBuilder(jobId).build().url(); - int count = 0; Request req = buildFetchRequest(url, accessToken); Response response = null; try { - do { - response = executeRequest(req); - checkAndThrowException(response); - count++; - } while (response.code() == ResourceConstants.API_LIMIT_EXCEED && count <= MAX_RETRIES); + response = executeRequest(req); AribaResponseContainer responseContainer = tokenResponse(response); - if (responseContainer.getHttpStatusCode() == HttpURLConnection.HTTP_OK) { InputStream responseStream = responseContainer.getResponseBody(); JsonNode responseNode = objectMapper.readTree(responseStream); @@ -418,7 +423,9 @@ public JsonNode fetchJobStatus(String accessToken, String jobId) } throw new AribaException(response.message(), response.code()); } finally { - response.close(); + if (response != null) { + response.close(); + } } } @@ -621,6 +628,8 @@ protected Request buildFetchRequest(URL endPoint, String accessToken) { } /** + * Executes the given Ariba request and returns the response. + * * @param req request * @return Response * @throws AribaException AribaException @@ -628,52 +637,65 @@ protected Request buildFetchRequest(URL endPoint, String accessToken) { * @throws IOException IOException */ public Response executeRequest(Request req) throws AribaException, InterruptedException, IOException { - OkHttpClient enhancedOkHttpClient = getConfiguredClient().build(); - Response response = null; + int actualMaxRetryCount = retryRequired ? maxRetryCount : 0; + RetryPolicy retryPolicy = RetryPolicy.builder() + .handle(AribaRetryableException.class) + .withBackoff(initialRetryDuration, maxRetryDuration, ChronoUnit.SECONDS, retryMultiplier) + .withMaxRetries(actualMaxRetryCount) + .onRetry(event -> LOG.info("Retrying Ariba call with plugin. Retry count: " + event.getAttemptCount())) + .onSuccess(event -> LOG.debug("Ariba plugin call has been executed successfully.")) + .onRetriesExceeded(event -> LOG.error("Retry limit for the Ariba plugin has been exceeded.", + event.getException())) + .build(); + try { - response = enhancedOkHttpClient.newCall(req).execute(); - if (response.code() != HttpURLConnection.HTTP_OK && AribaUtil.isNullOrEmpty(response.message())) { - AribaResponseContainer responseContainer = aribaResponse(response); - InputStream responseStream = responseContainer.getResponseBody(); - JsonNode jsonNode = objectMapper.readTree(responseStream); - String errMsg = jsonNode.get(ResourceConstants.MESSAGE).asText() != null - ? jsonNode.get(ResourceConstants.MESSAGE).asText() : - ResourceConstants.ERR_NOT_FOUND.getMsgForKey(); - throw new AribaException(errMsg, response.code()); - } - if (!isApiLimitExhausted(response)) { - checkAndThrowException(response); - return response; - } else { - response = enhancedOkHttpClient.newCall(req).execute(); + return Failsafe.with(retryPolicy).get(() -> executeRetryableRequest(req, actualMaxRetryCount > 0)); + } catch (FailsafeException fse) { + Throwable t = fse.getCause(); + if (t instanceof AribaException) { + throw (AribaException) t; + } else if (t instanceof InterruptedException) { + throw (InterruptedException) t; + } else if (t instanceof IOException) { + throw (IOException) t; + } else { + throw new RuntimeException(t); + } } - } catch (IOException e) { - throw new IOException("Endpoint is incorrect. Unable to validate the source with the provided.", e); } + /** + * Calls given Ariba API. + * @param req request + * @param shouldWait Do we to wait for time defined in header if API limit is exhausted + * @return Response + * @throws AribaException AribaException + * @throws InterruptedException InterruptedException + * @throws IOException IOException + * @throws AribaRetryableException + */ + public Response executeRetryableRequest(Request req, boolean shouldWait) + throws AribaException, InterruptedException, IOException, AribaRetryableException { + + LOG.debug("Retryable Ariba URL: " + req.url()); + OkHttpClient enhancedOkHttpClient = getConfiguredClient().build(); + Response response = enhancedOkHttpClient.newCall(req).execute(); + checkAndThrowException(response, shouldWait); return response; -} + } -/** + /** + * Calls given Ariba API. * @param jobId Ariba Job Id * @return JsonNode */ public JsonNode fetchData(String jobId, String fileName) throws IOException, InterruptedException, AribaException { - OkHttpClient enhancedOkHttpClient = getConfiguredClient().build(); HttpUrl.Builder zipUrl = zipBuilder(jobId, fileName); - Response zipResponse; - do { - zipResponse = enhancedOkHttpClient - .newCall(fetchZipFileData(zipUrl.build().url(), getAccessToken())).execute(); - if (zipResponse.code() == ResourceConstants.API_LIMIT_EXCEED) { - checkAndThrowException(zipResponse); - } - } while (zipResponse.code() == ResourceConstants.API_LIMIT_EXCEED); + Response zipResponse = executeRequest(fetchZipFileData(zipUrl.build().url(), getAccessToken())); - LOG.info("Fetch Data Response Code is: {} for Job Id: {} , and File: {}" - , zipResponse.code(), jobId, fileName); + LOG.info("Fetch Data Response Code is: {} for Job Id: {} , and File: {}", zipResponse.code(), jobId, fileName); AribaResponseContainer responseContainer = tokenResponse(zipResponse); try (InputStream responseStream = responseContainer.getResponseBody(); @@ -695,6 +717,7 @@ public JsonNode fetchData(String jobId, String fileName) * @return boolean */ public boolean isApiLimitExhausted(Response response) { + isDayLimitExhausted = isHourLimitExhausted = isMinuteLimitExhausted = isSecondsLimitExhausted = false; if (response.code() != HttpURLConnection.HTTP_OK && Integer.parseInt(Objects.requireNonNull(response.header(RATE_LIMIT_DAY))) < 1) { isDayLimitExhausted = true; @@ -719,14 +742,17 @@ public boolean isApiLimitExhausted(Response response) { * Check for limit and status code than throws exception accordingly * * @param response response + * @param shouldWait Do we to wait for time defined in header if API limit is exhausted * @throws AribaException * @throws InterruptedException */ @VisibleForTesting - void checkAndThrowException(Response response) throws AribaException, InterruptedException { + void checkAndThrowException(Response response, boolean shouldWait) throws AribaException, InterruptedException, + AribaRetryableException { if (response.code() == HttpURLConnection.HTTP_BAD_REQUEST && !AribaUtil.isNullOrEmpty(response.message())) { throw new AribaException(response.message(), response.code()); } + boolean limitExhausted = isApiLimitExhausted(response); if (limitExhausted && isDayLimitExhausted) { @@ -736,20 +762,35 @@ void checkAndThrowException(Response response) throws AribaException, Interrupte throw new AribaException(ResourceConstants.ERR_API_LIMIT_EXCEED_FOR_DAY.getMsgForKey(retryAfter), ResourceConstants.LIMIT_EXCEED_ERROR_CODE); } else if (limitExhausted && isHourLimitExhausted) { - int retryAfter = - (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER))) / 60) + 1; - LOG.info("API rate limit exceeded for the Hour, waiting for {} min", retryAfter); - TimeUnit.MINUTES.sleep(retryAfter); + if (shouldWait) { + int retryAfter = + (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER))) / 60) + 1; + LOG.info("API rate limit exceeded for the Hour, waiting for {} min", retryAfter); + TimeUnit.MINUTES.sleep(retryAfter); + } + String errorMsg = String.format("Call to Ariba failed. Status Code: %s, Root Cause: %s.", response.code(), + response.message()); + throw new AribaRetryableException(errorMsg, response.code()); } else if (limitExhausted && isMinuteLimitExhausted) { - int retryAfter = - (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER)))); - LOG.debug("API rate limit exceeded for the Minute, waiting for {} Seconds", retryAfter); - TimeUnit.SECONDS.sleep(retryAfter); + if (shouldWait) { + int retryAfter = + (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER)))); + LOG.debug("API rate limit exceeded for the Minute, waiting for {} Seconds", retryAfter); + TimeUnit.SECONDS.sleep(retryAfter); + } + String errorMsg = String.format("Call to Ariba failed. Status Code: %s, Root Cause: %s.", response.code(), + response.message()); + throw new AribaRetryableException(errorMsg, response.code()); } else if (limitExhausted && isSecondsLimitExhausted) { - int retryAfter = - (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER)))); - LOG.debug("API rate limit exceeded for the Second, waiting for {} Seconds", retryAfter); - TimeUnit.SECONDS.sleep(retryAfter); + if (shouldWait) { + int retryAfter = + (Integer.parseInt(Objects.requireNonNull(response.header(ResourceConstants.RETRY_AFTER)))); + LOG.debug("API rate limit exceeded for the Second, waiting for {} Seconds", retryAfter); + TimeUnit.SECONDS.sleep(retryAfter); + } + String errorMsg = String.format("Call to Ariba failed. Status Code: %s, Root Cause: %s.", response.code(), + response.message()); + throw new AribaRetryableException(errorMsg, response.code()); } else if (response.code() != HttpURLConnection.HTTP_OK) { throw new AribaException(response.message(), response.code()); } diff --git a/src/main/java/io/cdap/plugin/ariba/source/config/AribaPluginConfig.java b/src/main/java/io/cdap/plugin/ariba/source/config/AribaPluginConfig.java index 5e81b88..71c8ff2 100644 --- a/src/main/java/io/cdap/plugin/ariba/source/config/AribaPluginConfig.java +++ b/src/main/java/io/cdap/plugin/ariba/source/config/AribaPluginConfig.java @@ -53,6 +53,14 @@ public class AribaPluginConfig extends ReferencePluginConfig { public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; public static final String REFERENCE_NAME = "referenceName"; public static final String TOKEN_URL = "tokenURL"; + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_RETRY_MULTIPLIER = "retryMultiplier"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 2; + public static final int DEFAULT_RETRY_MULTIPLIER = 2; + public static final int DEFAULT_MAX_RETRY_COUNT = 3; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 10; private static final Logger LOG = LoggerFactory.getLogger(AribaPluginConfig.class); private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey(); @@ -93,6 +101,30 @@ public AribaConnectorConfig getConnection() { @Description("End date of the extract") private final String toDate; + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 2 seconds.") + @Nullable + @Macro + private final Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 300 seconds.") + @Nullable + @Macro + private final Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 3.") + @Nullable + @Macro + private final Integer maxRetryCount; + + @Name(NAME_RETRY_MULTIPLIER) + @Description("Multiplier for exponential backoff. Default is 2.") + @Nullable + @Macro + private final Integer retryMultiplier; + public AribaPluginConfig(String referenceName, String baseURL, @@ -104,13 +136,19 @@ public AribaPluginConfig(String referenceName, String apiKey, String tokenURL, @Nullable String fromDate, - @Nullable String toDate) { + @Nullable String toDate, + @Nullable Integer initialRetryDuration, @Nullable Integer maxRetryDuration, + @Nullable Integer retryMultiplier, @Nullable Integer maxRetryCount) { super(referenceName); this.viewTemplateName = viewTemplateName; this.connection = new AribaConnectorConfig(clientId, clientSecret, apiKey, baseURL, realm, systemType, tokenURL); this.fromDate = fromDate; this.toDate = toDate; + this.initialRetryDuration = initialRetryDuration; + this.maxRetryDuration = maxRetryDuration; + this.maxRetryCount = maxRetryCount; + this.retryMultiplier = retryMultiplier; } @@ -132,6 +170,22 @@ public String getToDate() { return toDate; } + public int getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public int getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public int getRetryMultiplier() { + return retryMultiplier == null ? DEFAULT_RETRY_MULTIPLIER : retryMultiplier; + } + + public int getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + /** * Validates the given {@code AribaPluginConfig} and throws the relative error messages. * @@ -153,6 +207,11 @@ public void validatePluginParameters(FailureCollector failureCollector) { if (AribaUtil.isNotNullOrEmpty(fromDate) || AribaUtil.isNotNullOrEmpty(toDate)) { validateAdvanceParameters(failureCollector); } + LOG.debug("Validating the retry parameters."); + if (!containsMacro(NAME_INITIAL_RETRY_DURATION) && !containsMacro(NAME_MAX_RETRY_DURATION) && + !containsMacro(NAME_MAX_RETRY_COUNT) && !containsMacro(NAME_RETRY_MULTIPLIER)) { + validateRetryConfiguration(failureCollector); + } failureCollector.getOrThrowException(); } @@ -237,6 +296,34 @@ private void validateAdvanceParameters(FailureCollector failureCollector) { } } + public void validateRetryConfiguration(FailureCollector failureCollector) { + if (initialRetryDuration != null && initialRetryDuration <= 0) { + failureCollector.addFailure("Initial retry duration must be greater than 0.", + "Please specify a valid initial retry duration.") + .withConfigProperty(NAME_INITIAL_RETRY_DURATION); + } + if (maxRetryDuration != null && maxRetryDuration <= 0) { + failureCollector.addFailure("Max retry duration must be greater than 0.", + "Please specify a valid max retry duration.") + .withConfigProperty(NAME_MAX_RETRY_DURATION); + } + if (maxRetryCount != null && maxRetryCount <= 0) { + failureCollector.addFailure("Max retry count must be greater than 0.", + "Please specify a valid max retry count.") + .withConfigProperty(NAME_MAX_RETRY_COUNT); + } + if (retryMultiplier != null && retryMultiplier <= 1) { + failureCollector.addFailure("Retry multiplier must be strictly greater than 1.", + "Please specify a valid retry multiplier.") + .withConfigProperty(NAME_RETRY_MULTIPLIER); + } + if (maxRetryDuration != null && initialRetryDuration != null && maxRetryDuration <= initialRetryDuration) { + failureCollector.addFailure("Max retry duration must be greater than initial retry duration.", + "Please specify a valid max retry duration.") + .withConfigProperty(NAME_MAX_RETRY_DURATION); + } + } + /** * Checks if the call to Ariba service is required for metadata creation. * condition parameters: ['host' | 'Realm' | 'Template' | 'Client Id' | 'Client Secret'] @@ -251,4 +338,104 @@ public boolean isSchemaBuildRequired() { && !containsMacro(CLIENT_SECRET) && !containsMacro(APIKEY) && !containsMacro(TOKEN_URL); } + /** + * Helper class to simplify {@link AribaPluginConfig} class creation. + */ + public static class Builder { + private String referenceName; + private String baseURL; + private String systemType; + private String realm; + private String viewTemplateName; + private String clientId; + private String clientSecret; + private String apiKey; + private String tokenURL; + private String fromDate; + private String toDate; + private Integer initialRetryDuration; + private Integer maxRetryDuration; + private Integer retryMultiplier; + private Integer maxRetryCount; + + public Builder referenceName(String referenceName) { + this.referenceName = referenceName; + return this; + } + + public Builder baseURL(String baseURL) { + this.baseURL = baseURL; + return this; + } + + public Builder systemType(String systemType) { + this.systemType = systemType; + return this; + } + + public Builder realm(String realm) { + this.realm = realm; + return this; + } + + public Builder viewTemplateName(String viewTemplateName) { + this.viewTemplateName = viewTemplateName; + return this; + } + + public Builder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public Builder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + public Builder tokenURL(String tokenURL) { + this.tokenURL = tokenURL; + return this; + } + + public Builder fromDate(String fromDate) { + this.fromDate = fromDate; + return this; + } + + public Builder toDate(String toDate) { + this.toDate = toDate; + return this; + } + + public Builder maxRetryCount(Integer maxRetryCount) { + this.maxRetryCount = maxRetryCount; + return this; + } + + public Builder initialRetryDuration(Integer initialRetryDuration) { + this.initialRetryDuration = initialRetryDuration; + return this; + } + + public Builder maxRetryDuration(Integer maxRetryDuration) { + this.maxRetryDuration = maxRetryDuration; + return this; + } + + public Builder retryMultiplier(Integer retryMultiplier) { + this.retryMultiplier = retryMultiplier; + return this; + } + + public AribaPluginConfig build() { + return new AribaPluginConfig(referenceName, baseURL, systemType, realm, viewTemplateName, clientId, clientSecret, + apiKey, tokenURL, fromDate, toDate, initialRetryDuration, maxRetryDuration, retryMultiplier, maxRetryCount); + } + } } diff --git a/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnector.java b/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnector.java index a91df43..89d3e97 100644 --- a/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnector.java +++ b/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnector.java @@ -88,6 +88,7 @@ public class AribaConnector implements DirectConnector { private static final String TOKEN = "PageToken"; private static final Gson GSON = new Gson(); private final AribaConnectorConfig config; + private final AribaServices aribaServices; private static final String AUTHORIZATION = "Authorization"; private String accessToken; private final ObjectMapper objectMapper = new ObjectMapper(); @@ -95,6 +96,12 @@ public class AribaConnector implements DirectConnector { public AribaConnector(AribaConnectorConfig config) { this.config = config; + aribaServices = new AribaServices(config, + AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT, + AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS, + AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS, + AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER, + false); } @Override @@ -126,7 +133,6 @@ public BrowseDetail browse(ConnectorContext connectorContext, BrowseRequest brow @Override public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSpecRequest connectorSpecRequest) throws IOException { - AribaServices aribaServices = new AribaServices(config); try { accessToken = aribaServices.getAccessToken(); } catch (AribaException e) { @@ -155,7 +161,6 @@ public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSp * returns the list of all the templates present in Ariba. */ private JsonArray listTemplates(String pageToken, JsonArray jsonElements) throws IOException { - AribaServices aribaServices = new AribaServices(config); try { accessToken = aribaServices.getAccessToken(); } catch (AribaException e) { @@ -188,7 +193,6 @@ private JsonArray listTemplates(String pageToken, JsonArray jsonElements) throws private List listTemplateData(String templateName) throws IOException, AribaException, InterruptedException { - AribaServices aribaServices = new AribaServices(config); try { accessToken = aribaServices.getAccessToken(); } catch (AribaException e) { diff --git a/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnectorConfig.java b/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnectorConfig.java index f51d4af..f5acdfc 100644 --- a/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnectorConfig.java +++ b/src/main/java/io/cdap/plugin/ariba/source/connector/AribaConnectorConfig.java @@ -22,6 +22,7 @@ import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.ariba.source.AribaServices; +import io.cdap.plugin.ariba.source.config.AribaPluginConfig; import io.cdap.plugin.ariba.source.exception.AribaException; import io.cdap.plugin.ariba.source.metadata.AribaResponseContainer; import io.cdap.plugin.ariba.source.util.AribaUtil; @@ -154,7 +155,13 @@ public void validateCredentials(FailureCollector failureCollector) { } public final void validateToken(FailureCollector collector) { - AribaServices aribaServices = new AribaServices(this); + AribaServices aribaServices = new AribaServices(this, + AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT, + AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS, + AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS, + AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER, + false); + try { String accessToken = aribaServices.getAccessToken(); URL viewTemplatesURL = HttpUrl.parse(this.getBaseURL()). diff --git a/src/main/java/io/cdap/plugin/ariba/source/exception/AribaRetryableException.java b/src/main/java/io/cdap/plugin/ariba/source/exception/AribaRetryableException.java new file mode 100644 index 0000000..016749c --- /dev/null +++ b/src/main/java/io/cdap/plugin/ariba/source/exception/AribaRetryableException.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.ariba.source.exception; + +/** + * This {@code AribaRetryableException} class is used to capture all the errors that are related to Ariba + * API limit issues. + */ + +public class AribaRetryableException extends Exception { + + private final Integer errorCode; + + public AribaRetryableException(String message) { + this(message, null, null); + } + + public AribaRetryableException(String message, Integer errorCode) { + this(message, errorCode, null); + } + + public AribaRetryableException(String message, Throwable cause) { + this(message, null, cause); + } + + public AribaRetryableException(String message, Integer errorCode, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; + } + + public Integer getErrorCode() { + return this.errorCode; + } +} diff --git a/src/test/java/io/cdap/plugin/ariba/source/AribaBatchSourceTest.java b/src/test/java/io/cdap/plugin/ariba/source/AribaBatchSourceTest.java index b26613a..886260a 100644 --- a/src/test/java/io/cdap/plugin/ariba/source/AribaBatchSourceTest.java +++ b/src/test/java/io/cdap/plugin/ariba/source/AribaBatchSourceTest.java @@ -17,7 +17,6 @@ package io.cdap.plugin.ariba.source; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; @@ -48,6 +47,7 @@ public class AribaBatchSourceTest { private static AribaBatchSource aribaBatchSource; private MockPipelineConfigurer pipelineConfigurer; private AribaPluginConfig pluginConfig; + private AribaPluginConfig.Builder pluginConfigBuilder; private AribaServices aribaServices; private Schema schema; private AribaInputSplit aribaInputSplit; @@ -56,30 +56,43 @@ public class AribaBatchSourceTest { @Mocked private BatchSourceContext context; - @Mocked - private FailureCollector failureCollector; - @Before public void setUp() { pipelineConfigurer = new MockPipelineConfigurer(null); - pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + pluginConfigBuilder = new AribaPluginConfig.Builder() + .referenceName("unit-test-ref-name") + .baseURL("https://openapi.ariba.com") + .systemType("prod") + .realm("test-realm") + .viewTemplateName("SourcingProjectFactSystemView") + .clientId("client-id") + .clientSecret("client-secret") + .apiKey("api-key") + .tokenURL("https://api.token.ariba.com") + .fromDate("2022-01-28T10:05:02Z") + .toDate("2022-01-31T10:05:02Z") + .initialRetryDuration(AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .maxRetryDuration(AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .retryMultiplier(AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .maxRetryCount(AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT); + + pluginConfig = pluginConfigBuilder.build(); + + aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), false); } @Test public void testConfigurePipelineWithInvalidBasicParam() { - pluginConfig = new AribaPluginConfig("referenceName", "", - "", "", - "", "clientId", - "clientSecret", "apiKey", "tokenURL", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + pluginConfig = pluginConfigBuilder.baseURL("") + .systemType("") + .realm("") + .viewTemplateName("") + .build(); + try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -104,11 +117,7 @@ public void testConfigurePipelineWithInvalidBasicParam() { @Test public void testConfigurePipelineWithEmptyReferenceName() { - pluginConfig = new AribaPluginConfig("", "url", - "sysType", "realm", - "template", "clientId", - "clientSecret", "apiKey", "tokenURL", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + pluginConfig = pluginConfigBuilder.referenceName("").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -124,14 +133,7 @@ public void testConfigurePipelineWithEmptyReferenceName() { @Test public void testConfigurePipelineWithEmptyClientIdAndClientSecret() { - pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "", "", "apiKey", - "https://api.au.cloud.ariba.com" - , "2022-01-31T10:05:02Z", - "2022-01-28T10:05:02Z"); + pluginConfig = pluginConfigBuilder.clientId("").clientSecret("").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -159,11 +161,7 @@ public void testValidateBasicParameters() { @Test public void testValidateCredentialParameters() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "", - "", "", "tokenURL", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + pluginConfig = pluginConfigBuilder.clientId("").clientSecret("").apiKey("").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -177,13 +175,7 @@ public void testValidateCredentialParameters() { @Test public void testValidateAdvancedParametersError() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", - "2022-01-31T10:05:02Z"); + pluginConfig = pluginConfigBuilder.build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -192,18 +184,13 @@ public void testValidateAdvancedParametersError() { List failures = s.getFailures(); Assert.assertEquals("Failures size does not match, " + "'CDF_ARIBA_01501 - Failed to call given Ariba service.'", - 3, failures.size()); + 1, failures.size()); } } @Test public void testValidateAdvancedParametersError2() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", ""); + pluginConfig = pluginConfigBuilder.toDate("").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -217,12 +204,7 @@ public void testValidateAdvancedParametersError2() { @Test public void testValidateAdvancedParametersError3() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2021-01-28T10:05:02Z"); + pluginConfig = pluginConfigBuilder.fromDate("2022-01-28T10:05:02Z").toDate("2021-01-28T10:05:02Z").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -238,12 +220,7 @@ public void testValidateAdvancedParametersError3() { @Test public void testValidateAdvancedParametersError4() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2025-01-28T10:05:02Z"); + pluginConfig = pluginConfigBuilder.fromDate("2022-01-28T10:05:02Z").toDate("2025-01-28T10:05:02Z").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -257,12 +234,7 @@ public void testValidateAdvancedParametersError4() { @Test public void testValidateAdvancedParametersError5() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "fromDate", "toDate"); + pluginConfig = pluginConfigBuilder.fromDate("fromDate").toDate("toDate").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -276,12 +248,7 @@ public void testValidateAdvancedParametersError5() { @Test public void testValidateAdvancedParametersError6() { - pluginConfig = new AribaPluginConfig("referenceName", "baseUrl", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", - "https://api.au.cloud.ariba.com", - "20220128T10:05:02Z", "20230128T10:05:02Z"); + pluginConfig = pluginConfigBuilder.fromDate("20220128T10:05:02Z").toDate("20230128T10:05:02Z").build(); try { aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.configurePipeline(pipelineConfigurer); @@ -295,19 +262,14 @@ public void testValidateAdvancedParametersError6() { @Test public void testIsSchemaBuildRequired() { - pluginConfig = new AribaPluginConfig("referenceName", - "https://stackoverflow.com/questions/17225948/" + - "parsing-error-for-date-field{{{browser_user_agent}}}", - "prod", "realm", - "viewTemplateName", "clientId", - "clientSecret", "apiKey", "tokenURL", - "2022-01-28T10:05:02Z", "2023-01-28T10:05:02Z"); + pluginConfig = pluginConfigBuilder + .baseURL("https://example.com/{{{browser_user_agent}}}") + .build(); Assert.assertTrue(pluginConfig.isSchemaBuildRequired()); } @Test public void testConfigurePipelineSchemaNotNull() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -327,7 +289,6 @@ public void testConfigurePipelineSchemaNotNull() throws IOException, AribaExcept @Test public void testConfigurePipelineForException() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -352,7 +313,6 @@ public void testConfigurePipelineForException() throws IOException, AribaExcepti @Test public void testConfigurePipelineForAribaException() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -376,7 +336,6 @@ public void testConfigurePipelineForAribaException() throws IOException, AribaEx @Test public void testConfigurePipelineForAribaException1() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -401,7 +360,6 @@ public void testConfigurePipelineForAribaException1() throws IOException, AribaE @Test public void testConfigurePipelineForAribaException2() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -426,7 +384,6 @@ public void testConfigurePipelineForAribaException2() throws IOException, AribaE @Test public void testConfigurePipelineForAribaException3() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaSchemaGenerator.class, AribaServices.class) { { aribaServices.getAccessToken(); @@ -543,6 +500,7 @@ public void testPrepareRun() throws Exception { } }; + pluginConfig = pluginConfigBuilder.build(); aribaBatchSource = new AribaBatchSource(pluginConfig); aribaBatchSource.prepareRun(context); schema = pipelineConfigurer.getOutputSchema(); @@ -551,7 +509,6 @@ public void testPrepareRun() throws Exception { @Test public void testPrepareRunForNullSchema() throws Exception { - aribaServices = new AribaServices(pluginConfig.getConnection()); aribaBatchSource = new AribaBatchSource(pluginConfig); new Expectations(AribaServices.class, AribaBatchSource.class) { { diff --git a/src/test/java/io/cdap/plugin/ariba/source/AribaInputFormatTest.java b/src/test/java/io/cdap/plugin/ariba/source/AribaInputFormatTest.java index de5b299..5c5ae77 100644 --- a/src/test/java/io/cdap/plugin/ariba/source/AribaInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/ariba/source/AribaInputFormatTest.java @@ -26,6 +26,7 @@ import mockit.Tested; import org.apache.hadoop.mapreduce.JobContext; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.io.IOException; @@ -52,18 +53,32 @@ public class AribaInputFormatTest { @Tested private AribaInputFormat aribaInputFormat; + private AribaPluginConfig pluginConfig; + + @Before + public void setup() { + AribaPluginConfig.Builder pluginConfigBuilder = new AribaPluginConfig.Builder() + .referenceName("unit-test-ref-name") + .baseURL("https://openapi.ariba.com") + .systemType("prod") + .realm("test-realm") + .viewTemplateName("SourcingProjectFactSystemView") + .clientId("client-id") + .clientSecret("client-secret") + .apiKey("api-key") + .tokenURL("https://api.token.ariba.com") + .fromDate("2022-01-28T10:05:02Z") + .toDate("2022-01-31T10:05:02Z") + .initialRetryDuration(AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .maxRetryDuration(AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .retryMultiplier(AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .maxRetryCount(AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT); + + pluginConfig = pluginConfigBuilder.build(); + } + @Test public void testCreateRecordReader() throws IOException, AribaException, InterruptedException { - AribaPluginConfig pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", - "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); aribaInputFormat = new AribaInputFormat(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode createJob = objectMapper.readTree(createJobResponse); @@ -90,16 +105,6 @@ public void testCreateRecordReader() throws IOException, AribaException, Interru @Test public void testCreateJobThrowError() throws IOException, AribaException, InterruptedException { - AribaPluginConfig pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", - "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); aribaInputFormat = new AribaInputFormat(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode jobData = objectMapper.readTree(createJobResponse1); @@ -128,16 +133,6 @@ public void testCreateJobThrowError() throws IOException, AribaException, Interr @Test public void testCreateJobThrowError1() throws IOException, AribaException, InterruptedException { - AribaPluginConfig pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", - "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); aribaInputFormat = new AribaInputFormat(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode crateJob = objectMapper.readTree(createJobResponse); @@ -167,16 +162,6 @@ public void testCreateJobThrowError1() throws IOException, AribaException, Inter @Test public void testCreateJobThrowError3() throws IOException, AribaException, InterruptedException { - AribaPluginConfig pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", - "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); aribaInputFormat = new AribaInputFormat(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode jobData = objectMapper.readTree(createJobResponse1); @@ -205,16 +190,6 @@ public void testCreateJobThrowError3() throws IOException, AribaException, Inter @Test public void testCreateJobThrowError4() throws IOException, AribaException, InterruptedException { - AribaPluginConfig pluginConfig = new AribaPluginConfig("unit-test-ref-name", - "https://openapi.au.cloud.ariba.com", - "prod", - "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", - "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", - "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); aribaInputFormat = new AribaInputFormat(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode crateJob = objectMapper.readTree(createJobResponse); diff --git a/src/test/java/io/cdap/plugin/ariba/source/AribaRecordReaderTest.java b/src/test/java/io/cdap/plugin/ariba/source/AribaRecordReaderTest.java index 7f06850..29e7476 100644 --- a/src/test/java/io/cdap/plugin/ariba/source/AribaRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/ariba/source/AribaRecordReaderTest.java @@ -19,14 +19,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.cdap.cdap.api.data.format.StructuredRecord; -import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.mock.common.MockPipelineConfigurer; import io.cdap.plugin.ariba.source.config.AribaPluginConfig; import io.cdap.plugin.ariba.source.exception.AribaException; import mockit.Expectations; import mockit.Mocked; -import org.apache.hadoop.io.NullWritable; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -46,17 +44,33 @@ public class AribaRecordReaderTest { @Before public void setUp() { pipelineConfigurer = new MockPipelineConfigurer(null); - pluginConfig = new AribaPluginConfig("unit-test-ref-name", "https://openapi.au.cloud.ariba.com", - "prod", "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + AribaPluginConfig.Builder pluginConfigBuilder = new AribaPluginConfig.Builder() + .referenceName("unit-test-ref-name") + .baseURL("https://openapi.ariba.com") + .systemType("prod") + .realm("test-realm") + .viewTemplateName("SourcingProjectFactSystemView") + .clientId("client-id") + .clientSecret("client-secret") + .apiKey("api-key") + .tokenURL("https://api.token.ariba.com") + .fromDate("2022-01-28T10:05:02Z") + .toDate("2022-01-31T10:05:02Z") + .initialRetryDuration(AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .maxRetryDuration(AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .retryMultiplier(AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .maxRetryCount(AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT); + pluginConfig = pluginConfigBuilder.build(); + + aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), false); } @Test public void testInitialize() throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); AribaRecordReader aribaRecordReader = new AribaRecordReader(aribaServices, getPluginSchema(), pluginConfig); AribaInputSplit aribaInputSplit = new AribaInputSplit("sourceView.zip", "3343ddsfsg3434"); AribaStructuredTransformer aribaStructuredTransformer = new AribaStructuredTransformer(); @@ -85,7 +99,6 @@ public void testInitialize() throws IOException, AribaException, InterruptedExce @Test public void testNextKeyValueFalse(@Mocked AribaInputSplit aribaInputSplit, @Mocked JsonNode node) throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); AribaRecordReader aribaRecordReader = new AribaRecordReader(aribaServices, pipelineConfigurer.getOutputSchema(), pluginConfig); new Expectations(AribaServices.class) { @@ -113,7 +126,6 @@ public void testReadFields() throws IOException { @Test(expected = IOException.class) public void testInitializeError() throws IOException, InterruptedException, AribaException { - aribaServices = new AribaServices(pluginConfig.getConnection()); AribaRecordReader aribaRecordReader = new AribaRecordReader(aribaServices, pipelineConfigurer.getOutputSchema(), pluginConfig); AribaInputSplit aribaInputSplit = new AribaInputSplit("sourceView.zip", "3343ddsfsg3434"); @@ -154,191 +166,191 @@ public void testInitializeError() throws IOException, InterruptedException, Arib "\":{\"EventType\":\"\"},\"SourceSystem\":{\"SourceSystemId\":\"ASM\"}}"; public static Schema getPluginSchema() throws IOException { - String schemaString = "{\n" + - " \"type\":\"record\",\n" + - " \"name\":\"AribaColumnMetadata\",\n" + - " \"fields\":[\n" + - " {\n" + - " \"name\":\"IsTestProject\",\n" + - " \"type\":[\n" + - " \"boolean\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Origin\",\n" + - " \"type\":[\n" + - " \"double\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Status\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Description\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"OnTimeOrLate\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ProcessStatus\",\n" + - " \"type\":[\n" + - " \"bytes\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"TargetSavingsPct\",\n" + - " \"type\":[\n" + - " \"double\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"LoadUpdateTime\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"AclId\",\n" + - " \"type\":[\n" + - " \"int\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ProjectId\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Duration\",\n" + - " \"type\":[\n" + - " \"double\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ProjectReason\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"BaselineSpend\",\n" + - " \"type\":[\n" + - " \"long\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ResultsDescription\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"SourcingMechanism\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"AwardJustification\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"LoadCreateTime\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ContractMonths\",\n" + - " \"type\":[\n" + - " \"double\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ActualSaving\",\n" + - " \"type\":[\n" + - " \"double\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"State\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Currency\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"ExecutionStrategy\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"Owner\",\n" + - " \"type\":[\n" + - " {\n" + - " \"type\":\"record\",\n" + - " \"name\":\"Owner\",\n" + - " \"fields\":[\n" + - " {\n" + - " \"name\":\"UserId\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\":\"SourceSystem\",\n" + - " \"type\":[\n" + - " \"string\",\n" + - " \"null\"\n" + - " ]\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"null\"\n" + - " ]\n" + - " }\n" + - " ]\n" + + String schemaString = "{" + + " \"type\":\"record\"," + + " \"name\":\"AribaColumnMetadata\"," + + " \"fields\":[" + + " {" + + " \"name\":\"IsTestProject\"," + + " \"type\":[" + + " \"boolean\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Origin\"," + + " \"type\":[" + + " \"double\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Status\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Description\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"OnTimeOrLate\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ProcessStatus\"," + + " \"type\":[" + + " \"bytes\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"TargetSavingsPct\"," + + " \"type\":[" + + " \"double\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"LoadUpdateTime\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"AclId\"," + + " \"type\":[" + + " \"int\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ProjectId\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Duration\"," + + " \"type\":[" + + " \"double\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ProjectReason\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"BaselineSpend\"," + + " \"type\":[" + + " \"long\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ResultsDescription\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"SourcingMechanism\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"AwardJustification\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"LoadCreateTime\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ContractMonths\"," + + " \"type\":[" + + " \"double\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ActualSaving\"," + + " \"type\":[" + + " \"double\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"State\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Currency\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"ExecutionStrategy\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"Owner\"," + + " \"type\":[" + + " {" + + " \"type\":\"record\"," + + " \"name\":\"Owner\"," + + " \"fields\":[" + + " {" + + " \"name\":\"UserId\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }," + + " {" + + " \"name\":\"SourceSystem\"," + + " \"type\":[" + + " \"string\"," + + " \"null\"" + + " ]" + + " }" + + " ]" + + " }," + + " \"null\"" + + " ]" + + " }" + + " ]" + "}"; return Schema.parseJson(schemaString); diff --git a/src/test/java/io/cdap/plugin/ariba/source/AribaServicesTest.java b/src/test/java/io/cdap/plugin/ariba/source/AribaServicesTest.java index 6f4e197..e3193db 100644 --- a/src/test/java/io/cdap/plugin/ariba/source/AribaServicesTest.java +++ b/src/test/java/io/cdap/plugin/ariba/source/AribaServicesTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.cdap.plugin.ariba.source.config.AribaPluginConfig; import io.cdap.plugin.ariba.source.exception.AribaException; +import io.cdap.plugin.ariba.source.exception.AribaRetryableException; import io.cdap.plugin.ariba.source.metadata.AribaColumnMetadata; import io.cdap.plugin.ariba.source.metadata.AribaResponseContainer; import io.cdap.plugin.ariba.source.metadata.AribaSchemaGenerator; @@ -55,113 +56,130 @@ public class AribaServicesTest { @Mocked AribaResponseContainer response; - String jsonNode = "{\n" + - " \"type\": \"object\",\n" + - " \"access_token\": \"jiuokiopu\",\n" + - " \"status\": \"completed\",\n" + - " \"totalNumOfPages\": \"1\",\n" + - " \"currentPageNum\": \"1\",\n" + - " \"message\": \"test\",\n" + - " \"properties\": {\n" + - " \"IsTestProject\": {\n" + - " \"title\": \"IsTestProject\",\n" + - " \"type\": [\n" + - " \"boolean\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": null,\n" + - " \"allowedValues\": null\n" + - " },\n" + - " \"Owner\": {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {\n" + - " \"SourceSystem\": {\n" + - " \"title\": \"Owner.SourceSystem\",\n" + - " \"type\": [\n" + - " \"string\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": 100,\n" + - " \"allowedValues\": null\n" + - " },\n" + - " \"UserId\": {\n" + - " \"title\": \"Owner.UserId\",\n" + - " \"type\": [\n" + - " \"string\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": 50,\n" + - " \"allowedValues\": null\n" + - " }\n" + - " }\n" + - " },\n" + - " \"Organization\": {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \"items\": [\n" + - " {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " },\n" + - " {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \n" + - " \"items\": [\n" + - " {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " },\n" + - " {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " }\n" + - " ]\n" + - " }\n" + - " \n" + - " ]\n" + - " }\n" + - " }\n" + + String jsonNode = "{" + + " \"type\": \"object\"," + + " \"access_token\": \"jiuokiopu\"," + + " \"status\": \"completed\"," + + " \"totalNumOfPages\": \"1\"," + + " \"currentPageNum\": \"1\"," + + " \"message\": \"test\"," + + " \"properties\": {" + + " \"IsTestProject\": {" + + " \"title\": \"IsTestProject\"," + + " \"type\": [" + + " \"boolean\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": null," + + " \"allowedValues\": null" + + " }," + + " \"Owner\": {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {" + + " \"SourceSystem\": {" + + " \"title\": \"Owner.SourceSystem\"," + + " \"type\": [" + + " \"string\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": 100," + + " \"allowedValues\": null" + + " }," + + " \"UserId\": {" + + " \"title\": \"Owner.UserId\"," + + " \"type\": [" + + " \"string\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": 50," + + " \"allowedValues\": null" + + " }" + + " }" + + " }," + + " \"Organization\": {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " \"items\": [" + + " {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }," + + " {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " " + + " \"items\": [" + + " {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }," + + " {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }" + + " ]" + + " }" + + " " + + " ]" + + " }" + + " }" + "}"; private AribaServices aribaServices; private AribaPluginConfig pluginConfig; + private AribaPluginConfig.Builder pluginConfigBuilder; @Before public void setUp() { - pluginConfig = new AribaPluginConfig("unit-test-ref-name", "https://openapi.au.cloud.ariba.com", - "prod", "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGkkhKSWUDqDRGE2Wds", "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", - "2022-01-28T10:05:02Z", "2022-01-31T10:05:02Z"); + pluginConfigBuilder = new AribaPluginConfig.Builder() + .referenceName("unit-test-ref-name") + .baseURL("https://openapi.ariba.com") + .systemType("prod") + .realm("test-realm") + .viewTemplateName("SourcingProjectFactSystemView") + .clientId("client-id") + .clientSecret("client-secret") + .apiKey("api-key") + .tokenURL("https://api.token.ariba.com") + .fromDate("2022-01-28T10:05:02Z") + .toDate("2022-01-31T10:05:02Z") + .initialRetryDuration(AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .maxRetryDuration(AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .retryMultiplier(AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .maxRetryCount(AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT); + + pluginConfig = pluginConfigBuilder.build(); + + aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), false); } @Test public void testGetAccessToken() throws AribaException, IOException { - aribaServices = new AribaServices(pluginConfig.getConnection()); - String tokenUrl = String.format("https://%s", "https://api.au.cloud.ariba.com/v2/oauth/token"); HttpUrl.Builder builder = Objects.requireNonNull(HttpUrl.parse(tokenUrl)) .newBuilder().addPathSegments("TOKEN_PATH"); @@ -193,15 +211,13 @@ public void testGetAccessToken() throws AribaException, IOException { } @Test - public void testGenerateTokenURL() throws AribaException { - aribaServices = new AribaServices(pluginConfig.getConnection()); - Assert.assertEquals("https://api.au.cloud.ariba.com/v2/oauth/token", + public void testGenerateTokenURL() { + Assert.assertEquals("https://api.token.ariba.com/v2/oauth/token", aribaServices.generateTokenURL().toString()); } @Test public void testCallAribaForTokenForError() throws IOException { - aribaServices = new AribaServices(pluginConfig.getConnection()); String tokenUrl = String.format("https://%s", "https://api.au.cloud.ariba.com/v2/oauth/token"); HttpUrl.Builder builder = Objects.requireNonNull(HttpUrl.parse(tokenUrl)) .newBuilder().addPathSegments("TOKEN_PATH"); @@ -231,7 +247,6 @@ public void testBuildOutputSchema() throws AribaException, IOException, Interrup AribaColumnMetadata columnList = columnDetail.build(); List columnDetails = new ArrayList<>(); columnDetails.add(columnList); - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaServices.class) { { aribaServices.getMetadata(anyString, anyString); @@ -246,7 +261,6 @@ public void testBuildOutputSchema() throws AribaException, IOException, Interrup @Test public void testGetMetadata() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; InputStream inputStream = new ByteArrayInputStream(jsonNode.getBytes()); new Expectations(AribaServices.class) { @@ -276,7 +290,6 @@ public void testGetMetadata() throws AribaException, IOException, InterruptedExc @Test public void testGetMetadataWithError() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; AribaResponseContainer responseContainer = new AribaResponseContainer(404, "URL not forund", null); @@ -311,7 +324,6 @@ public void testGetMetadataWithError() throws AribaException, IOException, Inter @Test public void testFetchAribaResponse() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; Request mockRequest = new Request.Builder() .url("https://some-url.com") @@ -340,7 +352,6 @@ public void testFetchAribaResponse() throws AribaException, IOException, Interru @Test public void testCreateJob() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); Request re = null; Request mockRequest = new Request.Builder() .url("https://some-url.com") @@ -389,7 +400,6 @@ public void testCreateJob() throws AribaException, IOException, InterruptedExcep @Test public void testCreateJobWithDate() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); Request re = null; Request mockRequest = new Request.Builder() .url("https://some-url.com") @@ -438,7 +448,6 @@ public void testCreateJobWithDate() throws AribaException, IOException, Interrup @Test public void testFetchData() { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; Request mockRequest = new Request.Builder() @@ -456,35 +465,29 @@ public void testFetchData() { aribaServices.fetchData("jobId", "fileName"); Assert.fail("testFetchData expected to fail with 'Call failed to get access token', but succeeded"); } catch (Exception e) { - Assert.assertEquals("Credentials are incorrect.", e.getMessage()); + Assert.assertEquals("Token Endpoint is incorrect.", e.getMessage()); } } @Test public void testFetchZipFileData() { - aribaServices = new AribaServices(pluginConfig.getConnection()); HttpUrl.Builder zipUrl = aribaServices.zipBuilder("jobId", "fileName"); Request request = aribaServices.fetchZipFileData(zipUrl.build().url(), "endpoint"); - Assert.assertEquals(request.url().toString(), - "https://openapi.au.cloud.ariba.com/api/" + - "analytics-reporting-jobresult/v1/prod/jobs/jobId/files" + - "/fileName?realm=CloudsufiDSAPP-T"); + Assert.assertEquals("https://openapi.ariba.com/api/analytics-reporting-jobresult/v1/" + + "prod/jobs/jobId/files/fileName?realm=test-realm", request.url().toString()); } @Test public void testZipBuilder() { - aribaServices = new AribaServices(pluginConfig.getConnection()); HttpUrl.Builder zipUrl = aribaServices.zipBuilder("jobId", "fileName"); Request request = aribaServices.fetchZipFileData(zipUrl.build().url(), "endpoint"); - Assert.assertEquals(request.url().toString(), - "https://openapi.au.cloud.ariba.com/api/analytics-reporting-jobresult/" + - "v1/prod/jobs/jobId/files/fileName?realm=CloudsufiDSAPP-T"); + Assert.assertEquals("https://openapi.ariba.com/api/analytics-reporting-jobresult/" + + "v1/prod/jobs/jobId/files/fileName?realm=test-realm", request.url().toString()); } @Test public void testFetchJobStatus() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -531,8 +534,7 @@ public void testFetchJobStatus() throws AribaException, IOException, Interrupted } @Test - public void testIsApiLimitExhaustedForDay() throws AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); + public void testIsApiLimitExhaustedForDay() throws InterruptedException { Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -550,18 +552,19 @@ public void testIsApiLimitExhaustedForDay() throws AribaException, InterruptedEx .build(); try { aribaServices.isApiLimitExhausted(response); - aribaServices.checkAndThrowException(response); + aribaServices.checkAndThrowException(response, true); Assert.fail("testIsApiLimitExhaustedForDay expected to fail with " + "'API rate limit exceeded for the Day', but succeeded"); } catch (AribaException e) { Assert.assertEquals("API rate limit exceeded for the Day, Please retry after 1 hours.", e.getMessage()); Assert.assertEquals(ResourceConstants.LIMIT_EXCEED_ERROR_CODE, e.getErrorCode()); + } catch (AribaRetryableException e) { + throw new RuntimeException(e); } } @Test - public void testIsApiLimitExhaustedForHour() throws AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); + public void testIsApiLimitExhaustedForHour() { Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -583,8 +586,7 @@ public void testIsApiLimitExhaustedForHour() throws AribaException, InterruptedE } @Test - public void testIsApiLimitExhaustedForMinute() throws AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); + public void testIsApiLimitExhaustedForMinute() { Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -608,8 +610,7 @@ public void testIsApiLimitExhaustedForMinute() throws AribaException, Interrupte } @Test - public void testIsApiLimitExhaustedForSecond() throws AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); + public void testIsApiLimitExhaustedForSecond() { Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -634,7 +635,6 @@ public void testIsApiLimitExhaustedForSecond() throws AribaException, Interrupte @Test public void checkUpdateFilter() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; InputStream inputStream = new ByteArrayInputStream(jsonNode.getBytes()); new Expectations(AribaServices.class) { @@ -662,10 +662,8 @@ public void checkUpdateFilter() throws AribaException, IOException, InterruptedE @Test public void testGetArraySchemaAsObject() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); Map arrayDocumentName = new HashMap<>(); arrayDocumentName.put("Suppliers", "SupplierDim"); - aribaServices = new AribaServices(pluginConfig.getConnection()); ObjectMapper mapper = new ObjectMapper(); URL url = null; InputStream inputStream = new ByteArrayInputStream(jsonNode.getBytes()); @@ -695,7 +693,6 @@ public void testGetArraySchemaAsObject() throws AribaException, IOException, Int @Test public void testBuildDataRequest() { - aribaServices = new AribaServices(pluginConfig.getConnection()); String tokenUrl = String.format("https://%s", "api.au.cloud.ariba.com/v2/oauth/token"); HttpUrl.Builder builder = Objects.requireNonNull(HttpUrl.parse(tokenUrl)) .newBuilder().addPathSegments("TOKEN_PATH"); @@ -705,7 +702,6 @@ public void testBuildDataRequest() { @Test public void testHttpAribaCall() throws AribaException, IOException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); Request mockRequest = new Request.Builder() .url("https://some-url.com") .build(); @@ -739,19 +735,50 @@ public void testHttpAribaCall() throws AribaException, IOException, InterruptedE @Test public void testFetchDataBuilder() { - aribaServices = new AribaServices(pluginConfig.getConnection()); HttpUrl.Builder builder = aribaServices.fetchDataBuilder("jobId"); - Assert.assertEquals(builder.build().url().toString(), "https://openapi.au.cloud.ariba.com/api/analytics" + - "-reporting-jobresult/v1/prod/jobs/jobId?realm=CloudsufiDSAPP-T"); + Assert.assertEquals( + "https://openapi.ariba.com/api/analytics-reporting-jobresult/v1/prod/jobs/jobId?realm=test-realm", + builder.build().url().toString()); } @Test public void testBuildFetchRequestr() { - aribaServices = new AribaServices(pluginConfig.getConnection()); String tokenUrl = String.format("https://%s", "https://api.au.cloud.ariba.com/v2/oauth/token"); HttpUrl.Builder builder = Objects.requireNonNull(HttpUrl.parse(tokenUrl)) .newBuilder().addPathSegments("TOKEN_PATH"); Request request = aribaServices.buildFetchRequest(builder.build().url(), "access_token"); - Assert.assertEquals(request.url().toString(), "https://https//api.au.cloud.ariba.com/v2/oauth/token/TOKEN_PATH"); + Assert.assertEquals(request.url().toString(), + "https://https//api.au.cloud.ariba.com/v2/oauth/token/TOKEN_PATH"); + } + + @Test + public void testApiLimitExhaustedFlagsReset() { + Response hourLimitExhaustedResponse = new Response.Builder() + .request(new Request.Builder().url("https://some-url.com").build()) + .protocol(Protocol.HTTP_2) + .code(429) + .message("Hourly limit exceeded") + .header("X-RateLimit-Remaining-Day", "1") + .header("X-RateLimit-Remaining-Hour", "0") + .body(ResponseBody.create(MediaType.parse("application/json"), "{}")) + .build(); + + Assert.assertTrue(aribaServices.isApiLimitExhausted(hourLimitExhaustedResponse)); + Assert.assertTrue(aribaServices.isHourLimitExhausted); + + Response minuteLimitExhaustedResponse = new Response.Builder() + .request(new Request.Builder().url("https://some-url.com").build()) + .protocol(Protocol.HTTP_2) + .code(429) + .message("Minute limit exceeded") + .header("X-RateLimit-Remaining-Day", "1") + .header("X-RateLimit-Remaining-Hour", "1") + .header("X-RateLimit-Remaining-Minute", "0") + .body(ResponseBody.create(MediaType.parse("application/json"), "{}")) + .build(); + + Assert.assertTrue(aribaServices.isApiLimitExhausted(minuteLimitExhaustedResponse)); + Assert.assertTrue(aribaServices.isMinuteLimitExhausted); + Assert.assertFalse(aribaServices.isHourLimitExhausted); } } diff --git a/src/test/java/io/cdap/plugin/ariba/source/metadata/proto/PropertiesTest.java b/src/test/java/io/cdap/plugin/ariba/source/metadata/proto/PropertiesTest.java index 2060012..6e1a8d7 100644 --- a/src/test/java/io/cdap/plugin/ariba/source/metadata/proto/PropertiesTest.java +++ b/src/test/java/io/cdap/plugin/ariba/source/metadata/proto/PropertiesTest.java @@ -70,95 +70,95 @@ public class PropertiesTest { AribaResponseContainer response; SampleRequest sampleRequest; - String jsonNode = "{\n" + - " \"type\": \"object\",\n" + - " \"access_token\": \"jiuokiopu\",\n" + - " \"status\": \"completed\",\n" + - " \"totalNumOfPages\": \"1\",\n" + - " \"currentPageNum\": \"1\",\n" + - " \"message\": \"test\",\n" + - " \"properties\": {\n" + - " \"IsTestProject\": {\n" + - " \"title\": \"IsTestProject\",\n" + - " \"type\": [\n" + - " \"boolean\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": null,\n" + - " \"allowedValues\": null\n" + - " },\n" + - " \"Owner\": {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {\n" + - " \"SourceSystem\": {\n" + - " \"title\": \"Owner.SourceSystem\",\n" + - " \"type\": [\n" + - " \"string\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": 100,\n" + - " \"allowedValues\": null\n" + - " },\n" + - " \"UserId\": {\n" + - " \"title\": \"Owner.UserId\",\n" + - " \"type\": [\n" + - " \"string\",\n" + - " \"null\"\n" + - " ],\n" + - " \"precision\": null,\n" + - " \"scale\": null,\n" + - " \"size\": 50,\n" + - " \"allowedValues\": null\n" + - " }\n" + - " }\n" + - " },\n" + - " \"Organization\": {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \"items\": [\n" + - " {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " },\n" + - " {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \n" + - " \"items\": [\n" + - " {\n" + - " \"type\": [\n" + - " \"object\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " },\n" + - " {\n" + - " \"type\": [\n" + - " \"array\",\n" + - " \"null\"\n" + - " ],\n" + - " \"properties\": {}\n" + - " }\n" + - " ]\n" + - " }\n" + - " \n" + - " ]\n" + - " }\n" + - " }\n" + + String jsonNode = "{" + + " \"type\": \"object\"," + + " \"access_token\": \"jiuokiopu\"," + + " \"status\": \"completed\"," + + " \"totalNumOfPages\": \"1\"," + + " \"currentPageNum\": \"1\"," + + " \"message\": \"test\"," + + " \"properties\": {" + + " \"IsTestProject\": {" + + " \"title\": \"IsTestProject\"," + + " \"type\": [" + + " \"boolean\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": null," + + " \"allowedValues\": null" + + " }," + + " \"Owner\": {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {" + + " \"SourceSystem\": {" + + " \"title\": \"Owner.SourceSystem\"," + + " \"type\": [" + + " \"string\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": 100," + + " \"allowedValues\": null" + + " }," + + " \"UserId\": {" + + " \"title\": \"Owner.UserId\"," + + " \"type\": [" + + " \"string\"," + + " \"null\"" + + " ]," + + " \"precision\": null," + + " \"scale\": null," + + " \"size\": 50," + + " \"allowedValues\": null" + + " }" + + " }" + + " }," + + " \"Organization\": {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " \"items\": [" + + " {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }," + + " {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " " + + " \"items\": [" + + " {" + + " \"type\": [" + + " \"object\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }," + + " {" + + " \"type\": [" + + " \"array\"," + + " \"null\"" + + " ]," + + " \"properties\": {}" + + " }" + + " ]" + + " }" + + " " + + " ]" + + " }" + + " }" + "}"; private AribaPluginConfig pluginConfig; @@ -170,12 +170,29 @@ public class PropertiesTest { @Before public void setUp() { properties = new Properties(); - pluginConfig = new AribaPluginConfig("unit-test-ref-name", "https://openapi.au.cloud.ariba.com", - "prod", "CloudsufiDSAPP-T", - "SourcingProjectFactSystemView", "08ee0299-4849-42a4-8464-3abed75fc74e", - "c3B5wvrEsjKucFGlGhKSWUDqDRGE2Wds", "xryi0757SU8pEyk7ePc7grc7vgDXdz8O", - "https://api.au.cloud.ariba.com", "2022-01-28T10:05:02Z", - "2022-01-31T10:05:02Z"); + AribaPluginConfig.Builder pluginConfigBuilder = new AribaPluginConfig.Builder() + .referenceName("unit-test-ref-name") + .baseURL("https://openapi.ariba.com") + .systemType("prod") + .realm("test-realm") + .viewTemplateName("SourcingProjectFactSystemView") + .clientId("client-id") + .clientSecret("client-secret") + .apiKey("api-key") + .tokenURL("https://api.token.ariba.com") + .fromDate("2022-01-28T10:05:02Z") + .toDate("2022-01-31T10:05:02Z") + .initialRetryDuration(AribaPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS) + .maxRetryDuration(AribaPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS) + .retryMultiplier(AribaPluginConfig.DEFAULT_RETRY_MULTIPLIER) + .maxRetryCount(AribaPluginConfig.DEFAULT_MAX_RETRY_COUNT); + pluginConfig = pluginConfigBuilder.build(); + + aribaServices = new AribaServices(pluginConfig.getConnection(), + pluginConfig.getMaxRetryCount(), + pluginConfig.getInitialRetryDuration(), + pluginConfig.getMaxRetryDuration(), + pluginConfig.getRetryMultiplier(), false); } @Test @@ -263,7 +280,6 @@ public void testValidateFieldsWithoutAccessToken() { public void testValidateToken() throws AribaException, IOException { MockFailureCollector collector = new MockFailureCollector(); AribaConnectorConfig connectorConfig = pluginConfig.getConnection(); - aribaServices = new AribaServices(pluginConfig.getConnection()); okHttpClient = new OkHttpClient(); Request mockRequest = new Request.Builder().url("https://some-url.com").build(); res = new Response.Builder() @@ -294,7 +310,6 @@ public void testValidateToken() throws AribaException, IOException { public void testValidateTokenWithInvalidResponse() throws AribaException, IOException { MockFailureCollector collector = new MockFailureCollector(); AribaConnectorConfig connectorConfig = pluginConfig.getConnection(); - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaServices.class) { { aribaServices.getAccessToken(); @@ -304,6 +319,7 @@ public void testValidateTokenWithInvalidResponse() throws AribaException, IOExce }; connectorConfig.validateToken(collector); Assert.assertEquals(1, collector.getValidationFailures().size()); + System.out.println(collector.getValidationFailures().get(0).getMessage()); } private void testTest(AribaConnector connector) { @@ -328,7 +344,6 @@ public void test() throws IOException, AribaException, InterruptedException { private void testGenerateSpec(AribaConnector connector) throws IOException, AribaException, InterruptedException { - aribaServices = new AribaServices(pluginConfig.getConnection()); URL url = null; InputStream inputStream = new ByteArrayInputStream(jsonNode.getBytes()); new Expectations(AribaServices.class) { @@ -358,7 +373,6 @@ private void testGenerateSpec(AribaConnector connector) throws IOException, Arib AribaColumnMetadata columnList = columnDetail.build(); List columnDetails = new ArrayList<>(); columnDetails.add(columnList); - aribaServices = new AribaServices(pluginConfig.getConnection()); new Expectations(AribaServices.class) { { aribaServices.getMetadata(anyString, anyString); @@ -390,7 +404,6 @@ private void testGenerateSpec(AribaConnector connector) throws IOException, Arib @Test public void testSample() throws AribaException, IOException, InterruptedException { AribaConnector aribaConnector = new AribaConnector(pluginConfig.getConnection()); - aribaServices = new AribaServices(pluginConfig.getConnection()); sampleRequest = SampleRequest.builder(1).build(); URL url = null; InputStream inputStream = new ByteArrayInputStream(jsonNode.getBytes()); @@ -412,6 +425,10 @@ public void testSample() throws AribaException, IOException, InterruptedExceptio result = "testToken"; minTimes = 0; + aribaServices.isApiLimitExhausted((Response) any); + result = false; + minTimes = 0; + sampleRequest.getPath(); result = "requisitionLineItemView"; minTimes = 0; @@ -421,7 +438,4 @@ public void testSample() throws AribaException, IOException, InterruptedExceptio Assert.assertTrue(aribaConnector.sample(new MockConnectorContext(new MockConnectorConfigurer()), sampleRequest). isEmpty()); } - - } - - +} diff --git a/widgets/Ariba-batchsource.json b/widgets/Ariba-batchsource.json index 19e10b1..aee6414 100644 --- a/widgets/Ariba-batchsource.json +++ b/widgets/Ariba-batchsource.json @@ -152,6 +152,42 @@ "widget-attributes": { "placeholder": "End date of the extraction, for example, 2022-03-29T00:00:00Z." } + }, + { + "widget-type": "hidden", + "label": "Initial Retry Duration (Seconds)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": "2", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Max Retry Duration (Seconds)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": "300", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Max Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": "3", + "minimum": "1" + } + }, + { + "widget-type": "hidden", + "label": "Retry Multiplier", + "name": "retryMultiplier", + "widget-attributes": { + "default": "2", + "placeholder": "The multiplier to use on retry attempts." + } } ] }