diff --git a/pom.xml b/pom.xml index c0a7d59..a3f7d5e 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ 4.9.1 2.0.0 2.27.2 + 2.7.0 ${project.basedir}/src/test/java/ UTF-8 UTF-8 @@ -305,7 +306,11 @@ jackson-core ${jackson.version} - + + io.cdap.plugin + hydrator-common + ${hydrator.version} + org.jmockit jmockit @@ -380,7 +385,7 @@ <_exportcontents> - io.cdap.plugin.successfactors.source.*; + io.cdap.plugin.successfactors.*; *;inline=false;scope=compile true diff --git a/src/main/java/io/cdap/plugin/successfactors/common/util/SuccessFactorsUtil.java b/src/main/java/io/cdap/plugin/successfactors/common/util/SuccessFactorsUtil.java index d295a09..e646586 100644 --- a/src/main/java/io/cdap/plugin/successfactors/common/util/SuccessFactorsUtil.java +++ b/src/main/java/io/cdap/plugin/successfactors/common/util/SuccessFactorsUtil.java @@ -97,8 +97,8 @@ public static String trim(String rawString) { * @return SuccessFactorsService instance */ public static SuccessFactorsService getSuccessFactorsService(SuccessFactorsPluginConfig pluginConfig) { - SuccessFactorsTransporter transporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), - pluginConfig.getPassword()); + SuccessFactorsTransporter transporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); SuccessFactorsService successFactorsService = new SuccessFactorsService(pluginConfig, transporter); return successFactorsService; } diff --git a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java new file mode 100644 index 0000000..6f4e723 --- /dev/null +++ b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java @@ -0,0 +1,265 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.successfactors.connector; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.connector.BrowseDetail; +import io.cdap.cdap.etl.api.connector.BrowseEntity; +import io.cdap.cdap.etl.api.connector.BrowseRequest; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.cdap.etl.api.connector.ConnectorContext; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.DirectConnector; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.cdap.etl.api.connector.SampleRequest; +import io.cdap.cdap.etl.api.validation.ValidationException; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException; +import io.cdap.plugin.successfactors.common.exception.TransportException; +import io.cdap.plugin.successfactors.common.util.ExceptionParser; +import io.cdap.plugin.successfactors.source.SuccessFactorsSource; +import io.cdap.plugin.successfactors.source.config.SuccessFactorsPluginConfig; +import io.cdap.plugin.successfactors.source.metadata.SuccessFactorsEntityProvider; +import io.cdap.plugin.successfactors.source.metadata.SuccessFactorsSchemaGenerator; +import io.cdap.plugin.successfactors.source.transform.SuccessFactorsTransformer; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsResponseContainer; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsTransporter; +import okhttp3.HttpUrl; + +import org.apache.olingo.odata2.api.edm.Edm; +import org.apache.olingo.odata2.api.edm.EdmEntitySet; +import org.apache.olingo.odata2.api.edm.EdmException; +import org.apache.olingo.odata2.api.ep.EntityProvider; +import org.apache.olingo.odata2.api.ep.EntityProviderException; +import org.apache.olingo.odata2.api.ep.EntityProviderReadProperties; +import org.apache.olingo.odata2.api.ep.entry.ODataEntry; +import org.apache.olingo.odata2.api.ep.feed.ODataFeed; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.ws.rs.core.MediaType; + +/** + * SuccessFactorsConnector Class + */ +@Plugin(type = Connector.PLUGIN_TYPE) +@Name(SuccessFactorsConnector.NAME) +@Description("Connection to access data in SuccessFactors.") +public class SuccessFactorsConnector implements DirectConnector { + public static final String NAME = "SuccessFactors"; + public static final String METADATA = "METADATA"; + public static final String PROPERTY_SEPARATOR = ","; + private static final String ENTITY_TYPE_ENTITY = "entity"; + private static final String METADATACALL = "$metadata"; + private static final String TOP_OPTION = "$top"; + private static final String SELECT_OPTION = "$select"; + private static final String ENTITY_SETS = "EntitySets"; + private final SuccessFactorsConnectorConfig config; + + public SuccessFactorsConnector(SuccessFactorsConnectorConfig config) { + this.config = config; + } + + @Override + public void test(ConnectorContext connectorContext) throws ValidationException { + FailureCollector collector = connectorContext.getFailureCollector(); + config.validateBasicCredentials(collector); + config.validateConnection(collector); + } + + @Override + public BrowseDetail browse(ConnectorContext connectorContext, BrowseRequest browseRequest) throws IOException { + BrowseDetail.Builder browseDetailBuilder = BrowseDetail.builder(); + int count = 0; + List entities = null; + try { + entities = listEntities(); + } catch (TransportException e) { + throw new IOException("Error in communicating SuccessFactors", e); + } + for (int i = 0; i < entities.size(); i++) { + String name = entities.get(i); + BrowseEntity.Builder entity = (BrowseEntity.builder(name, name, ENTITY_TYPE_ENTITY). + canBrowse(false).canSample(true)); + browseDetailBuilder.addEntity(entity.build()); + count++; + } + return browseDetailBuilder.setTotalCount(count).build(); + } + + @Override + public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSpecRequest connectorSpecRequest) + throws IOException { + ConnectorSpec.Builder specBuilder = ConnectorSpec.builder(); + Map properties = new HashMap<>(); + properties.put(io.cdap.plugin.common.ConfigUtil.NAME_USE_CONNECTION, "true"); + properties.put(ConfigUtil.NAME_CONNECTION, connectorSpecRequest.getConnectionWithMacro()); + String entity = connectorSpecRequest.getPath(); + if (entity != null) { + properties.put(SuccessFactorsPluginConfig.ENTITY_NAME, entity); + properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(entity)); + try { + Schema schema = getSchema(entity); + specBuilder.setSchema(schema); + } catch (SuccessFactorsServiceException | TransportException | EntityProviderException e) { + throw new IOException("Unable to create Schema", e); + } + } + return specBuilder.addRelatedPlugin(new PluginSpec(SuccessFactorsSource.NAME, BatchSource.PLUGIN_TYPE, + properties)).build(); + } + + List listEntities() throws TransportException, IOException { + URL dataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().build().url(); + SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config.getUsername(), + config.getPassword()); + SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsEntity + (dataURL, MediaType.APPLICATION_JSON, METADATA); + try (InputStream inputStream = responseContainer.getResponseStream()) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + String result = reader.lines().collect(Collectors.joining("")); + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(result, JsonObject.class); + JsonArray jsonArray = jsonObject.getAsJsonArray(ENTITY_SETS); + + Type type = new TypeToken>() { + }.getType(); + return gson.fromJson(jsonArray, type); + } + } + + Schema getSchema(String entityName) throws TransportException, EntityProviderException, + SuccessFactorsServiceException, IOException { + try (InputStream inputStream = getMetaDataStream(entityName)) { + Edm metadata = EntityProvider.readMetadata(inputStream, false); + SuccessFactorsEntityProvider edmData = new SuccessFactorsEntityProvider(metadata); + SuccessFactorsSchemaGenerator successFactorsSchemaGenerator = new SuccessFactorsSchemaGenerator(edmData); + return successFactorsSchemaGenerator.buildDefaultOutputSchema(entityName); + } + } + + @Override + public List sample(ConnectorContext connectorContext, SampleRequest sampleRequest) + throws IOException { + String entity = sampleRequest.getPath(); + if (entity == null) { + throw new IllegalArgumentException("Path should contain entity name."); + } + try { + return listEntityData(entity, sampleRequest.getLimit()); + } catch (EntityProviderException | SuccessFactorsServiceException | TransportException | EdmException e) { + throw new IOException("Unable to fetch data.", e); + } + } + + /** + * @return returns the list of the data for the selected entity. + */ + List listEntityData(String entity, long top) + throws EdmException, TransportException, EntityProviderException, SuccessFactorsServiceException, IOException { + try (InputStream inputStream = getMetaDataStream(entity)) { + Edm edm = EntityProvider.readMetadata(inputStream, false); + SuccessFactorsEntityProvider serviceHelper = new SuccessFactorsEntityProvider(edm); + EdmEntitySet edmEntitySet = serviceHelper.getEntitySet(entity); + try (InputStream dataStream = callEntityData(top, entity)) { + ODataFeed dataFeed = EntityProvider.readFeed(MediaType.APPLICATION_JSON, edmEntitySet, + dataStream, EntityProviderReadProperties.init().build()); + SuccessFactorsTransformer valueConverter = new SuccessFactorsTransformer(getSchema(entity)); + List oDataEntryList; + oDataEntryList = dataFeed != null ? dataFeed.getEntries() : Collections.emptyList(); + List data = new ArrayList<>(); + for (int i = 0; i < oDataEntryList.size(); i++) { + StructuredRecord dataRecord = valueConverter.buildCurrentRecord(oDataEntryList.get(i)); + data.add(dataRecord); + } + return data; + } + } + } + + /** + * @return returns the responseStream for the data of the selected entity. + */ + private InputStream callEntityData(long top, String entityName) + throws SuccessFactorsServiceException, TransportException, IOException, EntityProviderException, EdmException { + StringBuilder selectFields = new StringBuilder(String.join(PROPERTY_SEPARATOR, + getNonNavigationalProperties(entityName))); + URL dataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().addPathSegment(entityName). + addQueryParameter(TOP_OPTION, String.valueOf(top)).addQueryParameter(SELECT_OPTION, selectFields.toString()) + .build().url(); + SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config.getUsername(), + config.getPassword()); + SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL); + + ExceptionParser.checkAndThrowException("", responseContainer); + return responseContainer.getResponseStream(); + } + + List getNonNavigationalProperties(String entity) throws TransportException, EdmException, + EntityProviderException, IOException { + SuccessFactorsEntityProvider edmData = fetchServiceMetadata(entity); + SuccessFactorsSchemaGenerator successFactorsSchemaGenerator = new SuccessFactorsSchemaGenerator(edmData); + List columnDetailList = successFactorsSchemaGenerator.getNonNavigationalProperties + (entity); + return columnDetailList; + } + + SuccessFactorsEntityProvider fetchServiceMetadata(String entity) throws TransportException, + EntityProviderException, IOException { + try (InputStream metadataStream = getMetaDataStream(entity)) { + Edm metadata = EntityProvider.readMetadata(metadataStream, false); + return new SuccessFactorsEntityProvider(metadata); + } + } + /** + * @return returns the responseStream for metadata call. + */ + private InputStream getMetaDataStream(String entity) throws TransportException, IOException { + URL metadataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().addPathSegments(entity) + .addPathSegment(METADATACALL).build().url(); + SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config.getUsername(), + config.getPassword()); + SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient + .callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML, METADATA); + return responseContainer.getResponseStream(); + } + } + diff --git a/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java new file mode 100644 index 0000000..50f96c5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java @@ -0,0 +1,130 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.successfactors.connector; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.plugin.PluginConfig; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException; +import io.cdap.plugin.successfactors.common.exception.TransportException; +import io.cdap.plugin.successfactors.common.util.ResourceConstants; +import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsResponseContainer; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsTransporter; +import okhttp3.HttpUrl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.HttpURLConnection; +import java.net.URL; +import javax.ws.rs.core.MediaType; + +/** + * SuccessFactorsConnectorConfig Class + */ +public class SuccessFactorsConnectorConfig extends PluginConfig { + + public static final String BASE_URL = "baseURL"; + public static final String UNAME = "username"; + public static final String PASSWORD = "password"; + public static final String TEST = "TEST"; + private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey(); + private static final String SAP_SUCCESSFACTORS_USERNAME = "SAP SuccessFactors Username"; + private static final String SAP_SUCCESSFACTORS_PASSWORD = "SAP SuccessFactors Password"; + private static final String SAP_SUCCESSFACTORS_BASE_URL = "SAP SuccessFactors Base URL"; + private static final Logger LOG = LoggerFactory.getLogger(SuccessFactorsConnectorConfig.class); + + @Name(UNAME) + @Macro + @Description("SAP SuccessFactors Username for user authentication.") + private final String username; + + @Name(PASSWORD) + @Macro + @Description("SAP SuccessFactors password for user authentication.") + private final String password; + + @Macro + @Name(BASE_URL) + @Description("SuccessFactors Base URL.") + private final String baseURL; + + public SuccessFactorsConnectorConfig(String username, String password, String baseURL) { + this.username = username; + this.password = password; + this.baseURL = baseURL; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getBaseURL() { + return baseURL; + } + + public void validateBasicCredentials(FailureCollector failureCollector) { + + if (SuccessFactorsUtil.isNullOrEmpty(getUsername()) && !containsMacro(UNAME)) { + String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_USERNAME); + failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(UNAME); + } + if (SuccessFactorsUtil.isNullOrEmpty(getPassword()) && !containsMacro(PASSWORD)) { + String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_PASSWORD); + failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(PASSWORD); + } + if (SuccessFactorsUtil.isNullOrEmpty(getBaseURL()) && !containsMacro(BASE_URL)) { + String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_BASE_URL); + failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(BASE_URL); + } + if (SuccessFactorsUtil.isNotNullOrEmpty(getBaseURL()) && !containsMacro(BASE_URL)) { + if (HttpUrl.parse(getBaseURL()) == null) { + String errMsg = ResourceConstants.ERR_INVALID_BASE_URL.getMsgForKey(SAP_SUCCESSFACTORS_BASE_URL); + failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(BASE_URL); + } + } + } + + /** + * Method to validate the credential fields. + */ + public void validateConnection(FailureCollector collector) { + SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(getUsername(), getPassword()); + URL testerURL = HttpUrl.parse(getBaseURL()).newBuilder().build().url(); + SuccessFactorsResponseContainer responseContainer = null; + try { + responseContainer = + successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON, TEST); + } catch (TransportException e) { + LOG.error("Unable to fetch the response", e); + collector.addFailure("Unable to call SuccessFatorsEntity", "Please check the values"); + } + if (responseContainer.getHttpStatusCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { + String errMsg = ResourceConstants.ERR_INVALID_CREDENTIAL.getMsgForKey(); + collector.addFailure(errMsg, COMMON_ACTION); + } else if (responseContainer.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { + String errMsg = ResourceConstants.ERR_INVALID_BASE_URL.getMsgForKey(); + collector.addFailure(errMsg, COMMON_ACTION); + } + } +} diff --git a/src/main/java/io/cdap/plugin/successfactors/source/SuccessFactorsSource.java b/src/main/java/io/cdap/plugin/successfactors/source/SuccessFactorsSource.java index 1f2e1ea..8efe95b 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/SuccessFactorsSource.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/SuccessFactorsSource.java @@ -17,6 +17,8 @@ import com.google.gson.Gson; import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Metadata; +import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.batch.Input; @@ -27,6 +29,7 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.Connector; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.common.batch.JobUtils; @@ -35,6 +38,7 @@ import io.cdap.plugin.successfactors.common.util.ExceptionParser; import io.cdap.plugin.successfactors.common.util.ResourceConstants; import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil; +import io.cdap.plugin.successfactors.connector.SuccessFactorsConnector; import io.cdap.plugin.successfactors.source.config.SuccessFactorsPluginConfig; import io.cdap.plugin.successfactors.source.input.SuccessFactorsInputFormat; import io.cdap.plugin.successfactors.source.input.SuccessFactorsInputSplit; @@ -61,6 +65,7 @@ @Plugin(type = BatchSource.PLUGIN_TYPE) @Name(SuccessFactorsSource.NAME) @Description("Reads the SuccessFactors data which is exposed as OData services from SAP.") +@Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = SuccessFactorsConnector.NAME)}) public class SuccessFactorsSource extends BatchSource { public static final String NAME = "SuccessFactors"; public static final String OUTPUT_SCHEMA = "outputSchema"; @@ -120,18 +125,21 @@ public void prepareRun(BatchSourceContext context) throws Exception { */ @Nullable private Schema getOutputSchema(FailureCollector failureCollector) { - SuccessFactorsTransporter transporter = new SuccessFactorsTransporter(config.getUsername(), config.getPassword()); - SuccessFactorsService successFactorsServices = new SuccessFactorsService(config, transporter); - try { - //validate if the given parameters form a valid SuccessFactors URL. - successFactorsServices.checkSuccessFactorsURL(); - return successFactorsServices.buildOutputSchema(); - } catch (TransportException te) { - String errorMsg = ExceptionParser.buildTransportError(te); - errorMsg = ResourceConstants.ERR_ODATA_SERVICE_CALL.getMsgForKeyWithCode(errorMsg); - failureCollector.addFailure(errorMsg, null).withConfigProperty(SuccessFactorsPluginConfig.BASE_URL); - } catch (SuccessFactorsServiceException ose) { - attachFieldWithError(ose, failureCollector); + if (config.getConnection() != null) { + SuccessFactorsTransporter transporter = new SuccessFactorsTransporter(config.getConnection().getUsername(), + config.getConnection().getPassword()); + SuccessFactorsService successFactorsServices = new SuccessFactorsService(config, transporter); + try { + //validate if the given parameters form a valid SuccessFactors URL. + successFactorsServices.checkSuccessFactorsURL(); + return successFactorsServices.buildOutputSchema(); + } catch (TransportException te) { + String errorMsg = ExceptionParser.buildTransportError(te); + errorMsg = ResourceConstants.ERR_ODATA_SERVICE_CALL.getMsgForKeyWithCode(errorMsg); + failureCollector.addFailure(errorMsg, null).withConfigProperty(SuccessFactorsPluginConfig.BASE_URL); + } catch (SuccessFactorsServiceException ose) { + attachFieldWithError(ose, failureCollector); + } } failureCollector.getOrThrowException(); return null; diff --git a/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java b/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java index bb8d41f..bab6de6 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java @@ -24,10 +24,11 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.successfactors.common.util.ResourceConstants; import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil; -import okhttp3.HttpUrl; +import io.cdap.plugin.successfactors.connector.SuccessFactorsConnectorConfig; import java.io.IOException; import java.util.regex.Pattern; @@ -49,15 +50,7 @@ public class SuccessFactorsPluginConfig extends PluginConfig { private static final String PAGINATION_TYPE = "paginationType"; private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey(); private static final Pattern PATTERN = Pattern.compile("\\(.*\\)"); - private static final String SAP_SUCCESSFACTORS_BASE_URL = "SAP SuccessFactors Base URL"; - private static final String SAP_SUCCESSFACTORS_USERNAME = "SAP SuccessFactors Username"; - private static final String SAP_SUCCESSFACTORS_PASSWORD = "SAP SuccessFactors Password"; private static final String SAP_SUCCESSFACTORS_ENTITY_NAME = "Entity Name"; - - @Macro - @Name(BASE_URL) - @Description("SuccessFactors Base URL.") - private final String baseURL; @Macro @Name(ENTITY_NAME) @@ -70,19 +63,6 @@ public class SuccessFactorsPluginConfig extends PluginConfig { @Description("Name of the Associated Entity to be extracted.") private final String associateEntityName; - /** - * Credentials parameters - */ - @Name(UNAME) - @Macro - @Description("SAP SuccessFactors Username for user authentication.") - private final String username; - - @Name(PASSWORD) - @Macro - @Description("SAP SuccessFactors password for user authentication.") - private final String password; - /** * Advanced parameters */ @@ -126,6 +106,17 @@ public class SuccessFactorsPluginConfig extends PluginConfig { "automatically forces client offset pagination on the query.y. Default is Server-side Pagination.") private String paginationType; + @Name(ConfigUtil.NAME_USE_CONNECTION) + @Nullable + @Description("Whether to use an existing connection.") + private Boolean useConnection; + + @Name(ConfigUtil.NAME_CONNECTION) + @Macro + @Nullable + @Description("The existing connection to use.") + private SuccessFactorsConnectorConfig connection; + @VisibleForTesting public SuccessFactorsPluginConfig(String referenceName, String baseURL, @@ -137,19 +128,19 @@ public SuccessFactorsPluginConfig(String referenceName, @Nullable String selectOption, @Nullable String expandOption, String paginationType) { - + this.connection = new SuccessFactorsConnectorConfig(username, password, baseURL); this.referenceName = referenceName; - this.baseURL = baseURL; this.entityName = entityName; this.associateEntityName = associateEntityName; - this.username = username; - this.password = password; this.filterOption = filterOption; this.selectOption = selectOption; this.expandOption = expandOption; this.paginationType = paginationType; } - + @Nullable + public SuccessFactorsConnectorConfig getConnection() { + return connection; + } public static Builder builder() { return new Builder(); } @@ -158,10 +149,6 @@ public String getReferenceName() { return this.referenceName; } - public String getBaseURL() { - return SuccessFactorsUtil.trim(this.baseURL); - } - public String getEntityName() { return SuccessFactorsUtil.trim(this.entityName); } @@ -170,16 +157,6 @@ public String getAssociatedEntityName() { return SuccessFactorsUtil.trim(this.associateEntityName); } - @Nullable - public String getUsername() { - return SuccessFactorsUtil.trim(this.username); - } - - @Nullable - public String getPassword() { - return this.password; - } - @Nullable public String getFilterOption() { // Plugin UI field is 'textarea' so the user can input multiline filter statement @@ -252,16 +229,6 @@ public void validatePluginParameters(FailureCollector failureCollector) { private void validateMandatoryParameters(FailureCollector failureCollector) { IdUtils.validateReferenceName(getReferenceName(), failureCollector); - if (SuccessFactorsUtil.isNullOrEmpty(getBaseURL()) && !containsMacro(BASE_URL)) { - String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_BASE_URL); - failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(BASE_URL); - } - if (SuccessFactorsUtil.isNotNullOrEmpty(getBaseURL()) && !containsMacro(BASE_URL)) { - if (HttpUrl.parse(getBaseURL()) == null) { - String errMsg = ResourceConstants.ERR_INVALID_BASE_URL.getMsgForKey(SAP_SUCCESSFACTORS_BASE_URL); - failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(BASE_URL); - } - } if (SuccessFactorsUtil.isNullOrEmpty(getEntityName()) && !containsMacro(ENTITY_NAME)) { String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_ENTITY_NAME); failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(ENTITY_NAME); @@ -274,14 +241,8 @@ private void validateMandatoryParameters(FailureCollector failureCollector) { * @param failureCollector {@code FailureCollector} */ private void validateBasicCredentials(FailureCollector failureCollector) { - - if (SuccessFactorsUtil.isNullOrEmpty(getUsername()) && !containsMacro(UNAME)) { - String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_USERNAME); - failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(UNAME); - } - if (SuccessFactorsUtil.isNullOrEmpty(getPassword()) && !containsMacro(PASSWORD)) { - String errMsg = ResourceConstants.ERR_MISSING_PARAM_PREFIX.getMsgForKey(SAP_SUCCESSFACTORS_PASSWORD); - failureCollector.addFailure(errMsg, COMMON_ACTION).withConfigProperty(PASSWORD); + if (connection != null) { + connection.validateBasicCredentials(failureCollector); } } diff --git a/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainer.java b/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainer.java index 9c1118a..e27b648 100644 --- a/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainer.java +++ b/src/main/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsUrlContainer.java @@ -64,7 +64,7 @@ public SuccessFactorsUrlContainer(SuccessFactorsPluginConfig pluginConfig) { * @return tester URL. */ public URL getTesterURL() { - HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getBaseURL()) + HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getConnection().getBaseURL()) .newBuilder() .addPathSegment(pluginConfig.getEntityName()); @@ -82,7 +82,7 @@ public URL getTesterURL() { * @return metadata URL. */ public URL getMetadataURL() { - URL metadataURL = HttpUrl.parse(pluginConfig.getBaseURL()) + URL metadataURL = HttpUrl.parse(pluginConfig.getConnection().getBaseURL()) .newBuilder() .addPathSegments(pluginConfig.getEntityName()) .addPathSegment(METADATA) @@ -90,7 +90,7 @@ public URL getMetadataURL() { .url(); if (SuccessFactorsUtil.isNotNullOrEmpty(pluginConfig.getAssociatedEntityName())) { - metadataURL = HttpUrl.parse(pluginConfig.getBaseURL()) + metadataURL = HttpUrl.parse(pluginConfig.getConnection().getBaseURL()) .newBuilder() .addPathSegments(pluginConfig.getEntityName() .concat(PROPERTY_SEPARATOR) @@ -147,7 +147,7 @@ private HttpUrl.Builder buildQueryOptions(HttpUrl.Builder urlBuilder, Boolean is * @return total available record count URL. */ public URL getTotalRecordCountURL() { - HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getBaseURL()) + HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getConnection().getBaseURL()) .newBuilder() .addPathSegment(pluginConfig.getEntityName()) .addPathSegment(COUNT); @@ -168,7 +168,7 @@ public URL getTotalRecordCountURL() { * @return data URL with provided '$skip' and '$top' parameters. */ public URL getDataFetchURL(@Nullable Long skip, @Nullable Long top) { - HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getBaseURL()) + HttpUrl.Builder builder = HttpUrl.parse(pluginConfig.getConnection().getBaseURL()) .newBuilder() .addPathSegment(pluginConfig.getEntityName()); diff --git a/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java b/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java new file mode 100644 index 0000000..acfbfb8 --- /dev/null +++ b/src/test/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorTest.java @@ -0,0 +1,369 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.successfactors.connector; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.BrowseDetail; +import io.cdap.cdap.etl.api.connector.BrowseEntity; +import io.cdap.cdap.etl.api.connector.BrowseRequest; +import io.cdap.cdap.etl.api.connector.ConnectorContext; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.cdap.etl.api.connector.SampleRequest; +import io.cdap.cdap.etl.mock.common.MockConnectorConfigurer; +import io.cdap.cdap.etl.mock.common.MockConnectorContext; +import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException; +import io.cdap.plugin.successfactors.common.exception.TransportException; +import io.cdap.plugin.successfactors.source.SuccessFactorsSource; +import io.cdap.plugin.successfactors.source.config.SuccessFactorsPluginConfig; +import io.cdap.plugin.successfactors.source.metadata.SuccessFactorsEntityProvider; +import io.cdap.plugin.successfactors.source.metadata.SuccessFactorsSchemaGenerator; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsResponseContainer; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsTransporter; +import io.cdap.plugin.successfactors.source.transport.SuccessFactorsUrlContainer; +import mockit.Expectations; +import mockit.Mocked; +import mockit.Tested; +import org.apache.olingo.odata2.api.edm.Edm; +import org.apache.olingo.odata2.api.edm.EdmException; +import org.apache.olingo.odata2.api.ep.EntityProviderException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class SuccessFactorsConnectorTest { + @Tested + private SuccessFactorsPluginConfig.Builder pluginConfigBuilder; + private SuccessFactorsPluginConfig pluginConfig; + private SuccessFactorsTransporter successFactorsTransporter; + private SuccessFactorsConnector successFactorsConnector; + + @Mocked + private Edm edm; + + @Mocked + private BatchSourceContext context; + private SuccessFactorsSchemaGenerator successFactorsSchemaGenerator; + + @Before + public void testConfiguration() throws TransportException, SuccessFactorsServiceException { + pluginConfigBuilder = SuccessFactorsPluginConfig.builder() + .referenceName("unit-test-ref-name") + .baseURL("http://localhost") + .entityName("entity-name") + .username("username") + .password("password"); + + pluginConfig = pluginConfigBuilder.build(); + } + + @Test + public void testValidateSuccessfulConnection() throws TransportException, SuccessFactorsServiceException { + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, + SuccessFactorsSchemaGenerator.class) { + { + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getSuccessfulResponseContainer(); + minTimes = 1; + } + }; + pluginConfig.getConnection().validateConnection(context.getFailureCollector()); + Assert.assertEquals(0, context.getFailureCollector().getValidationFailures().size()); + } + + @Test + public void testValidateUnauthorisedConnection() throws TransportException, SuccessFactorsServiceException { + MockFailureCollector collector = new MockFailureCollector(); + ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, + SuccessFactorsSchemaGenerator.class) { + { + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getUnauthorisedResponseContainer(); + minTimes = 1; + } + }; + pluginConfig.getConnection().validateConnection(collector); + Assert.assertEquals(1, collector.getValidationFailures().size()); + } + + @Test + public void testValidateNotFoundConnection() throws TransportException, SuccessFactorsServiceException { + MockFailureCollector collector = new MockFailureCollector(); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsUrlContainer.class, SuccessFactorsTransporter.class, + SuccessFactorsSchemaGenerator.class) { + { + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getNotFoundResponseContainer(); + minTimes = 1; + } + }; + pluginConfig.getConnection().validateConnection(collector); + Assert.assertEquals(1, collector.getValidationFailures().size()); + } + + private SuccessFactorsResponseContainer getSuccessfulResponseContainer() { + return new SuccessFactorsResponseContainer(200, "ok", + "2.0", new byte[]{50}); + } + + private SuccessFactorsResponseContainer getUnauthorisedResponseContainer() { + return new SuccessFactorsResponseContainer(401, "", + "2.0", new byte[]{50}); + } + + private SuccessFactorsResponseContainer getNotFoundResponseContainer() { + return new SuccessFactorsResponseContainer(404, "", + "2.0", new byte[]{50}); + } + + + @Test(expected = IOException.class) + public void testGenerateSpec() throws TransportException, IOException { + ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + MockFailureCollector collector = new MockFailureCollector(); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsTransporter.class) { + { + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getSuccessfulResponseContainer(); + minTimes = 1; + + } + }; + SuccessFactorsConnector successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + successFactorsConnector.test(context); + ConnectorSpec connectorSpec = successFactorsConnector.generateSpec(new MockConnectorContext + (new MockConnectorConfigurer()), + ConnectorSpecRequest.builder().setPath + (pluginConfig.getEntityName()) + .setConnection("${conn(connection-id)}"). + build()); + + Set relatedPlugins = connectorSpec.getRelatedPlugins(); + Assert.assertEquals(1, relatedPlugins.size()); + PluginSpec pluginSpec = relatedPlugins.iterator().next(); + Assert.assertEquals(SuccessFactorsSource.NAME, pluginSpec.getName()); + Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec.getType()); + Map properties = pluginSpec.getProperties(); + Assert.assertEquals("true", properties.get(ConfigUtil.NAME_USE_CONNECTION)); + Assert.assertEquals("${conn(connection-id)}", properties.get(ConfigUtil.NAME_CONNECTION)); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testGenerateSpecWithSchema() throws TransportException, IOException, EntityProviderException, + SuccessFactorsServiceException { + ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + MockFailureCollector collector = new MockFailureCollector(); + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsConnector.class, SuccessFactorsTransporter.class) { + { + + successFactorsConnector.getSchema(anyString); + result = getPluginSchema(); + minTimes = 1; + + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getResponseContainer(); + minTimes = 1; + } + }; + successFactorsConnector.test(context); + ConnectorSpec connectorSpec = successFactorsConnector.generateSpec(new MockConnectorContext + (new MockConnectorConfigurer()), + ConnectorSpecRequest.builder().setPath + (pluginConfig.getEntityName()) + .setConnection("${conn(connection-id)}"). + build()); + + Schema schema = connectorSpec.getSchema(); + for (Schema.Field field : schema.getFields()) { + Assert.assertNotNull(field.getSchema()); + } + Set relatedPlugins = connectorSpec.getRelatedPlugins(); + Assert.assertEquals(1, relatedPlugins.size()); + PluginSpec pluginSpec = relatedPlugins.iterator().next(); + Assert.assertEquals(SuccessFactorsSource.NAME, pluginSpec.getName()); + Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec.getType()); + Map properties = pluginSpec.getProperties(); + Assert.assertEquals("true", properties.get(ConfigUtil.NAME_USE_CONNECTION)); + Assert.assertEquals("${conn(connection-id)}", properties.get(ConfigUtil.NAME_CONNECTION)); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testBrowse() throws IOException, TransportException { + ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + List entities = new ArrayList<>(); + entities.add("Achievement"); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + + new Expectations(SuccessFactorsTransporter.class, SuccessFactorsTransporter.class, SuccessFactorsConnector.class) { + { + successFactorsConnector.listEntities(); + result = entities; + minTimes = 1; + + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getResponseContainer(); + minTimes = 1; + } + }; + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + successFactorsConnector.test(context); + + BrowseDetail detail = successFactorsConnector.browse(new MockConnectorContext(new MockConnectorConfigurer()), + BrowseRequest.builder("/").build()); + Assert.assertTrue(detail.getTotalCount() > 0); + Assert.assertTrue(detail.getEntities().size() > 0); + for (BrowseEntity entity : detail.getEntities()) { + Assert.assertFalse(entity.canBrowse()); + Assert.assertTrue(entity.canSample()); + } + } + + + /** + * This will return null as no call is made here to fetch the data. + */ + @Test(expected = IOException.class) + public void testSampleWithoutSampleData() throws IOException, TransportException { + ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + new Expectations(SuccessFactorsTransporter.class, SuccessFactorsTransporter.class, SuccessFactorsConnector.class) { + { + successFactorsTransporter.callSuccessFactorsEntity(null, anyString, anyString); + result = getResponseContainer(); + minTimes = 1; + } + }; + String entityName = "entity"; + List records = new ArrayList<>(); + StructuredRecord structuredRecord = Mockito.mock(StructuredRecord.class); + records.add(structuredRecord); + SuccessFactorsConnector connector = new SuccessFactorsConnector(pluginConfig.getConnection()); + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + successFactorsConnector.test(context); + List sample = connector.sample(new MockConnectorContext(new MockConnectorConfigurer()), + SampleRequest.builder(1).setPath(entityName).build()); + Assert.assertNull(sample); + } + + @Test + public void testSampleWithSampleData() throws IOException, TransportException, EntityProviderException, + SuccessFactorsServiceException, EdmException { + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); + String entityName = "entity"; + List records = new ArrayList<>(); + StructuredRecord structuredRecord = Mockito.mock(StructuredRecord.class); + records.add(structuredRecord); + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + new Expectations(SuccessFactorsConnector.class) { + { + successFactorsConnector.listEntityData(anyString, anyLong); + result = records; + minTimes = 1; + + } + }; + + List sample = successFactorsConnector.sample(new MockConnectorContext + (new MockConnectorConfigurer()), SampleRequest. + builder(1).setPath(entityName).build()); + Assert.assertNotNull(sample); + } + + @Test + public void testGetNonNavigationalProperties() + throws EntityProviderException, TransportException, EdmException, IOException { + Edm edmMetadata = Mockito.mock(Edm.class); + SuccessFactorsEntityProvider edmData = new SuccessFactorsEntityProvider(edmMetadata); + successFactorsSchemaGenerator = new SuccessFactorsSchemaGenerator(new SuccessFactorsEntityProvider(edm)); + successFactorsConnector = new SuccessFactorsConnector(pluginConfig.getConnection()); + List columnDetailList = new ArrayList<>(); + columnDetailList.add("name"); + new Expectations(SuccessFactorsConnector.class, SuccessFactorsSchemaGenerator.class) { + { + successFactorsConnector.fetchServiceMetadata(anyString); + result = edmData; + minTimes = 1; + + successFactorsSchemaGenerator.getNonNavigationalProperties(anyString); + result = columnDetailList; + minTimes = 1; + } + }; + Assert.assertEquals("name", successFactorsConnector.getNonNavigationalProperties("entity").get(0)); + } + + /** + * exception is expected as entity is null. + */ + @Test(expected = IllegalArgumentException.class) + public void testSampleWithEntityNull() throws IOException { + String entityName = null; + List records = new ArrayList<>(); + StructuredRecord structuredRecord = Mockito.mock(StructuredRecord.class); + records.add(structuredRecord); + SuccessFactorsConnector connector = new SuccessFactorsConnector(pluginConfig.getConnection()); + connector.sample(new MockConnectorContext(new MockConnectorConfigurer()), + SampleRequest.builder(1).setPath(entityName).build()); + } + + private Schema getPluginSchema() throws IOException { + String schemaString = "{\"type\":\"record\",\"name\":\"SuccessFactorsColumnMetadata\",\"fields\":[{\"name\":" + + "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"endDate\",\"type\":[{\"type\":\"long\"," + + "\"logicalType\":\"timestamp-micros\"},\"null\"]},{\"name\":\"lastModifiedDate\",\"type\":" + + "[{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"},\"null\"]},{\"name\":\"project\",\"type\":" + + "\"string\"},{\"name\":\"startDate\",\"type\":[{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}," + + "\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; + + return Schema.parseJson(schemaString); + } + + private SuccessFactorsResponseContainer getResponseContainer() { + return new SuccessFactorsResponseContainer(200, "ok", + "2.0", new byte[]{50}); + } +} diff --git a/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java b/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java index 74a7e31..c496581 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/SuccessFactorsSourceTest.java @@ -160,7 +160,8 @@ public void testConfigurePipelineWSchemaNotNull() throws SuccessFactorsServiceEx .password("password"); pluginConfig = pluginConfigBuilder.build(); - successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), pluginConfig.getPassword()); + successFactorsTransporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); successFactorsUrlContainer = new SuccessFactorsUrlContainer(pluginConfig); successFactorsSchemaGenerator = new SuccessFactorsSchemaGenerator(new SuccessFactorsEntityProvider(edm)); diff --git a/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java b/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java index 62b0250..b30c39f 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfigTest.java @@ -158,7 +158,7 @@ public void testValidateEntityForKeyBasedExtraction() { @Test public void testRefactoredPluginPropertyValues() { SuccessFactorsPluginConfig pluginConfig = pluginConfigBuilder - .baseURL(" http://localhost:5000 ") + .baseURL("http://localhost:5000") .entityName("entity-name") .associateEntityName("AssEntity") .filterOption("amount le 20 and amount gt 4") @@ -170,7 +170,7 @@ public void testRefactoredPluginPropertyValues() { Assert.assertEquals("Type1", pluginConfig.getPaginationType()); Assert.assertEquals("col1", pluginConfig.getExpandOption()); Assert.assertEquals("amount le 20 and amount gt 4", pluginConfig.getFilterOption()); - Assert.assertEquals("Base URL not trimmed", "http://localhost:5000", pluginConfig.getBaseURL()); + Assert.assertEquals("Base URL not trimmed", "http://localhost:5000", pluginConfig.getConnection().getBaseURL()); Assert.assertEquals("Entity name not trimmed", "entity-name", pluginConfig.getEntityName()); Assert.assertEquals("Select option not trimmed", "col1,col2,parent/col1,col3", pluginConfig.getSelectOption()); } diff --git a/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java b/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java index 27b0283..79db565 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/input/SuccessFactorsInputFormatTest.java @@ -15,6 +15,8 @@ */ package io.cdap.plugin.successfactors.source.input; +import com.google.gson.Gson; +import io.cdap.plugin.successfactors.connector.SuccessFactorsConnectorConfig; import io.cdap.plugin.successfactors.source.SuccessFactorsSource; import io.cdap.plugin.successfactors.source.config.SuccessFactorsPluginConfig; import io.cdap.plugin.successfactors.source.metadata.TestSuccessFactorsUtil; @@ -28,15 +30,15 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; import java.nio.charset.StandardCharsets; import java.util.Base64; @RunWith(PowerMockRunner.class) -@PrepareForTest(SuccessFactorsTransporter.class) +@PrepareForTest({SuccessFactorsTransporter.class, SuccessFactorsConnectorConfig.class, Gson.class}) public class SuccessFactorsInputFormatTest { public static final String SUCCESSFACTORS_PLUGIN_PROPERTIES = "successFactorsPluginProperties"; public static final String ENCODED_ENTITY_METADATA_STRING = "encodedMetadataString"; @@ -65,17 +67,21 @@ public void testCreateRecordReader() throws Exception { "[{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"},\"null\"]},{\"name\":\"project\",\"type\":" + "\"string\"},{\"name\":\"startDate\",\"type\":[{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}," + "\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; + SuccessFactorsInputFormat successFactorsInputFormat = new SuccessFactorsInputFormat(); Configuration configuration = Mockito.mock(Configuration.class); - SuccessFactorsTransporter transporter = PowerMockito.mock(SuccessFactorsTransporter.class); - PowerMockito.whenNew(SuccessFactorsTransporter.class).withArguments(pluginConfig.getUsername(), - pluginConfig.getPassword()). - thenReturn(transporter); - SuccessFactorsInputSplit split = Mockito.mock(SuccessFactorsInputSplit.class); TaskAttemptContext taskAttemptContext = Mockito.mock(TaskAttemptContext.class); Mockito.when(taskAttemptContext.getConfiguration()).thenReturn(configuration); + Mockito.when(configuration.get(SUCCESSFACTORS_PLUGIN_PROPERTIES)).thenReturn(schemaString); + Gson gson = Mockito.mock(Gson.class); + Whitebox.setInternalState(SuccessFactorsInputFormat.class, "GSON", gson); + Mockito.when(gson.fromJson(schemaString, SuccessFactorsPluginConfig.class)).thenReturn(pluginConfig); + SuccessFactorsConnectorConfig connectorConfig = Mockito.mock(SuccessFactorsConnectorConfig.class); + Mockito.when(pluginConfig.getConnection()).thenReturn(connectorConfig); + Mockito.when(connectorConfig.getUsername()).thenReturn("user"); + Mockito.when(connectorConfig.getPassword()).thenReturn("password"); + SuccessFactorsInputSplit split = Mockito.mock(SuccessFactorsInputSplit.class); Mockito.when(taskAttemptContext.getConfiguration().get(SuccessFactorsSource.OUTPUT_SCHEMA)). thenReturn(schemaString); - Mockito.when(taskAttemptContext.getConfiguration().get(SUCCESSFACTORS_PLUGIN_PROPERTIES)).thenReturn(schemaString); String metadataString = TestSuccessFactorsUtil.convertInputStreamToString(TestSuccessFactorsUtil.readResource ("successfactors-metadata2.xml")); String encodedMetaData = Base64.getEncoder().encodeToString(metadataString.getBytes(StandardCharsets.UTF_8)); @@ -83,7 +89,6 @@ public void testCreateRecordReader() throws Exception { Edm edm = Mockito.mock(Edm.class); SuccessFactorsService successFactorsService = Mockito.mock(SuccessFactorsService.class); Mockito.when(successFactorsService.getSuccessFactorsServiceEdm("encodedMetadataString")).thenReturn(edm); - SuccessFactorsInputFormat successFactorsInputFormat = new SuccessFactorsInputFormat(); Assert.assertNotNull(successFactorsInputFormat.createRecordReader(split, taskAttemptContext)); } } diff --git a/src/test/java/io/cdap/plugin/successfactors/source/transport/RuntimeFunctionalTest.java b/src/test/java/io/cdap/plugin/successfactors/source/transport/RuntimeFunctionalTest.java index 0f4e6d9..847f94a 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/transport/RuntimeFunctionalTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/transport/RuntimeFunctionalTest.java @@ -104,7 +104,8 @@ public void runPipelineWithDefaultValues() throws Exception { long availableRowCount = 3; List partitionList = new SuccessFactorsPartitionBuilder().buildSplits(availableRowCount); - transporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), pluginConfig.getPassword()); + transporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), pluginConfig. + getConnection().getPassword()); successFactorsService = new SuccessFactorsService(pluginConfig, transporter); prepareStubForMetadata(pluginConfig); edmData = successFactorsService.getSuccessFactorsServiceEdm(encodedMetadataString); @@ -140,7 +141,8 @@ public void verifyFailToDecodeMetadataString() throws SuccessFactorsServiceExcep exceptionRule.expect(SuccessFactorsServiceException.class); exceptionRule .expectMessage(ResourceConstants.ERR_METADATA_DECODE.getMsgForKeyWithCode(pluginConfig.getEntityName())); - transporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), pluginConfig.getPassword()); + transporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), pluginConfig. + getConnection().getPassword()); successFactorsService = new SuccessFactorsService(pluginConfig, transporter); successFactorsService.getSuccessFactorsServiceEdm("encodedMetadataString"); } @@ -152,7 +154,8 @@ public void verifyDataCorrectness() prepareStubForMetadata(pluginConfig); long availableRowCount = 3; List partitionList = new SuccessFactorsPartitionBuilder().buildSplits(availableRowCount); - transporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), pluginConfig.getPassword()); + transporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); successFactorsService = new SuccessFactorsService(pluginConfig, transporter); edmData = successFactorsService.getSuccessFactorsServiceEdm(encodedMetadataString); for (SuccessFactorsInputSplit inputSplit : partitionList) { @@ -238,7 +241,8 @@ private void prepareStubForRun(SuccessFactorsPluginConfig pluginConfig) { String expectedBody = TestSuccessFactorsUtil.convertInputStreamToString(TestSuccessFactorsUtil.readResource ("successfactors-data.json")); WireMock.stubFor(WireMock.get(WireMock.urlPathMatching("/odata/v2/Background_SpecialAssign")) - .withBasicAuth(pluginConfig.getUsername(), pluginConfig.getPassword()) + .withBasicAuth(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()) .willReturn(WireMock.ok() .withHeader(SuccessFactorsTransporter.SERVICE_VERSION, "2.0") .withBody(expectedBody))); diff --git a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java index f52fbdd..d638be8 100644 --- a/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java +++ b/src/test/java/io/cdap/plugin/successfactors/source/transport/SuccessFactorsTransporterTest.java @@ -124,14 +124,16 @@ public void setUp() { .password("secret"); pluginConfig = pluginConfigBuilder.build(); successFactorsURL = new SuccessFactorsUrlContainer(pluginConfig); - transporter = new SuccessFactorsTransporter(pluginConfig.getUsername(), pluginConfig.getPassword()); + transporter = new SuccessFactorsTransporter(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()); } @Test public void testCallSuccessFactors() throws TransportException { String expectedBody = "{\"d\": [{\"ID\": 0,\"Name\": \"Bread\"}}]}"; WireMock.stubFor(WireMock.get("/Entity?%24top=1") - .withBasicAuth(pluginConfig.getUsername(), pluginConfig.getPassword()) + .withBasicAuth(pluginConfig.getConnection().getUsername(), + pluginConfig.getConnection().getPassword()) .willReturn(WireMock.ok() .withHeader(SuccessFactorsTransporter.SERVICE_VERSION, "2.0") .withBody(expectedBody))); diff --git a/widgets/SuccessFactors-batchsource.json b/widgets/SuccessFactors-batchsource.json index 4475b86..ec36453 100644 --- a/widgets/SuccessFactors-batchsource.json +++ b/widgets/SuccessFactors-batchsource.json @@ -15,14 +15,6 @@ "placeholder": "Used to uniquely identify this source for lineage, annotating metadata etc." } }, - { - "widget-type": "textbox", - "label": "SAP SuccessFactors Base URL", - "name": "baseURL", - "widget-attributes": { - "placeholder": "SAP SuccessFactors base url, for example, https:///odata/v2" - } - }, { "widget-type": "textbox", "label": "Entity Name", @@ -30,17 +22,51 @@ "widget-attributes": { "placeholder": "SAP SuccessFactors Entity name. For example, People" }, + "plugin-function": { "method": "POST", "widget": "outputSchema", "plugin-method": "getSchema" } + }, + { + "label": "browse", + "widget-type": "connection-browser", + "widget-category": "plugin", + "widget-attributes": { + "connectionType": " SUCCESSFACTORS", + "label": "Browse" + } } ] }, { - "label": "Credentials", + "label": "Connection", "properties": [ + { + "widget-type": "toggle", + "label": "Use connection", + "name": "useConnection", + "widget-attributes": { + "on": { + "value": "true", + "label": "YES" + }, + "off": { + "value": "false", + "label": "NO" + }, + "default": "false" + } + }, + { + "widget-type": "connection-select", + "label": "Connection", + "name": "connection", + "widget-attributes": { + "connectionType": "SuccessFactors" + } + }, { "widget-type": "textbox", "label": "SAP SuccessFactors Logon Username", @@ -56,6 +82,14 @@ "widget-attributes": { "placeholder": "" } + }, + { + "widget-type": "textbox", + "label": "SAP SuccessFactors Base URL", + "name": "baseURL", + "widget-attributes": { + "placeholder": "SAP SuccessFactors base url, for example, https:///odata/v2" + } } ] }, @@ -116,6 +150,40 @@ ] } ], + "filters":[ + { + "name": "showConnectionProperties ", + "condition": { + "expression": "useConnection == false" + }, + "show": [ + { + "type": "property", + "name": "username" + }, + { + "type": "property", + "name": "password" + }, + { + "type": "property", + "name": "baseURL" + } + ] + }, + { + "name": "showConnectionId", + "condition": { + "expression": "useConnection == true" + }, + "show": [ + { + "type": "property", + "name": "connection" + } + ] + } + ], "outputs": [ { "name": "schema", diff --git a/widgets/SuccessFactors-connector.json b/widgets/SuccessFactors-connector.json new file mode 100644 index 0000000..7ab0324 --- /dev/null +++ b/widgets/SuccessFactors-connector.json @@ -0,0 +1,38 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "SAP SuccessFactors", + "configuration-groups": [ + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "SAP SuccessFactors Logon Username", + "name": "username", + "widget-attributes": { + "placeholder": "" + } + }, + { + "widget-type": "password", + "label": "SAP SuccessFactors Logon Password", + "name": "password", + "widget-attributes": { + "placeholder": "" + } + }, + { + "widget-type": "textbox", + "label": "SAP SuccessFactors Base URL", + "name": "baseURL", + "widget-attributes": { + "placeholder": "SAP SuccessFactors base url, for example, https:///odata/v2" + } + } + ] + } + ], + "outputs": [] +} \ No newline at end of file