Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Move drive to blob #4

Merged
merged 6 commits into from
Aug 10, 2023
Merged

feat: Move drive to blob #4

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d711bbf
taxonomy
Grasso2000 Aug 9, 2023
ed4b23d
taxonomy
marcomeretta Aug 9, 2023
29b049e
taxonomy
marcomeretta Aug 9, 2023
fb6ab7e
taxonomy
marcomeretta Aug 9, 2023
e124ef8
Added loggers
Giampiero-Porfiri Aug 9, 2023
f9eee4e
taxonomy
marcomeretta Aug 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .devops/deploy-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,21 +36,21 @@ variables:
AZURE_SUBSCRIPTION: $(DEV_AZURE_SUBSCRIPTION)
APP_NAME: $(DEV_WEB_APP_NAME)
STAGE: "d"
RESOURCE_GROUP: 'pagopa-d-weu-shared-rg'
RESOURCE_GROUP: 'pagopa-d-weu-shared-txnm-rg'
dockerRegistryServiceConnection: $(DEV_CONTAINER_REGISTRY_SERVICE_CONN)
dockerNamespace: $(DEV_CONTAINER_NAMESPACE)
${{ if eq(parameters['ENV'], 'uat') }}:
AZURE_SUBSCRIPTION: $(UAT_AZURE_SUBSCRIPTION)
APP_NAME: $(UAT_WEB_APP_NAME)
STAGE: "u"
RESOURCE_GROUP: 'pagopa-u-weu-shared-rg'
RESOURCE_GROUP: 'pagopa-d-weu-shared-txnm-rg'
marcomeretta marked this conversation as resolved.
Show resolved Hide resolved
dockerRegistryServiceConnection: $(UAT_CONTAINER_REGISTRY_SERVICE_CONN)
dockerNamespace: $(UAT_CONTAINER_NAMESPACE)
${{ if eq(parameters['ENV'], 'prod') }}:
AZURE_SUBSCRIPTION: $(PROD_AZURE_SUBSCRIPTION)
APP_NAME: $(PROD_WEB_APP_NAME)
STAGE: "p"
RESOURCE_GROUP: 'pagopa-p-weu-shared-rg'
RESOURCE_GROUP: 'pagopa-d-weu-shared-txnm-rg'
marcomeretta marked this conversation as resolved.
Show resolved Hide resolved
dockerRegistryServiceConnection: $(PROD_CONTAINER_REGISTRY_SERVICE_CONN)
dockerNamespace: $(PROD_CONTAINER_NAMESPACE)

Expand Down Expand Up @@ -155,7 +155,7 @@ stages:
condition: in('${{ parameters.ENV }}', 'dev', 'uat')
inputs:
azureSubscription: $(AZURE_SUBSCRIPTION)
appName: "${{variables.APP_NAME}}-shared-taxonomy-fn"
appName: "${{variables.APP_NAME}}-shared-txnm-fn"
imageName: "${{variables.dockerNamespace}}/${{ variables.imageRepository }}:latest"
slotName: production
resourceGroupName: $(RESOURCE_GROUP)
Expand All @@ -164,7 +164,7 @@ stages:
condition: eq('${{ parameters.ENV }}', 'prod')
inputs:
azureSubscription: $(AZURE_SUBSCRIPTION)
appName: "${{variables.APP_NAME}}-shared-taxonomy-fn"
appName: "${{variables.APP_NAME}}-shared-txnm-fn"
imageName: "${{variables.dockerNamespace}}/${{ variables.imageRepository }}:latest"
deployToSlotOrASE: true
slotName: staging
Expand Down Expand Up @@ -198,7 +198,7 @@ stages:
displayName: Swapping App Service Deploy
inputs:
ConnectedServiceName: $(AZURE_SUBSCRIPTION)
WebAppName: "${{variables.APP_NAME}}-shared-taxonomy-fn"
WebAppName: "${{variables.APP_NAME}}-shared-txnm-fn"
ResourceGroupName: $(RESOURCE_GROUP)
SourceSlot: staging
SwapWithProduction: true
Expand Down
27 changes: 16 additions & 11 deletions src/main/java/it/gov/pagopa/taxonomy/TaxonomyGetFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,24 @@
public class TaxonomyGetFunction {

private static final String storageConnString = System.getenv("STORAGE_ACCOUNT_CONN_STRING");
private static final String blobContainerName = System.getenv("BLOB_CONTAINER_NAME");
private static final String blobName = System.getenv("JSON_NAME");

private static final String blobContainerNameOuput = System.getenv("BLOB_CONTAINER_NAME_OUTPUT");
private static final String jsonName = System.getenv("JSON_NAME");
private static ObjectMapper objectMapper = null;

private static BlobContainerClient blobContainerClient;
private static BlobContainerClient blobContainerClientOutput;
private static BlobServiceClient blobServiceClient;
private static BlobServiceClient getBlobServiceClient(){
if(blobServiceClient == null){
blobServiceClient = new BlobServiceClientBuilder().connectionString(storageConnString).buildClient();
}
return blobServiceClient;
}

private static BlobContainerClient getBlobContainerClient(){
if(blobContainerClient == null){
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(storageConnString).buildClient();
blobContainerClient = blobServiceClient.createBlobContainerIfNotExists(blobContainerName);
private static BlobContainerClient getBlobContainerClientOutput(){
if(blobContainerClientOutput == null){
blobContainerClientOutput = getBlobServiceClient().createBlobContainerIfNotExists(blobContainerNameOuput);
}
return blobContainerClient;
return blobContainerClientOutput;
}

private static ObjectMapper getObjectMapper(){
Expand All @@ -57,7 +62,7 @@ public HttpResponseMessage getTaxonomy(
methods = {HttpMethod.GET},
route = "taxonomy",
authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage<Optional<String>> request,
final ExecutionContext context) throws JsonProcessingException {
final ExecutionContext context) {
Logger logger = context.getLogger();

try {
Expand Down Expand Up @@ -100,7 +105,7 @@ private static TaxonomyJson getTaxonomy(Logger logger) {
try {
Instant now = Instant.now();
logger.info("Retrieving standard json from the blob storage at: [" + now + "]");
String content = getBlobContainerClient().getBlobClient(blobName).downloadContent().toString();
String content = getBlobContainerClientOutput().getBlobClient(jsonName).downloadContent().toString();

logger.info("Versioning the json");
return getObjectMapper().readValue(content, TaxonomyJson.class);
Expand Down
60 changes: 31 additions & 29 deletions src/main/java/it/gov/pagopa/taxonomy/TaxonomyUpdateFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,7 @@
import it.gov.pagopa.taxonomy.model.json.TaxonomyJson;
import it.gov.pagopa.taxonomy.util.AppUtil;
import org.modelmapper.ModelMapper;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.ConnectException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
Expand All @@ -39,24 +32,39 @@

public class TaxonomyUpdateFunction {

private static final String csvUrl = System.getenv("CSV_URL");
private static final String storageConnString = System.getenv("STORAGE_ACCOUNT_CONN_STRING");
private static final String blobContainerName = System.getenv("BLOB_CONTAINER_NAME");
private static final String blobName = System.getenv("JSON_NAME");

private static final String blobContainerNameInput = System.getenv("BLOB_CONTAINER_NAME_INPUT");
private static final String blobContainerNameOuput = System.getenv("BLOB_CONTAINER_NAME_OUTPUT");
private static final String jsonName = System.getenv("JSON_NAME");
private static final String csvName = System.getenv("CSV_NAME");
private static ObjectMapper objectMapper = null;

private static ModelMapper modelMapper = null;
private static BlobContainerClient blobContainerClient;
private static BlobContainerClient blobContainerClientInput;
private static BlobContainerClient blobContainerClientOutput;
private static BlobServiceClient blobServiceClient;
private static BlobServiceClient getBlobServiceClient(){
if(blobServiceClient == null){
blobServiceClient = new BlobServiceClientBuilder().connectionString(storageConnString).buildClient();
}
return blobServiceClient;
}

private static BlobContainerClient getBlobContainerClientInput(){
if(blobContainerClientInput == null){
blobContainerClientInput = getBlobServiceClient().createBlobContainerIfNotExists(blobContainerNameInput);
}
return blobContainerClientInput;
}

private static BlobContainerClient getBlobContainerClient(){
if(blobContainerClient == null){
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(storageConnString).buildClient();
blobContainerClient = blobServiceClient.createBlobContainerIfNotExists(blobContainerName);
private static BlobContainerClient getBlobContainerClientOutput(){
if(blobContainerClientOutput == null){
blobContainerClientOutput = getBlobServiceClient().createBlobContainerIfNotExists(blobContainerNameOuput);
}
return blobContainerClient;
return blobContainerClientOutput;
}


private static ObjectMapper getObjectMapper(){
if(objectMapper == null){
objectMapper = new ObjectMapper();
Expand Down Expand Up @@ -85,6 +93,7 @@ public HttpResponseMessage updateTaxonomy(
try {
updateTaxonomy(logger);
String payload = AppUtil.getPayload(getObjectMapper(), Message.builder().message("Taxonomy updated successfully").build());
logger.info("Done fine processo");
return AppUtil.writeResponse(request,
HttpStatus.OK,
payload
Expand Down Expand Up @@ -117,8 +126,9 @@ public HttpResponseMessage updateTaxonomy(

private static void updateTaxonomy(Logger logger) {
try {
logger.info("Download csv [" + csvUrl + "]");
InputStreamReader inputStreamReader = new InputStreamReader(new URL(csvUrl).openStream(), StandardCharsets.UTF_8);
logger.info("Download csv file ["+csvName+"] from blob at ["+Instant.now().toString()+"]");

InputStreamReader inputStreamReader = new InputStreamReader(getBlobContainerClientInput().getBlobClient(csvName).downloadContent().toStream());

logger.info("Transform csv to json");
List<TaxonomyCsv> taxonomyCsvList = new CsvToBeanBuilder<TaxonomyCsv>(inputStreamReader)
Expand All @@ -141,18 +151,10 @@ private static void updateTaxonomy(Logger logger) {
byte[] jsonBytes = getObjectMapper().writeValueAsBytes(taxonomyJson);

logger.info("Upload json id=[" + id + "] created at : [" + now + "]");
getBlobContainerClient().getBlobClient(blobName).upload(BinaryData.fromBytes(jsonBytes), true);

} catch (ConnectException connException) {
throw new AppException(connException, AppErrorCodeMessageEnum.CONNECTION_REFUSED, csvUrl);
} catch (FileNotFoundException fileNotFoundException) {
throw new AppException(fileNotFoundException, AppErrorCodeMessageEnum.FILE_DOES_NOT_EXIST);
} catch (MalformedURLException malformedURLException) {
throw new AppException(malformedURLException, AppErrorCodeMessageEnum.MALFORMED_URL);
getBlobContainerClientOutput().getBlobClient(jsonName).upload(BinaryData.fromBytes(jsonBytes), true);

} catch (JsonProcessingException | IllegalStateException parsingException) {
throw new AppException(parsingException, AppErrorCodeMessageEnum.CSV_PARSING_ERROR);
} catch (IOException ioException) {
throw new AppException(ioException, AppErrorCodeMessageEnum.ERROR_READING_WRITING);
}
}
}
Loading