From 3cccc583c0be61f663d6ba86675a1e3eabaf0421 Mon Sep 17 00:00:00 2001 From: Chris Morris Date: Tue, 12 Sep 2023 09:51:31 +0100 Subject: [PATCH] Latest set of gpconnect analytics changes --- .../gpconnect-analytics.DAL/BatchService.cs | 161 ++++++++++++++++++ source/gpconnect-analytics.DAL/BlobService.cs | 5 +- .../ConfigurationService.cs | 7 + source/gpconnect-analytics.DAL/DataService.cs | 6 +- .../gpconnect-analytics.DAL/ImportService.cs | 56 +++++- .../Interfaces/IBatchService.cs | 19 +++ .../Interfaces/IBlobService.cs | 2 +- .../Interfaces/IConfigurationService.cs | 2 + .../Interfaces/IImportService.cs | 10 +- .../Interfaces/ILoggingService.cs | 9 + .../Interfaces/ISplunkService.cs | 5 +- .../gpconnect-analytics.DAL/LoggingService.cs | 21 +++ .../gpconnect-analytics.DAL/SplunkService.cs | 145 ++++++++-------- .../gpconnect-analytics.DAL.csproj | 2 + .../Request/RequestUri.cs | 12 ++ .../Response/Queue/Message.cs | 1 + .../Response/Splunk/Extract.cs | 2 + .../Response/Splunk/ExtractResponse.cs | 4 +- .../ExecuteImport.cs | 36 ---- .../ExecuteImportByTrigger.cs | 24 +++ .../GetDataFromApi.cs | 95 ----------- .../GetDataFromApiByDateRange.cs | 39 +++++ .../GetDataFromApiByTrigger.cs | 36 ++++ .../GetDataFromApiManual.cs | 26 +++ .../GetDataFromApiToday.cs | 39 +++++ .../PurgeErrorLogByTrigger.cs | 23 +++ .../gpconnect-analytics.Functions/Startup.cs | 2 + .../gpconnect-analytics.Functions.csproj | 5 +- .../gpconnect-analytics.Functions/host.json | 18 +- .../DateFormatConstants.cs | 4 +- .../DateTimeHelper.cs | 14 ++ 31 files changed, 602 insertions(+), 228 deletions(-) create mode 100644 source/gpconnect-analytics.DAL/BatchService.cs create mode 100644 source/gpconnect-analytics.DAL/Interfaces/IBatchService.cs create mode 100644 source/gpconnect-analytics.DAL/Interfaces/ILoggingService.cs create mode 100644 source/gpconnect-analytics.DAL/LoggingService.cs create mode 100644 source/gpconnect-analytics.DTO/Request/RequestUri.cs delete mode 100644 source/gpconnect-analytics.Functions/ExecuteImport.cs create mode 100644 source/gpconnect-analytics.Functions/ExecuteImportByTrigger.cs delete mode 100644 source/gpconnect-analytics.Functions/GetDataFromApi.cs create mode 100644 source/gpconnect-analytics.Functions/GetDataFromApiByDateRange.cs create mode 100644 source/gpconnect-analytics.Functions/GetDataFromApiByTrigger.cs create mode 100644 source/gpconnect-analytics.Functions/GetDataFromApiManual.cs create mode 100644 source/gpconnect-analytics.Functions/GetDataFromApiToday.cs create mode 100644 source/gpconnect-analytics.Functions/PurgeErrorLogByTrigger.cs create mode 100644 source/gpconnect-analytics.Helpers/DateTimeHelper.cs diff --git a/source/gpconnect-analytics.DAL/BatchService.cs b/source/gpconnect-analytics.DAL/BatchService.cs new file mode 100644 index 0000000..d6787f5 --- /dev/null +++ b/source/gpconnect-analytics.DAL/BatchService.cs @@ -0,0 +1,161 @@ +using Dapper; +using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.DTO.Request; +using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.Helpers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Web; + +namespace gpconnect_analytics.DAL +{ + public class BatchService : IBatchService + { + private readonly IConfigurationService _configurationService; + private readonly ILogger _logger; + private SplunkClient _splunkClient; + private readonly IDataService _dataService; + private readonly ISplunkService _splunkService; + private readonly IImportService _importService; + + public BatchService(IConfigurationService configurationService, IImportService importService, ISplunkService splunkService, ILogger logger, IDataService dataService) + { + _configurationService = configurationService; + _logger = logger; + _dataService = dataService; + _splunkService = splunkService; + _importService = importService; + } + + public async Task StartBatchDownloadForTodayAsync(FileTypes fileTypes) + { + var dateInScope = DateTime.Today.AddDays(1); + var fileType = await _configurationService.GetFileType(fileTypes); + var uriList = await GetBatchDownloadUriList(fileType, DateTimeHelper.EachDay(dateInScope, dateInScope).ToList()); + + await RemovePreviousDownloads(fileType, dateInScope, dateInScope); + return await ProcessUrls(fileType, uriList, true); + } + + public async Task StartBatchDownloadAsync(HttpRequest req, FileTypes fileTypes) + { + if (req != null) + { + var startDate = DateTime.TryParse(req.Query["StartDate"].ToString(), out DateTime start) ? start : DateTime.Today; + var endDate = DateTime.TryParse(req.Query["EndDate"].ToString(), out DateTime end) ? end : DateTime.Today; + + if (endDate >= startDate) + { + var fileType = await _configurationService.GetFileType(fileTypes); + var uriList = await GetBatchDownloadUriList(fileType, DateTimeHelper.EachDay(startDate, endDate).ToList()); + + await RemovePreviousDownloads(fileType, startDate, endDate); + + return await ProcessUrls(fileType, uriList, false); + } + } + return new BadRequestObjectResult("Bad request"); + } + + private async Task ProcessUrls(FileType fileType, List uriList, bool isToday) + { + for (var i = 0; i < uriList.Count; i++) + { + var downloadTasksQuery = + from requestUri in uriList.Skip(i).Take(1) + select ExecuteBatchDownloadFromSplunk(fileType, requestUri, isToday); + + var downloadTasks = downloadTasksQuery.ToList(); + + while (downloadTasks.Any()) + { + Task finishedTask = await Task.WhenAny(downloadTasks); + downloadTasks.Remove(finishedTask); + } + await Task.Delay(TimeSpan.FromSeconds(10)); + } + return new OkObjectResult($"Batch download complete: {uriList.Count} requests processed"); + } + + private async Task ExecuteBatchDownloadFromSplunk(FileType fileType, UriRequest uriRequest, bool isToday) + { + try + { + if (FileTypeEnabled(fileType)) + { + var extractResponse = await _splunkService.DownloadCSVDateRangeAsync(fileType, uriRequest, isToday); + await _importService.AddObjectFileMessage(fileType, extractResponse); + } + else + { + _logger?.LogWarning($"Filetype {fileType.FileTypeFilePrefix} is not enabled. Please check if this is correct"); + } + } + catch (Exception exc) + { + _logger?.LogError(exc, $"An error has occurred while attempting to execute an Azure function"); + throw; + } + } + + public async Task> GetBatchDownloadUriList(FileType fileType, List dateTimeList) + { + var uriList = new List(); + _splunkClient = await _configurationService.GetSplunkClientConfiguration(); + + foreach (var dateTime in dateTimeList) + { + var earliestDate = dateTime.AddDays(-2); + var latestDate = dateTime.AddDays(-1); + + for (var i = 0; i < 24; i++) + { + var splunkQuery = fileType.SplunkQuery; + var hour = TimeSpan.Zero.Add(TimeSpan.FromHours(i)); + + splunkQuery = splunkQuery.Replace("{earliest}", earliestDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate)); + splunkQuery = splunkQuery.Replace("{latest}", latestDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate)); + splunkQuery = splunkQuery.Replace("{hour}", hour.ToString(Helpers.DateFormatConstants.SplunkQueryHour)); + + var uriBuilder = new UriBuilder + { + Scheme = Uri.UriSchemeHttps, + Host = _splunkClient.HostName, + Port = _splunkClient.HostPort, + Path = _splunkClient.BaseUrl, + Query = string.Format(_splunkClient.QueryParameters, HttpUtility.UrlEncode(splunkQuery)) + }; + + uriList.Add(new UriRequest() + { + Request = uriBuilder.Uri, + EarliestDate = earliestDate, + LatestDate = latestDate, + Hour = hour + }); + } + } + return uriList; + } + + public async Task RemovePreviousDownloads(FileType fileType, DateTime startDate, DateTime endDate) + { + var procedureName = "Import.RemovePreviousDownload"; + var parameters = new DynamicParameters(); + parameters.Add("@FileTypeId", fileType.FileTypeId); + parameters.Add("@StartDate", startDate.AddDays(-2)); + parameters.Add("@EndDate", endDate.AddDays(-1)); + await _dataService.ExecuteStoredProcedure(procedureName, parameters); + } + + private bool FileTypeEnabled(FileType fileType) + { + return (fileType != null && fileType.Enabled); + } + } +} diff --git a/source/gpconnect-analytics.DAL/BlobService.cs b/source/gpconnect-analytics.DAL/BlobService.cs index 2e6b510..9ffb709 100644 --- a/source/gpconnect-analytics.DAL/BlobService.cs +++ b/source/gpconnect-analytics.DAL/BlobService.cs @@ -57,7 +57,7 @@ public async Task AddObjectToBlob(ExtractResponse extractRespon } } - public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName) + public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName, bool overrideEntry = false) { try { @@ -66,7 +66,8 @@ public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, stri var message = new DTO.Response.Queue.Message { FileTypeId = fileTypeId, - BlobName = blobName + BlobName = blobName, + Override = overrideEntry }; var messageText = JsonConvert.SerializeObject(message); diff --git a/source/gpconnect-analytics.DAL/ConfigurationService.cs b/source/gpconnect-analytics.DAL/ConfigurationService.cs index 4b689b6..b4cc928 100644 --- a/source/gpconnect-analytics.DAL/ConfigurationService.cs +++ b/source/gpconnect-analytics.DAL/ConfigurationService.cs @@ -1,5 +1,6 @@ using gpconnect_analytics.DAL.Interfaces; using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.Helpers; using Microsoft.Extensions.Logging; using System.Collections.Generic; using System.Linq; @@ -39,6 +40,12 @@ public async Task> GetFileTypes() return result; } + public async Task GetFileType(FileTypes fileTypes) + { + var result = await _dataService.ExecuteStoredProcedure("[Configuration].[GetFileTypes]"); + return result.FirstOrDefault(ft => ft.FileTypeFilePrefix == fileTypes.ToString()); + } + public async Task GetSplunkClientConfiguration() { var result = await _dataService.ExecuteStoredProcedure("[Configuration].[GetSplunkClientConfiguration]"); diff --git a/source/gpconnect-analytics.DAL/DataService.cs b/source/gpconnect-analytics.DAL/DataService.cs index f2a5068..c01af79 100644 --- a/source/gpconnect-analytics.DAL/DataService.cs +++ b/source/gpconnect-analytics.DAL/DataService.cs @@ -30,7 +30,7 @@ public async Task> ExecuteStoredProcedure(string procedureName, Dynam { sqlConnection.InfoMessage += SqlConnection_InfoMessage; _logger.LogInformation($"Executing stored procedure {procedureName}", parameters); - var results = await sqlConnection.QueryAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600); + var results = await sqlConnection.QueryAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0); return results.AsList(); } catch (Exception exc) @@ -49,7 +49,7 @@ public async Task ExecuteStoredProcedureWithOutputParameters( { sqlConnection.InfoMessage += SqlConnection_InfoMessage; _logger.LogInformation($"Executing stored procedure {procedureName}", parameters); - await SqlMapper.ExecuteAsync(sqlConnection, procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600); + await SqlMapper.ExecuteAsync(sqlConnection, procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0); return parameters; } catch (Exception exc) @@ -69,7 +69,7 @@ public async Task ExecuteStoredProcedure(string procedureName, DynamicParam { sqlConnection.InfoMessage += SqlConnection_InfoMessage; _logger.LogInformation($"Executing stored procedure {procedureName}", parameters); - var result = await sqlConnection.ExecuteAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600); + var result = await sqlConnection.ExecuteAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0); return result; } catch (Exception exc) diff --git a/source/gpconnect-analytics.DAL/ImportService.cs b/source/gpconnect-analytics.DAL/ImportService.cs index 1c010d4..285ee89 100644 --- a/source/gpconnect-analytics.DAL/ImportService.cs +++ b/source/gpconnect-analytics.DAL/ImportService.cs @@ -1,7 +1,13 @@ using Dapper; using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.DTO.Response.Configuration; using gpconnect_analytics.DTO.Response.Queue; +using gpconnect_analytics.DTO.Response.Splunk; +using gpconnect_analytics.Helpers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using System; using System.Data; using System.Threading.Tasks; @@ -11,19 +17,61 @@ public class ImportService : IImportService { private readonly ILogger _logger; private readonly IDataService _dataService; + private readonly IBlobService _blobService; + private readonly IConfigurationService _configurationService; - public ImportService(IDataService dataService, ILogger logger) + public ImportService(IConfigurationService configurationService, IDataService dataService, IBlobService blobService, ILogger logger) { _logger = logger; + _configurationService = configurationService; _dataService = dataService; + _blobService = blobService; } - public async Task AddFile(int fileTypeId, string filePath) + public async Task AddDownloadedFileManually(HttpRequest req) + { + var fileTypes = (FileTypes?)Enum.Parse(typeof(FileTypes), req.Query["FileType"].ToString()); + if (fileTypes != null) + { + var fileType = await _configurationService.GetFileType((FileTypes)fileTypes); + var filePath = req.Query["FilePath"].ToString(); + await AddFileMessage(fileType, new ExtractResponse() { FilePath = filePath }); + return new OkObjectResult($"Import of {filePath} complete"); + } + return new BadRequestObjectResult("Bad request"); + } + + public async Task AddObjectFileMessage(FileType fileType, ExtractResponse extractResponse) + { + switch (extractResponse?.ExtractResponseMessage.StatusCode) + { + case System.Net.HttpStatusCode.OK: + var uploadedBlob = await _blobService.AddObjectToBlob(extractResponse); + if (uploadedBlob != null) + { + await AddFileMessage(fileType, extractResponse); + } + break; + default: + _logger?.LogWarning(extractResponse?.ExtractResponseMessage.ToString()); + break; + throw new Exception($"Splunk has returned the following HTTP status code {extractResponse?.ExtractResponseMessage.StatusCode}"); + } + } + + public async Task AddFileMessage(FileType fileType, ExtractResponse extractResponse) + { + var fileAddedCount = await AddFile(fileType.FileTypeId, extractResponse.FilePath, true); + await _blobService.AddMessageToBlobQueue(fileAddedCount, fileType.FileTypeId, extractResponse.FilePath, true); + } + + public async Task AddFile(int fileTypeId, string filePath, bool overrideFile) { var procedureName = "ApiReader.AddFile"; var parameters = new DynamicParameters(); parameters.Add("@FileTypeId", fileTypeId); parameters.Add("@FilePath", filePath); + parameters.Add("@Override", overrideFile); var result = await _dataService.ExecuteStoredProcedure(procedureName, parameters); return result; } @@ -34,6 +82,10 @@ public async Task InstallData(Message queueItem) var procedureName = "Import.InstallNextFile"; var parameters = new DynamicParameters(); parameters.Add("@FileTypeId", queueItem.FileTypeId); + if(queueItem.Override) + { + parameters.Add("@Override", queueItem.Override, dbType: DbType.Boolean, direction: ParameterDirection.Input); + } parameters.Add("@MoreFilesToInstall", dbType: DbType.Boolean, direction: ParameterDirection.Output); while (moreFilesToInstall) diff --git a/source/gpconnect-analytics.DAL/Interfaces/IBatchService.cs b/source/gpconnect-analytics.DAL/Interfaces/IBatchService.cs new file mode 100644 index 0000000..2e279ce --- /dev/null +++ b/source/gpconnect-analytics.DAL/Interfaces/IBatchService.cs @@ -0,0 +1,19 @@ +using gpconnect_analytics.DTO.Request; +using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.Helpers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace gpconnect_analytics.DAL.Interfaces +{ + public interface IBatchService + { + Task> GetBatchDownloadUriList(FileType fileType, List dateTimeList); + Task RemovePreviousDownloads(FileType fileType, DateTime startDate, DateTime endDate); + Task StartBatchDownloadForTodayAsync(FileTypes fileTypes); + Task StartBatchDownloadAsync(HttpRequest req, FileTypes fileTypes); + } +} diff --git a/source/gpconnect-analytics.DAL/Interfaces/IBlobService.cs b/source/gpconnect-analytics.DAL/Interfaces/IBlobService.cs index f637ae9..8264c8b 100644 --- a/source/gpconnect-analytics.DAL/Interfaces/IBlobService.cs +++ b/source/gpconnect-analytics.DAL/Interfaces/IBlobService.cs @@ -6,7 +6,7 @@ namespace gpconnect_analytics.DAL.Interfaces { public interface IBlobService { - Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName); + Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName, bool overrideEntry = false); Task AddObjectToBlob(ExtractResponse extractResponse); } } diff --git a/source/gpconnect-analytics.DAL/Interfaces/IConfigurationService.cs b/source/gpconnect-analytics.DAL/Interfaces/IConfigurationService.cs index 513a1eb..7db3757 100644 --- a/source/gpconnect-analytics.DAL/Interfaces/IConfigurationService.cs +++ b/source/gpconnect-analytics.DAL/Interfaces/IConfigurationService.cs @@ -1,4 +1,5 @@ using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.Helpers; using System.Collections.Generic; using System.Threading.Tasks; @@ -9,6 +10,7 @@ public interface IConfigurationService Task GetBlobStorageConfiguration(); Task GetFilePathConstants(); Task> GetFileTypes(); + Task GetFileType(FileTypes fileTypes); Task GetSplunkClientConfiguration(); Task GetSplunkInstance(Helpers.SplunkInstances splunkInstance); } diff --git a/source/gpconnect-analytics.DAL/Interfaces/IImportService.cs b/source/gpconnect-analytics.DAL/Interfaces/IImportService.cs index c563da2..3b0c163 100644 --- a/source/gpconnect-analytics.DAL/Interfaces/IImportService.cs +++ b/source/gpconnect-analytics.DAL/Interfaces/IImportService.cs @@ -1,4 +1,8 @@ -using gpconnect_analytics.DTO.Response.Queue; +using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.DTO.Response.Queue; +using gpconnect_analytics.DTO.Response.Splunk; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using System.Threading.Tasks; namespace gpconnect_analytics.DAL.Interfaces @@ -6,6 +10,8 @@ namespace gpconnect_analytics.DAL.Interfaces public interface IImportService { Task InstallData(Message message); - Task AddFile(int fileTypeId, string filePath); + Task AddFile(int fileTypeId, string filePath, bool overrideFile); + Task AddDownloadedFileManually(HttpRequest req); + Task AddObjectFileMessage(FileType fileType, ExtractResponse extractResponse); } } diff --git a/source/gpconnect-analytics.DAL/Interfaces/ILoggingService.cs b/source/gpconnect-analytics.DAL/Interfaces/ILoggingService.cs new file mode 100644 index 0000000..545973d --- /dev/null +++ b/source/gpconnect-analytics.DAL/Interfaces/ILoggingService.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace gpconnect_analytics.DAL.Interfaces +{ + public interface ILoggingService + { + Task PurgeErrorLog(); + } +} diff --git a/source/gpconnect-analytics.DAL/Interfaces/ISplunkService.cs b/source/gpconnect-analytics.DAL/Interfaces/ISplunkService.cs index f1068c3..dd25a4e 100644 --- a/source/gpconnect-analytics.DAL/Interfaces/ISplunkService.cs +++ b/source/gpconnect-analytics.DAL/Interfaces/ISplunkService.cs @@ -1,4 +1,5 @@ -using gpconnect_analytics.DTO.Response.Configuration; +using gpconnect_analytics.DTO.Request; +using gpconnect_analytics.DTO.Response.Configuration; using gpconnect_analytics.DTO.Response.Splunk; using System.Threading.Tasks; @@ -6,6 +7,6 @@ namespace gpconnect_analytics.DAL.Interfaces { public interface ISplunkService { - Task DownloadCSV(FileType fileType); + Task DownloadCSVDateRangeAsync(FileType fileType, UriRequest uriRequest, bool isToday); } } diff --git a/source/gpconnect-analytics.DAL/LoggingService.cs b/source/gpconnect-analytics.DAL/LoggingService.cs new file mode 100644 index 0000000..52c7d5b --- /dev/null +++ b/source/gpconnect-analytics.DAL/LoggingService.cs @@ -0,0 +1,21 @@ +using gpconnect_analytics.DAL.Interfaces; +using System.Threading.Tasks; + +namespace gpconnect_analytics.DAL +{ + public class LoggingService : ILoggingService + { + private readonly IDataService _dataService; + + public LoggingService(IDataService dataService) + { + _dataService = dataService; + } + + public async Task PurgeErrorLog() + { + var procedureName = "Logging.PurgeErrorLog"; + await _dataService.ExecuteStoredProcedure(procedureName); + } + } +} diff --git a/source/gpconnect-analytics.DAL/SplunkService.cs b/source/gpconnect-analytics.DAL/SplunkService.cs index f891810..1e6b84b 100644 --- a/source/gpconnect-analytics.DAL/SplunkService.cs +++ b/source/gpconnect-analytics.DAL/SplunkService.cs @@ -1,5 +1,6 @@ using Dapper; using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.DTO.Request; using gpconnect_analytics.DTO.Response.Configuration; using gpconnect_analytics.DTO.Response.Splunk; using Microsoft.Extensions.Logging; @@ -30,32 +31,28 @@ public SplunkService(IConfigurationService configurationService, IDataService da _dataService = dataService; _logger = logger; _httpClientFactory = httpClientFactory; - _extract = new Extract - { - ExtractRequired = false, - QueryFromDate = DateTime.Today.AddDays(-2), - QueryToDate = DateTime.Today.AddDays(-1), - }; + _extract = new Extract(); } - public async Task DownloadCSV(FileType fileType) + public async Task DownloadCSVDateRangeAsync(FileType fileType, UriRequest uriRequest, bool isToday) { try { _filePathConstants = await _configurationService.GetFilePathConstants(); var splunkInstance = await _configurationService.GetSplunkInstance(Helpers.SplunkInstances.cloud); - await GetNextExtractDetails(fileType.FileTypeId); - if (_extract.ExtractRequired) - { - var extractResponseMessage = await GetSearchResults(fileType.SplunkQuery); - var filePath = ConstructFilePath(splunkInstance, fileType); - extractResponseMessage.FilePath = filePath; - extractResponseMessage.ExtractRequestDetails = _extract; + _extract.Override = true; + _extract.QueryFromDate = uriRequest.EarliestDate; + _extract.QueryToDate = uriRequest.LatestDate; + _extract.QueryHour = uriRequest.Hour; - return extractResponseMessage; - } - return null; + var filePath = ConstructFilePath(splunkInstance, fileType, isToday, true); + var extractResponse = await GetSearchResultFromRequestUri(uriRequest); + + extractResponse.FilePath = filePath; + extractResponse.ExtractRequestDetails = _extract; + + return extractResponse; } catch (TimeoutException timeoutException) { @@ -67,53 +64,9 @@ public async Task DownloadCSV(FileType fileType) _logger.LogError(exc, "An error occurred in trying to execute a GET request"); throw; } - } - - private string ConstructFilePath(SplunkInstance splunkInstance, FileType fileType) - { - var filePathString = new StringBuilder(); - filePathString.Append(fileType.DirectoryName); - filePathString.Append(_filePathConstants.PathSeparator); - filePathString.Append(splunkInstance.Source); - filePathString.Append(_filePathConstants.PathSeparator); - filePathString.Append(_extract.QueryFromDate.ToString(Helpers.DateFormatConstants.FilePathQueryDateYearMonth)); - filePathString.Append(_filePathConstants.PathSeparator); - filePathString.Append(_filePathConstants.ProjectNameFilePrefix); - filePathString.Append(_filePathConstants.ComponentSeparator); - filePathString.Append(fileType.FileTypeFilePrefix); - filePathString.Append(_filePathConstants.ComponentSeparator); - filePathString.Append(_extract.QueryFromDate.ToString(Helpers.DateFormatConstants.FilePathQueryDate)); - filePathString.Append(_filePathConstants.ComponentSeparator); - filePathString.Append(_extract.QueryToDate.ToString(Helpers.DateFormatConstants.FilePathQueryDate)); - filePathString.Append(_filePathConstants.ComponentSeparator); - filePathString.Append(splunkInstance.Source); - filePathString.Append(_filePathConstants.ComponentSeparator); - filePathString.Append(DateTime.UtcNow.ToString(Helpers.DateFormatConstants.FilePathNowDate)); - filePathString.Append(_filePathConstants.FileExtension); - return filePathString.ToString(); - } - - private async Task GetNextExtractDetails(int fileTypeId) - { - var procedureName = "ApiReader.DetermineNextExtract"; - var parameters = new DynamicParameters(); - parameters.Add("@FileTypeId", fileTypeId); - parameters.Add("@ExtractRequired", dbType: DbType.Boolean, direction: ParameterDirection.Output); - parameters.Add("@QueryFromDate", dbType: DbType.DateTime2, direction: ParameterDirection.Output); - parameters.Add("@QueryToDate", dbType: DbType.DateTime2, direction: ParameterDirection.Output); - - _logger.LogInformation("Determining next extract details", parameters); - var result = await _dataService.ExecuteStoredProcedureWithOutputParameters(procedureName, parameters); - - if (result.Get("@ExtractRequired")) - { - _extract.ExtractRequired = result.Get("@ExtractRequired"); - _extract.QueryFromDate = result.Get("@QueryFromDate"); - _extract.QueryToDate = result.Get("@QueryToDate"); - } - } + } - private async Task GetSearchResults(string splunkQuery) + private async Task GetSearchResultFromRequestUri(UriRequest uriRequest) { var extractResponseMessage = new ExtractResponse { @@ -121,9 +74,6 @@ private async Task GetSearchResults(string splunkQuery) }; try { - splunkQuery = splunkQuery.Replace("{earliest}", _extract.QueryFromDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate)); - splunkQuery = splunkQuery.Replace("{latest}", _extract.QueryToDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate)); - _splunkClient = await _configurationService.GetSplunkClientConfiguration(); var apiTokenExpiry = HasApiTokenExpired(_splunkClient.ApiToken); @@ -133,22 +83,14 @@ private async Task GetSearchResults(string splunkQuery) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _splunkClient.ApiToken); client.Timeout = new TimeSpan(0, 0, _splunkClient.QueryTimeout); - var uriBuilder = new UriBuilder - { - Scheme = Uri.UriSchemeHttps, - Host = _splunkClient.HostName, - Port = _splunkClient.HostPort, - Path = _splunkClient.BaseUrl, - Query = string.Format(_splunkClient.QueryParameters, HttpUtility.UrlEncode(splunkQuery)) - }; - - var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, uriBuilder.Uri); + var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, uriRequest.Request); var response = await client.SendAsync(httpRequestMessage); var responseStream = await response.Content.ReadAsStreamAsync(); extractResponseMessage.ExtractResponseStream = responseStream; extractResponseMessage.ExtractResponseMessage = response; extractResponseMessage.ExtractRequestDetails = _extract; + extractResponseMessage.UriRequest = uriRequest; } else { @@ -169,6 +111,57 @@ private async Task GetSearchResults(string splunkQuery) return extractResponseMessage; } + private string ConstructFilePath(SplunkInstance splunkInstance, FileType fileType, bool isToday, bool setDateAsMidnight = false) + { + var filePathString = new StringBuilder(); + filePathString.Append(fileType.DirectoryName); + filePathString.Append(_filePathConstants.PathSeparator); + filePathString.Append(splunkInstance.Source); + filePathString.Append(_filePathConstants.PathSeparator); + filePathString.Append(_extract.QueryFromDate.ToString(Helpers.DateFormatConstants.FilePathQueryDateYearMonth)); + filePathString.Append(_filePathConstants.PathSeparator); + filePathString.Append(_filePathConstants.ProjectNameFilePrefix); + filePathString.Append(_filePathConstants.ComponentSeparator); + filePathString.Append(fileType.FileTypeFilePrefix); + filePathString.Append(_filePathConstants.ComponentSeparator); + filePathString.Append($"{_extract.QueryFromDate.ToString(Helpers.DateFormatConstants.FilePathQueryDate)}T{_extract.QueryHour.ToString(Helpers.DateFormatConstants.FilePathQueryHour)}"); + filePathString.Append(_filePathConstants.ComponentSeparator); + filePathString.Append($"{_extract.QueryToDate.ToString(Helpers.DateFormatConstants.FilePathQueryDate)}T{_extract.QueryHour.ToString(Helpers.DateFormatConstants.FilePathQueryHour)}"); + filePathString.Append(_filePathConstants.ComponentSeparator); + filePathString.Append(splunkInstance.Source); + filePathString.Append(_filePathConstants.ComponentSeparator); + if (!isToday) + { + filePathString.Append(setDateAsMidnight ? DateTime.Today.ToString(Helpers.DateFormatConstants.FilePathNowDate) : DateTime.UtcNow.ToString(Helpers.DateFormatConstants.FilePathNowDate)); + } + else + { + filePathString.Append(DateTime.Today.AddDays(1).AddSeconds(-1).ToString(Helpers.DateFormatConstants.FilePathNowDate)); + } + filePathString.Append(_filePathConstants.FileExtension); + return filePathString.ToString(); + } + + private async Task GetNextExtractDetails(int fileTypeId) + { + var procedureName = "ApiReader.DetermineNextExtract"; + var parameters = new DynamicParameters(); + parameters.Add("@FileTypeId", fileTypeId); + parameters.Add("@ExtractRequired", dbType: DbType.Boolean, direction: ParameterDirection.Output); + parameters.Add("@QueryFromDate", dbType: DbType.DateTime2, direction: ParameterDirection.Output); + parameters.Add("@QueryToDate", dbType: DbType.DateTime2, direction: ParameterDirection.Output); + + _logger.LogInformation("Determining next extract details", parameters); + var result = await _dataService.ExecuteStoredProcedureWithOutputParameters(procedureName, parameters); + + if (result.Get("@ExtractRequired")) + { + _extract.ExtractRequired = result.Get("@ExtractRequired"); + _extract.QueryFromDate = result.Get("@QueryFromDate"); + _extract.QueryToDate = result.Get("@QueryToDate"); + } + } + private (bool, DateTime) HasApiTokenExpired(string apiToken) { var jwtToken = new JwtSecurityToken(apiToken); diff --git a/source/gpconnect-analytics.DAL/gpconnect-analytics.DAL.csproj b/source/gpconnect-analytics.DAL/gpconnect-analytics.DAL.csproj index 2843ef5..eaf6218 100644 --- a/source/gpconnect-analytics.DAL/gpconnect-analytics.DAL.csproj +++ b/source/gpconnect-analytics.DAL/gpconnect-analytics.DAL.csproj @@ -11,6 +11,8 @@ + + diff --git a/source/gpconnect-analytics.DTO/Request/RequestUri.cs b/source/gpconnect-analytics.DTO/Request/RequestUri.cs new file mode 100644 index 0000000..e4798d6 --- /dev/null +++ b/source/gpconnect-analytics.DTO/Request/RequestUri.cs @@ -0,0 +1,12 @@ +using System; + +namespace gpconnect_analytics.DTO.Request +{ + public class UriRequest + { + public Uri Request { get; set; } + public DateTime EarliestDate { get; set; } + public DateTime LatestDate { get; set; } + public TimeSpan Hour { get; set; } + } +} diff --git a/source/gpconnect-analytics.DTO/Response/Queue/Message.cs b/source/gpconnect-analytics.DTO/Response/Queue/Message.cs index 07c8852..0f5e84f 100644 --- a/source/gpconnect-analytics.DTO/Response/Queue/Message.cs +++ b/source/gpconnect-analytics.DTO/Response/Queue/Message.cs @@ -4,5 +4,6 @@ public class Message { public int FileTypeId { get; set; } public string BlobName { get; set; } + public bool Override { get; set; } = false; } } diff --git a/source/gpconnect-analytics.DTO/Response/Splunk/Extract.cs b/source/gpconnect-analytics.DTO/Response/Splunk/Extract.cs index b3a1bf7..1935562 100644 --- a/source/gpconnect-analytics.DTO/Response/Splunk/Extract.cs +++ b/source/gpconnect-analytics.DTO/Response/Splunk/Extract.cs @@ -7,5 +7,7 @@ public class Extract public bool ExtractRequired { get; set; } public DateTime QueryFromDate { get; set; } public DateTime QueryToDate { get; set; } + public TimeSpan QueryHour { get; set; } + public bool Override { get; set; } = false; } } diff --git a/source/gpconnect-analytics.DTO/Response/Splunk/ExtractResponse.cs b/source/gpconnect-analytics.DTO/Response/Splunk/ExtractResponse.cs index f9e161f..49e5e39 100644 --- a/source/gpconnect-analytics.DTO/Response/Splunk/ExtractResponse.cs +++ b/source/gpconnect-analytics.DTO/Response/Splunk/ExtractResponse.cs @@ -1,4 +1,5 @@ -using System.IO; +using gpconnect_analytics.DTO.Request; +using System.IO; using System.Net.Http; namespace gpconnect_analytics.DTO.Response.Splunk @@ -9,5 +10,6 @@ public class ExtractResponse public Stream ExtractResponseStream { get; set; } public Extract ExtractRequestDetails { get; set; } public string FilePath { get; set; } + public UriRequest UriRequest { get; set; } } } diff --git a/source/gpconnect-analytics.Functions/ExecuteImport.cs b/source/gpconnect-analytics.Functions/ExecuteImport.cs deleted file mode 100644 index 7be3f97..0000000 --- a/source/gpconnect-analytics.Functions/ExecuteImport.cs +++ /dev/null @@ -1,36 +0,0 @@ -using gpconnect_analytics.DAL.Interfaces; -using gpconnect_analytics.DTO.Response.Queue; -using Microsoft.Azure.WebJobs; -using Microsoft.Extensions.Logging; -using System; -using System.Threading.Tasks; - -namespace gpconnect_analytics.Functions -{ - public class ExecuteImport - { - private readonly ILogger _logger; - private readonly IImportService _importService; - - public ExecuteImport(ILogger logger, IImportService importService) - { - _logger = logger; - _importService = importService; - } - - [FunctionName("ExecuteImport")] - public async Task Run([QueueTrigger("%QueueName%")] Message queueItem, ILogger log) - { - try - { - await _importService.InstallData(queueItem); - } - catch (Exception exc) - { - _logger?.LogError(exc, $"An error has occurred while attempting to execute an Azure function"); - throw; - } - - } - } -} diff --git a/source/gpconnect-analytics.Functions/ExecuteImportByTrigger.cs b/source/gpconnect-analytics.Functions/ExecuteImportByTrigger.cs new file mode 100644 index 0000000..99675cd --- /dev/null +++ b/source/gpconnect-analytics.Functions/ExecuteImportByTrigger.cs @@ -0,0 +1,24 @@ +using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.DTO.Response.Queue; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class ExecuteImportByTrigger + { + private readonly IImportService _importService; + + public ExecuteImportByTrigger(IImportService importService) + { + _importService = importService; + } + + [FunctionName("ExecuteImportByTrigger")] + public async Task Run([QueueTrigger("%QueueName%")] Message queueItem, ILogger log) + { + await _importService.InstallData(queueItem); + } + } +} diff --git a/source/gpconnect-analytics.Functions/GetDataFromApi.cs b/source/gpconnect-analytics.Functions/GetDataFromApi.cs deleted file mode 100644 index 0378466..0000000 --- a/source/gpconnect-analytics.Functions/GetDataFromApi.cs +++ /dev/null @@ -1,95 +0,0 @@ -using gpconnect_analytics.DAL.Interfaces; -using gpconnect_analytics.DTO.Response.Configuration; -using Microsoft.Azure.WebJobs; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace gpconnect_analytics.Functions -{ - public class GetDataFromApi - { - private readonly ILogger _logger; - private readonly IBlobService _blobService; - private readonly IImportService _importService; - private readonly ISplunkService _splunkService; - private readonly IConfigurationService _configurationService; - private readonly List _fileTypes; - - public GetDataFromApi(ILogger logger, IBlobService blobService, IImportService importService, ISplunkService splunkService, IConfigurationService configurationService) - { - _logger = logger; - _importService = importService; - _splunkService = splunkService; - _blobService = blobService; - _configurationService = configurationService; - if (_configurationService != null) - { - _fileTypes = _configurationService.GetFileTypes().Result; - } - } - - [FunctionName("GetDataFromAsidLookup")] - public async Task GetDataFromAsidLookup([TimerTrigger("0 0 5 1-7 * MON", RunOnStartup = false)] TimerInfo myTimer, ILogger log) - { - var fileType = _fileTypes.FirstOrDefault(x => x.FileTypeFilePrefix == Helpers.FileTypes.asidlookup.ToString()); - await ExecuteDownloadFromSplunk(fileType); - } - - [FunctionName("GetDataFromSspTrans")] - public async Task GetDataFromSspTrans([TimerTrigger("0 0 7 * * *", RunOnStartup = false)] TimerInfo myTimer, ILogger log) - { - var fileType = _fileTypes.FirstOrDefault(x => x.FileTypeFilePrefix == Helpers.FileTypes.ssptrans.ToString()); - await ExecuteDownloadFromSplunk(fileType); - } - - [FunctionName("GetDataFromMeshTrans")] - public async Task GetDataFromMeshTrans([TimerTrigger("0 0 8 * * *", RunOnStartup = false)] TimerInfo myTimer, ILogger log) - { - var fileType = _fileTypes.FirstOrDefault(x => x.FileTypeFilePrefix == Helpers.FileTypes.meshtrans.ToString()); - await ExecuteDownloadFromSplunk(fileType); - } - - private async Task ExecuteDownloadFromSplunk(FileType fileType) - { - try - { - if (FileTypeEnabled(fileType)) - { - var result = await _splunkService.DownloadCSV(fileType); - switch (result?.ExtractResponseMessage.StatusCode) - { - case System.Net.HttpStatusCode.OK: - var uploadedBlob = await _blobService.AddObjectToBlob(result); - if (uploadedBlob != null) - { - var fileAddedCount = await _importService.AddFile(fileType.FileTypeId, result.FilePath); - await _blobService.AddMessageToBlobQueue(fileAddedCount, fileType.FileTypeId, result.FilePath); - } - break; - default: - _logger?.LogWarning(result?.ExtractResponseMessage.ToString()); - break; - throw new Exception($"Splunk has returned the following HTTP status code {result?.ExtractResponseMessage.StatusCode}"); - } - } - else - { - _logger?.LogWarning($"Filetype {fileType.FileTypeFilePrefix} is not enabled. Please check if this is correct"); - } - } - catch (Exception exc) - { - _logger?.LogError(exc, $"An error has occurred while attempting to execute an Azure function"); - throw; - } - } - - private bool FileTypeEnabled(FileType fileType) - { - return (fileType != null && fileType.Enabled); - } - } -} diff --git a/source/gpconnect-analytics.Functions/GetDataFromApiByDateRange.cs b/source/gpconnect-analytics.Functions/GetDataFromApiByDateRange.cs new file mode 100644 index 0000000..f7f91ce --- /dev/null +++ b/source/gpconnect-analytics.Functions/GetDataFromApiByDateRange.cs @@ -0,0 +1,39 @@ +using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.Helpers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class GetDataFromApiByDateRange + { + private readonly IBatchService _batchService; + + public GetDataFromApiByDateRange(IBatchService batchService) + { + _batchService = batchService; + } + + [FunctionName("GetDataFromApiByDateRangeSspTrans")] + public async Task GetDataFromSspTransByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadAsync(req, FileTypes.ssptrans); + } + + [FunctionName("GetDataFromApiByDateRangeMeshTrans")] + public async Task GetDataFromMeshTransByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadAsync(req, FileTypes.meshtrans); + } + + [FunctionName("GetDataFromApiByDateRangeAsidLookup")] + public async Task GetDataFromAsidLookupByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadAsync(req, FileTypes.asidlookup); + } + } +} diff --git a/source/gpconnect-analytics.Functions/GetDataFromApiByTrigger.cs b/source/gpconnect-analytics.Functions/GetDataFromApiByTrigger.cs new file mode 100644 index 0000000..5741505 --- /dev/null +++ b/source/gpconnect-analytics.Functions/GetDataFromApiByTrigger.cs @@ -0,0 +1,36 @@ +using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.Helpers; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class GetDataFromApiByTrigger + { + private readonly IBatchService _batchService; + + public GetDataFromApiByTrigger(IBatchService batchService) + { + _batchService = batchService; + } + + [FunctionName("GetDataFromApiByTriggerAsidLookup")] + public async Task GetDataFromAsidLookup([TimerTrigger("%GetDataFromApiByTriggerAsidLookupSchedule%", RunOnStartup = false)] TimerInfo myTimer, ILogger log) + { + await _batchService.StartBatchDownloadForTodayAsync(FileTypes.asidlookup); + } + + [FunctionName("GetDataFromApiByTriggerSspTrans")] + public async Task GetDataFromSspTrans([TimerTrigger("%GetDataFromApiByTriggerSspTransSchedule%", RunOnStartup = false)] TimerInfo myTimer, ILogger log) + { + await _batchService.StartBatchDownloadForTodayAsync(FileTypes.ssptrans); + } + + [FunctionName("GetDataFromApiByTriggerMeshTrans")] + public async Task GetDataFromMeshTrans([TimerTrigger("%GetDataFromApiByTriggerMeshTransSchedule%", RunOnStartup = false)] TimerInfo myTimer, ILogger log) + { + await _batchService.StartBatchDownloadForTodayAsync(FileTypes.meshtrans); + } + } +} diff --git a/source/gpconnect-analytics.Functions/GetDataFromApiManual.cs b/source/gpconnect-analytics.Functions/GetDataFromApiManual.cs new file mode 100644 index 0000000..a474b5f --- /dev/null +++ b/source/gpconnect-analytics.Functions/GetDataFromApiManual.cs @@ -0,0 +1,26 @@ +using gpconnect_analytics.DAL.Interfaces; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class GetDataFromApiManual + { + private readonly IImportService _importService; + + public GetDataFromApiManual(IImportService importService) + { + _importService = importService; + } + + [FunctionName("GetDataFromApiManual")] + public async Task AddDownloadedFile([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _importService.AddDownloadedFileManually(req); + } + } +} diff --git a/source/gpconnect-analytics.Functions/GetDataFromApiToday.cs b/source/gpconnect-analytics.Functions/GetDataFromApiToday.cs new file mode 100644 index 0000000..03fe9be --- /dev/null +++ b/source/gpconnect-analytics.Functions/GetDataFromApiToday.cs @@ -0,0 +1,39 @@ +using gpconnect_analytics.DAL.Interfaces; +using gpconnect_analytics.Helpers; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class GetDataFromApiToday + { + private readonly IBatchService _batchService; + + public GetDataFromApiToday(IBatchService batchService) + { + _batchService = batchService; + } + + [FunctionName("GetDataFromApiTodaySspTrans")] + public async Task GetDataFromSspTransByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadForTodayAsync(FileTypes.ssptrans); + } + + [FunctionName("GetDataFromApiTodayMeshTrans")] + public async Task GetDataFromMeshTransByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadForTodayAsync(FileTypes.meshtrans); + } + + [FunctionName("GetDataFromApiTodayAsidLookup")] + public async Task GetDataFromAsidLookupByDateRange([HttpTrigger(AuthorizationLevel.Function, "GET", Route = null)] HttpRequest req, ILogger log) + { + return await _batchService.StartBatchDownloadForTodayAsync(FileTypes.asidlookup); + } + } +} diff --git a/source/gpconnect-analytics.Functions/PurgeErrorLogByTrigger.cs b/source/gpconnect-analytics.Functions/PurgeErrorLogByTrigger.cs new file mode 100644 index 0000000..8761068 --- /dev/null +++ b/source/gpconnect-analytics.Functions/PurgeErrorLogByTrigger.cs @@ -0,0 +1,23 @@ +using gpconnect_analytics.DAL.Interfaces; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace gpconnect_analytics.Functions +{ + public class PurgeErrorLogByTrigger + { + private readonly ILoggingService _loggingService; + + public PurgeErrorLogByTrigger(ILoggingService loggingService) + { + _loggingService = loggingService; + } + + [FunctionName("PurgeErrorLogByTrigger")] + public async Task PurgeErrorLog([TimerTrigger("%PurgeErrorLogByTriggerSchedule%", RunOnStartup = false)] TimerInfo myTimer, ILogger log) + { + await _loggingService.PurgeErrorLog(); + } + } +} diff --git a/source/gpconnect-analytics.Functions/Startup.cs b/source/gpconnect-analytics.Functions/Startup.cs index 82c4eba..f3fa4dd 100644 --- a/source/gpconnect-analytics.Functions/Startup.cs +++ b/source/gpconnect-analytics.Functions/Startup.cs @@ -19,6 +19,8 @@ public override void Configure(IFunctionsHostBuilder builder) builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); var configuration = builder.GetContext().Configuration; builder.Services.AddLogging(loggingBuilder => LoggingExtensions.ConfigureLoggingServices(loggingBuilder, configuration)); diff --git a/source/gpconnect-analytics.Functions/gpconnect-analytics.Functions.csproj b/source/gpconnect-analytics.Functions/gpconnect-analytics.Functions.csproj index a0e31fd..d2c95d7 100644 --- a/source/gpconnect-analytics.Functions/gpconnect-analytics.Functions.csproj +++ b/source/gpconnect-analytics.Functions/gpconnect-analytics.Functions.csproj @@ -1,7 +1,7 @@ netcoreapp3.1 - v3 + v4 gpconnect_analytics.Functions /home/site/wwwroot Linux @@ -10,7 +10,8 @@ - + + diff --git a/source/gpconnect-analytics.Functions/host.json b/source/gpconnect-analytics.Functions/host.json index c6a64f7..c07c3c7 100644 --- a/source/gpconnect-analytics.Functions/host.json +++ b/source/gpconnect-analytics.Functions/host.json @@ -1,14 +1,22 @@ { "version": "2.0", + "functionTimeout": "12:00:00", + "Values": { + "GetDataFromApiByTriggerAsidLookupSchedule": "0 0 2 1-7 * MON", + "GetDataFromApiByTriggerSspTransSchedule": "0 0 3 * * *", + "GetDataFromApiByTriggerMeshTransSchedule": "0 0 4 * * *" + }, "logging": { - "logLevel": { - "Default": "Information" - }, "applicationInsights": { "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" + "isEnabled": true } + }, + "fileLoggingMode": "always", + "logLevel": { + "Default": "Information", + "Host.Controllers.Host": "Warning", + "Microsoft": "Warning" } } } diff --git a/source/gpconnect-analytics.Helpers/DateFormatConstants.cs b/source/gpconnect-analytics.Helpers/DateFormatConstants.cs index 6d517ba..b30d318 100644 --- a/source/gpconnect-analytics.Helpers/DateFormatConstants.cs +++ b/source/gpconnect-analytics.Helpers/DateFormatConstants.cs @@ -2,9 +2,11 @@ { public class DateFormatConstants { - public const string FilePathQueryDate = "yyyyMMddT000000"; + public const string FilePathQueryDate = "yyyyMMdd"; + public const string FilePathQueryHour = "hhmmss"; public const string FilePathQueryDateYearMonth = "yyyy-MM"; public const string FilePathNowDate = "yyyyMMddTHHmmss"; public const string SplunkQueryDate = "MM/dd/yyyy:HH:mm:ss"; + public const string SplunkQueryHour = "hhmm"; } } diff --git a/source/gpconnect-analytics.Helpers/DateTimeHelper.cs b/source/gpconnect-analytics.Helpers/DateTimeHelper.cs new file mode 100644 index 0000000..c4db187 --- /dev/null +++ b/source/gpconnect-analytics.Helpers/DateTimeHelper.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; + +namespace gpconnect_analytics.Helpers +{ + public static class DateTimeHelper + { + public static IEnumerable EachDay(DateTime from, DateTime to) + { + for (var day = from.Date; day.Date <= to.Date; day = day.AddDays(1)) + yield return day; + } + } +} \ No newline at end of file