Latest set of gpconnect analytics changes

source/gpconnect-analytics.DAL/BatchService.cs

@@ -0,0 +1,161 @@
+using Dapper;
+using gpconnect_analytics.DAL.Interfaces;
+using gpconnect_analytics.DTO.Request;
+using gpconnect_analytics.DTO.Response.Configuration;
+using gpconnect_analytics.Helpers;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Web;
+
+namespace gpconnect_analytics.DAL
+{
+    public class BatchService : IBatchService
+    {
+        private readonly IConfigurationService _configurationService;
+        private readonly ILogger<BatchService> _logger;
+        private SplunkClient _splunkClient;
+        private readonly IDataService _dataService;
+        private readonly ISplunkService _splunkService;
+        private readonly IImportService _importService;
+
+        public BatchService(IConfigurationService configurationService, IImportService importService, ISplunkService splunkService, ILogger<BatchService> logger, IDataService dataService)
+        {
+            _configurationService = configurationService;
+            _logger = logger;
+            _dataService = dataService;
+            _splunkService = splunkService;
+            _importService = importService;
+        }
+
+        public async Task<IActionResult> StartBatchDownloadForTodayAsync(FileTypes fileTypes)
+        {
+            var dateInScope = DateTime.Today.AddDays(1);
+            var fileType = await _configurationService.GetFileType(fileTypes);
+            var uriList = await GetBatchDownloadUriList(fileType, DateTimeHelper.EachDay(dateInScope, dateInScope).ToList());
+
+            await RemovePreviousDownloads(fileType, dateInScope, dateInScope);
+            return await ProcessUrls(fileType, uriList, true);
+        }
+
+        public async Task<IActionResult> StartBatchDownloadAsync(HttpRequest req, FileTypes fileTypes)
+        {
+            if (req != null)
+            {
+                var startDate = DateTime.TryParse(req.Query["StartDate"].ToString(), out DateTime start) ? start : DateTime.Today;
+                var endDate = DateTime.TryParse(req.Query["EndDate"].ToString(), out DateTime end) ? end : DateTime.Today;
+
+                if (endDate >= startDate)
+                {
+                    var fileType = await _configurationService.GetFileType(fileTypes);
+                    var uriList = await GetBatchDownloadUriList(fileType, DateTimeHelper.EachDay(startDate, endDate).ToList());
+
+                    await RemovePreviousDownloads(fileType, startDate, endDate);
+
+                    return await ProcessUrls(fileType, uriList, false);
+                }
+            }
+            return new BadRequestObjectResult("Bad request");
+        }
+
+        private async Task<IActionResult> ProcessUrls(FileType fileType, List<UriRequest> uriList, bool isToday)
+        {
+            for (var i = 0; i < uriList.Count; i++)
+            {
+                var downloadTasksQuery =
+                from requestUri in uriList.Skip(i).Take(1)
+                select ExecuteBatchDownloadFromSplunk(fileType, requestUri, isToday);
+
+                var downloadTasks = downloadTasksQuery.ToList();
+
+                while (downloadTasks.Any())
+                {
+                    Task finishedTask = await Task.WhenAny(downloadTasks);
+                    downloadTasks.Remove(finishedTask);
+                }
+                await Task.Delay(TimeSpan.FromSeconds(10));
+            }
+            return new OkObjectResult($"Batch download complete: {uriList.Count} requests processed");
+        }
+
+        private async Task ExecuteBatchDownloadFromSplunk(FileType fileType, UriRequest uriRequest, bool isToday)
+        {
+            try
+            {
+                if (FileTypeEnabled(fileType))
+                {
+                    var extractResponse = await _splunkService.DownloadCSVDateRangeAsync(fileType, uriRequest, isToday);
+                    await _importService.AddObjectFileMessage(fileType, extractResponse);
+                }
+                else
+                {
+                    _logger?.LogWarning($"Filetype {fileType.FileTypeFilePrefix} is not enabled. Please check if this is correct");
+                }
+            }
+            catch (Exception exc)
+            {
+                _logger?.LogError(exc, $"An error has occurred while attempting to execute an Azure function");
+                throw;
+            }
+        }
+
+        public async Task<List<UriRequest>> GetBatchDownloadUriList(FileType fileType, List<DateTime> dateTimeList)
+        {
+            var uriList = new List<UriRequest>();
+            _splunkClient = await _configurationService.GetSplunkClientConfiguration();
+
+            foreach (var dateTime in dateTimeList)
+            {
+                var earliestDate = dateTime.AddDays(-2);
+                var latestDate = dateTime.AddDays(-1);
+
+                for (var i = 0; i < 24; i++)
+                {
+                    var splunkQuery = fileType.SplunkQuery;
+                    var hour = TimeSpan.Zero.Add(TimeSpan.FromHours(i));
+
+                    splunkQuery = splunkQuery.Replace("{earliest}", earliestDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate));
+                    splunkQuery = splunkQuery.Replace("{latest}", latestDate.ToString(Helpers.DateFormatConstants.SplunkQueryDate));
+                    splunkQuery = splunkQuery.Replace("{hour}", hour.ToString(Helpers.DateFormatConstants.SplunkQueryHour));
+
+                    var uriBuilder = new UriBuilder
+                    {
+                        Scheme = Uri.UriSchemeHttps,
+                        Host = _splunkClient.HostName,
+                        Port = _splunkClient.HostPort,
+                        Path = _splunkClient.BaseUrl,
+                        Query = string.Format(_splunkClient.QueryParameters, HttpUtility.UrlEncode(splunkQuery))
+                    };
+
+                    uriList.Add(new UriRequest()
+                    {
+                        Request = uriBuilder.Uri,
+                        EarliestDate = earliestDate,
+                        LatestDate = latestDate,
+                        Hour = hour
+                    });
+                }
+            }
+            return uriList;
+        }
+
+        public async Task RemovePreviousDownloads(FileType fileType, DateTime startDate, DateTime endDate)
+        {
+            var procedureName = "Import.RemovePreviousDownload";
+            var parameters = new DynamicParameters();
+            parameters.Add("@FileTypeId", fileType.FileTypeId);
+            parameters.Add("@StartDate", startDate.AddDays(-2));
+            parameters.Add("@EndDate", endDate.AddDays(-1));
+            await _dataService.ExecuteStoredProcedure(procedureName, parameters);        
+        }
+
+        private bool FileTypeEnabled(FileType fileType)
+        {
+            return (fileType != null && fileType.Enabled);
+        }
+    }
+}

source/gpconnect-analytics.DAL/BlobService.cs

@@ -57,7 +57,7 @@ public async Task<BlobContentInfo> AddObjectToBlob(ExtractResponse extractRespon
             }
         }
 
-        public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName)
+        public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName, bool overrideEntry = false)
         {
             try
             {
@@ -66,7 +66,8 @@ public async Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, stri
                     var message = new DTO.Response.Queue.Message
                     {
                         FileTypeId = fileTypeId,
-                        BlobName = blobName
+                        BlobName = blobName,
+                        Override = overrideEntry
                     };
 
                     var messageText = JsonConvert.SerializeObject(message);

source/gpconnect-analytics.DAL/ConfigurationService.cs

@@ -1,5 +1,6 @@
 using gpconnect_analytics.DAL.Interfaces;
 using gpconnect_analytics.DTO.Response.Configuration;
+using gpconnect_analytics.Helpers;
 using Microsoft.Extensions.Logging;
 using System.Collections.Generic;
 using System.Linq;
@@ -39,6 +40,12 @@ public async Task<List<FileType>> GetFileTypes()
             return result;
         }
 
+        public async Task<FileType> GetFileType(FileTypes fileTypes)
+        {
+            var result = await _dataService.ExecuteStoredProcedure<FileType>("[Configuration].[GetFileTypes]");
+            return result.FirstOrDefault(ft => ft.FileTypeFilePrefix == fileTypes.ToString());
+        }
+
         public async Task<SplunkClient> GetSplunkClientConfiguration()
         {
             var result = await _dataService.ExecuteStoredProcedure<SplunkClient>("[Configuration].[GetSplunkClientConfiguration]");

source/gpconnect-analytics.DAL/DataService.cs

@@ -30,7 +30,7 @@ public async Task<List<T>> ExecuteStoredProcedure<T>(string procedureName, Dynam
                 {
                     sqlConnection.InfoMessage += SqlConnection_InfoMessage;
                     _logger.LogInformation($"Executing stored procedure {procedureName}", parameters);
-                    var results = await sqlConnection.QueryAsync<T>(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600);
+                    var results = await sqlConnection.QueryAsync<T>(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0);
                     return results.AsList();
                 }
                 catch (Exception exc)
@@ -49,7 +49,7 @@ public async Task<DynamicParameters> ExecuteStoredProcedureWithOutputParameters(
                 {
                     sqlConnection.InfoMessage += SqlConnection_InfoMessage;
                     _logger.LogInformation($"Executing stored procedure {procedureName}", parameters);
-                    await SqlMapper.ExecuteAsync(sqlConnection, procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600);
+                    await SqlMapper.ExecuteAsync(sqlConnection, procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0);
                     return parameters;
                 }
                 catch (Exception exc)
@@ -69,7 +69,7 @@ public async Task<int> ExecuteStoredProcedure(string procedureName, DynamicParam
                 {
                     sqlConnection.InfoMessage += SqlConnection_InfoMessage;
                     _logger.LogInformation($"Executing stored procedure {procedureName}", parameters);
-                    var result = await sqlConnection.ExecuteAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 600);
+                    var result = await sqlConnection.ExecuteAsync(procedureName, parameters, commandType: System.Data.CommandType.StoredProcedure, commandTimeout: 0);
                     return result;
                 }
                 catch (Exception exc)

source/gpconnect-analytics.DAL/ImportService.cs

@@ -1,7 +1,13 @@
 using Dapper;
 using gpconnect_analytics.DAL.Interfaces;
+using gpconnect_analytics.DTO.Response.Configuration;
 using gpconnect_analytics.DTO.Response.Queue;
+using gpconnect_analytics.DTO.Response.Splunk;
+using gpconnect_analytics.Helpers;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using System;
 using System.Data;
 using System.Threading.Tasks;
 
@@ -11,19 +17,61 @@ public class ImportService : IImportService
     {
         private readonly ILogger<ImportService> _logger;
         private readonly IDataService _dataService;
+        private readonly IBlobService _blobService;
+        private readonly IConfigurationService _configurationService;
 
-        public ImportService(IDataService dataService, ILogger<ImportService> logger)
+        public ImportService(IConfigurationService configurationService, IDataService dataService, IBlobService blobService, ILogger<ImportService> logger)
         {
             _logger = logger;
+            _configurationService = configurationService;
             _dataService = dataService;
+            _blobService = blobService;
         }
 
-        public async Task<int> AddFile(int fileTypeId, string filePath)
+        public async Task<IActionResult> AddDownloadedFileManually(HttpRequest req)
+        {
+            var fileTypes = (FileTypes?)Enum.Parse(typeof(FileTypes), req.Query["FileType"].ToString());
+            if (fileTypes != null)
+            {
+                var fileType = await _configurationService.GetFileType((FileTypes)fileTypes);
+                var filePath = req.Query["FilePath"].ToString();
+                await AddFileMessage(fileType, new ExtractResponse() { FilePath = filePath });
+                return new OkObjectResult($"Import of {filePath} complete");
+            }
+            return new BadRequestObjectResult("Bad request");
+        }
+
+        public async Task AddObjectFileMessage(FileType fileType, ExtractResponse extractResponse)
+        {
+            switch (extractResponse?.ExtractResponseMessage.StatusCode)
+            {
+                case System.Net.HttpStatusCode.OK:
+                    var uploadedBlob = await _blobService.AddObjectToBlob(extractResponse);
+                    if (uploadedBlob != null)
+                    {
+                        await AddFileMessage(fileType, extractResponse);
+                    }
+                    break;
+                default:
+                    _logger?.LogWarning(extractResponse?.ExtractResponseMessage.ToString());
+                    break;
+                    throw new Exception($"Splunk has returned the following HTTP status code {extractResponse?.ExtractResponseMessage.StatusCode}");
+            }
+        }
+
+        public async Task AddFileMessage(FileType fileType, ExtractResponse extractResponse)
+        {
+            var fileAddedCount = await AddFile(fileType.FileTypeId, extractResponse.FilePath, true);
+            await _blobService.AddMessageToBlobQueue(fileAddedCount, fileType.FileTypeId, extractResponse.FilePath, true);
+        }
+
+        public async Task<int> AddFile(int fileTypeId, string filePath, bool overrideFile)
         {
             var procedureName = "ApiReader.AddFile";
             var parameters = new DynamicParameters();
             parameters.Add("@FileTypeId", fileTypeId);
             parameters.Add("@FilePath", filePath);
+            parameters.Add("@Override", overrideFile);
             var result = await _dataService.ExecuteStoredProcedure(procedureName, parameters);
             return result;
         }
@@ -34,6 +82,10 @@ public async Task InstallData(Message queueItem)
             var procedureName = "Import.InstallNextFile";
             var parameters = new DynamicParameters();
             parameters.Add("@FileTypeId", queueItem.FileTypeId);
+            if(queueItem.Override)
+            {
+                parameters.Add("@Override", queueItem.Override, dbType: DbType.Boolean, direction: ParameterDirection.Input);
+            }
             parameters.Add("@MoreFilesToInstall", dbType: DbType.Boolean, direction: ParameterDirection.Output);
 
             while (moreFilesToInstall)

source/gpconnect-analytics.DAL/Interfaces/IBatchService.cs

@@ -0,0 +1,19 @@
+using gpconnect_analytics.DTO.Request;
+using gpconnect_analytics.DTO.Response.Configuration;
+using gpconnect_analytics.Helpers;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace gpconnect_analytics.DAL.Interfaces
+{
+    public interface IBatchService
+    {
+        Task<List<UriRequest>> GetBatchDownloadUriList(FileType fileType, List<DateTime> dateTimeList);
+        Task RemovePreviousDownloads(FileType fileType, DateTime startDate, DateTime endDate);
+        Task<IActionResult> StartBatchDownloadForTodayAsync(FileTypes fileTypes);
+        Task<IActionResult> StartBatchDownloadAsync(HttpRequest req, FileTypes fileTypes);
+    }
+}

source/gpconnect-analytics.DAL/Interfaces/IBlobService.cs

@@ -6,7 +6,7 @@ namespace gpconnect_analytics.DAL.Interfaces
 {
     public interface IBlobService
     {
-        Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName);
+        Task AddMessageToBlobQueue(int fileAddedCount, int fileTypeId, string blobName, bool overrideEntry = false);
         Task<BlobContentInfo> AddObjectToBlob(ExtractResponse extractResponse);
     }
 }

source/gpconnect-analytics.DAL/Interfaces/IConfigurationService.cs

@@ -1,4 +1,5 @@
 using gpconnect_analytics.DTO.Response.Configuration;
+using gpconnect_analytics.Helpers;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -9,6 +10,7 @@ public interface IConfigurationService
         Task<BlobStorage> GetBlobStorageConfiguration();
         Task<FilePathConstants> GetFilePathConstants();
         Task<List<FileType>> GetFileTypes();
+        Task<FileType> GetFileType(FileTypes fileTypes);
         Task<SplunkClient> GetSplunkClientConfiguration();
         Task<SplunkInstance> GetSplunkInstance(Helpers.SplunkInstances splunkInstance);
     }

source/gpconnect-analytics.DAL/Interfaces/IImportService.cs

@@ -1,11 +1,17 @@
-using gpconnect_analytics.DTO.Response.Queue;
+using gpconnect_analytics.DTO.Response.Configuration;
+using gpconnect_analytics.DTO.Response.Queue;
+using gpconnect_analytics.DTO.Response.Splunk;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
 using System.Threading.Tasks;
 
 namespace gpconnect_analytics.DAL.Interfaces
 {
     public interface IImportService
     {
         Task InstallData(Message message);
-        Task<int> AddFile(int fileTypeId, string filePath);
+        Task<int> AddFile(int fileTypeId, string filePath, bool overrideFile);
+        Task<IActionResult> AddDownloadedFileManually(HttpRequest req);
+        Task AddObjectFileMessage(FileType fileType, ExtractResponse extractResponse);
     }
 }

source/gpconnect-analytics.DAL/Interfaces/ILoggingService.cs

@@ -0,0 +1,9 @@
+using System.Threading.Tasks;
+
+namespace gpconnect_analytics.DAL.Interfaces
+{
+    public interface ILoggingService
+    {
+        Task PurgeErrorLog();
+    }
+}