From b29b48234605c91ffa579d4c6caa7614e71173e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Krch?= Date: Tue, 25 Jul 2023 18:26:27 +0200 Subject: [PATCH 1/7] 9466233 project/syntax cleanup --- .../ServiceCollectionExtensions.cs | 54 +++++-------------- .../Services/ImportService/ImportService.cs | 54 +++++++------------ 2 files changed, 32 insertions(+), 76 deletions(-) diff --git a/src/Kentico.Xperience.Contacts.Importer/ServiceCollectionExtensions.cs b/src/Kentico.Xperience.Contacts.Importer/ServiceCollectionExtensions.cs index cfe95f5..e9572e6 100644 --- a/src/Kentico.Xperience.Contacts.Importer/ServiceCollectionExtensions.cs +++ b/src/Kentico.Xperience.Contacts.Importer/ServiceCollectionExtensions.cs @@ -57,7 +57,7 @@ async Task SendProgressReport(string message) { if (webSocket.State == WebSocketState.Open) { - var payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "msg", payload = $"{message}" })); + byte[] payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "msg", payload = $"{message}" })); await webSocket.SendAsync(new ArraySegment(payload, 0, payload.Length), WebSocketMessageType.Text, true, CancellationToken.None); } } @@ -66,7 +66,7 @@ async Task SendTooFastReport() { if (webSocket.State == WebSocketState.Open) { - var payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "toofast", payload = $"" })); + byte[] payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "toofast", payload = $"" })); await webSocket.SendAsync(new ArraySegment(payload, 0, payload.Length), WebSocketMessageType.Text, true, CancellationToken.None); } } @@ -75,7 +75,7 @@ async Task SendProgressFinished() { if (webSocket.State == WebSocketState.Open) { - var payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "finished", payload = $"" })); + byte[] payload = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "finished", payload = $"" })); await webSocket.SendAsync(new ArraySegment(payload, 0, payload.Length), WebSocketMessageType.Text, true, CancellationToken.None); } } @@ -84,7 +84,7 @@ async Task SendConfirmHeader() { if (webSocket.State == WebSocketState.Open) { - var msg = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "headerConfirmed", payload = "" })); + byte[] msg = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "headerConfirmed", payload = "" })); await webSocket.SendAsync(new ArraySegment(msg, 0, msg.Length), WebSocketMessageType.Text, true, CancellationToken.None); } } @@ -99,7 +99,7 @@ async Task SendConfirmHeader() await SendConfirmHeader(); - var consumerIsRunning = true; + bool consumerIsRunning = true; var ms = new AsynchronousStream(1024 * 32 * 500); var consumerTask = Task.Run(async () => @@ -113,13 +113,7 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => async exception => { await SendProgressReport($"{exception}"); } ); await SendProgressReport($"...finished"); - // await SendProgressFinished(); } - // catch (Exception ex) - // { - // logService.LogException(SOURCE, "CONSUMER", ex); - // await SendProgressReport($"Consumer error: {ex}"); - // } finally { consumerIsRunning = false; @@ -129,9 +123,7 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => var producerTask = Task.Run(async () => { WebSocketReceiveResult? receiveResult = null; - // try - // { - var bufferSize = 1024 * 32; + int bufferSize = 1024 * 32; while (true) { @@ -140,7 +132,7 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => break; } - var buffer = new byte[bufferSize]; + byte[] buffer = new byte[bufferSize]; receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); if (!receiveResult.CloseStatus.HasValue) @@ -153,8 +145,8 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => ms.Flush(); } - var count = receiveResult.Count; - var response = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "progress", payload = count })); + int count = receiveResult.Count; + byte[] response = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new { type = "progress", payload = count })); await webSocket.SendAsync(new ArraySegment(response, 0, response.Length), WebSocketMessageType.Text, true, CancellationToken.None); if (ms.CachedBlocks > 3500) @@ -169,28 +161,9 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => break; } } - // } - // catch (Exception ex) - // { - // logService.LogException(SOURCE, "PRODUCER", ex); - // await SendProgressReport($"Producer error: {ex}"); - // await SendProgressFinished(); - // } - // finally - // { - // - // // await consumerTask; - // // if (receiveResult != null) - // // { - // // await webSocket.CloseAsync( - // // WebSocketCloseStatus.NormalClosure, - // // receiveResult.CloseStatusDescription, - // // CancellationToken.None); - // // } - // } }); - var socketAvailable = true; + bool socketAvailable = true; try { @@ -225,9 +198,6 @@ await importService.RunImport(ms, context, async (result, totalProcessed) => logService.LogException(SOURCE, "CONSUMER", e); await SendProgressReport($"{e}"); } - // finally - // { - // } if (socketAvailable) { @@ -247,7 +217,7 @@ await webSocket.CloseAsync( while (true) { - var buffer = new byte[bufferSize]; + byte[] buffer = new byte[bufferSize]; var receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); if (!receiveResult.CloseStatus.HasValue) @@ -274,7 +244,7 @@ await webSocket.CloseAsync( ms.Seek(0, SeekOrigin.Begin); using var sr = new StreamReader(ms); - var msg = await sr.ReadToEndAsync(); + string msg = await sr.ReadToEndAsync(); var deserialized = JObject.Parse(msg); return deserialized; } diff --git a/src/Kentico.Xperience.Contacts.Importer/Services/ImportService/ImportService.cs b/src/Kentico.Xperience.Contacts.Importer/Services/ImportService/ImportService.cs index a7f9cc4..56d6574 100644 --- a/src/Kentico.Xperience.Contacts.Importer/Services/ImportService/ImportService.cs +++ b/src/Kentico.Xperience.Contacts.Importer/Services/ImportService/ImportService.cs @@ -39,7 +39,11 @@ public ContactInfoMap() private sealed class ContactDeleteArgument { + // Pragma disable reason: used implicitly +#pragma warning disable S3459 + // ReSharper disable once InconsistentNaming // kentico naming convention public Guid ContactGUID { get; set; } +#pragma warning restore S3459 }; private sealed class SimplifiedMap : ClassMap @@ -74,7 +78,7 @@ private async Task BulkDeleteContactFromCsvAsync(Stream csvStream, ImportContext { var config = new CsvConfiguration(CultureInfo.InvariantCulture) { - Delimiter = context.Delimiter, //";" // TODO tomas.krch: 2023-07-12 to config/dialog + Delimiter = context.Delimiter, PrepareHeaderForMatch = args => args.Header.ToLower(), }; @@ -83,23 +87,18 @@ private async Task BulkDeleteContactFromCsvAsync(Stream csvStream, ImportContext csv.Context.RegisterClassMap(); - var records = csv.GetRecords(); + var records = csv.GetRecordsAsync(); int totalProcessed = 0; - - async IAsyncEnumerable> Pipe2TransformBatches(IEnumerable models) + + async IAsyncEnumerable> Pipe2TransformBatches(IAsyncEnumerable models) { var currentBatch = new List(context.BatchSize); - foreach (var item in models) + await foreach (var item in models) { try { - // if (contactGuids.Contains(item.ContactGUID)) - // { - // continue; - // } - currentBatch.Add(item.ContactGUID); } catch (Exception ex) @@ -174,7 +173,7 @@ private async Task InsertContactsFromCsvAsync(Stream csvStream, ImportContext co csv.Context.RegisterClassMap(); var records = csv.GetRecords(); - var totalProcessed = 0; + int totalProcessed = 0; IEnumerable> Pipe2TransformBatches(IEnumerable models) { @@ -216,14 +215,8 @@ private async Task InsertContactsFromCsvAsync(Stream csvStream, ImportContext co LogEvents = false, }) { - // insert is not immediate (all items stored in memory before insert) so direct piping is not possible - // ContactInfoProvider.ProviderObject.BulkInsertInfos(Pipe2Transform(records), new BulkInsertSettings - // { - // BatchSize = context.BatchSize, - // Options = SqlBulkCopyOptions.Default, - // }); - - // Task? previousInsertGroupMemberBatch = null; + // we cannot use ContactInfoProvider.ProviderObject.BulkInsertInfos - insert is not immediate (all items stored in memory before insert) so direct piping is not possible + foreach (var contactBatch in Pipe2TransformBatches(records)) { ContactInfoProvider.ProviderObject.BulkInsertInfos(contactBatch.Where(x => x.insert).Select(x => x.info), new BulkInsertSettings @@ -234,28 +227,17 @@ private async Task InsertContactsFromCsvAsync(Stream csvStream, ImportContext co if (group != null) { - // if (previousInsertGroupMemberBatch != null) - // { - // await previousInsertGroupMemberBatch; - // } - - // previousInsertGroupMemberBatch = // cannot employ async insert, it is not stable (bricks bulk contact sql connection) await InsertGroupMembersAsync(contactBatch.Select(x => x.info.ContactGUID), group); } } - - // if (previousInsertGroupMemberBatch != null) - // { - // await previousInsertGroupMemberBatch; - // } } } private Task InsertGroupMembersAsync(IEnumerable contactGuids, ContactGroupInfo group) => Task.Run(() => { - var query = @" + string query = @" INSERT INTO [dbo].[OM_ContactGroupMember] ([ContactGroupMemberContactGroupID], [ContactGroupMemberType], [ContactGroupMemberRelatedID], [ContactGroupMemberFromCondition], [ContactGroupMemberFromAccount], [ContactGroupMemberFromManual]) -- OUTPUT [inserted].[ContactGroupMemberContactGroupID] @@ -270,7 +252,7 @@ FROM [dbo].[OM_ContactGroupMember] [CGM] AND [CGM].[ContactGroupMemberRelatedID] = [C].[ContactID]) "; - var jsonGuidArray = JsonConvert.SerializeObject(contactGuids); + string jsonGuidArray = JsonConvert.SerializeObject(contactGuids); ConnectionHelper.ExecuteNonQuery(query, new QueryDataParameters { @@ -283,6 +265,7 @@ FROM [dbo].[OM_ContactGroupMember] [CGM] private Task DeletedContactsAsync(List contactGuids, int batchLimit) { // for future implementation of bulk delete +#pragma warning disable S125 // var query = @"WITH [CTE]([Guid]) // AS // (SELECT CAST([l].[value] AS UNIQUEIDENTIFIER) [Guid] @@ -291,13 +274,16 @@ private Task DeletedContactsAsync(List contactGuids, int batchLimit) // FROM [dbo].[OM_Contact] // OUTPUT [deleted].[ContactID] // WHERE EXISTS (SELECT 1 FROM [CTE] WHERE [CTE].[Guid] = [ContactGUID])"; +#pragma warning restore S125 if (contactGuids.Count == 0) + { return Task.CompletedTask; + } return Task.Run(() => { - var jsonGuidArray = JsonConvert.SerializeObject(contactGuids); - var whereCondition = $""" + string jsonGuidArray = JsonConvert.SerializeObject(contactGuids); + string whereCondition = $""" EXISTS (SELECT 1 FROM OPENJSON('{jsonGuidArray}', '$') [l] WHERE CAST([l].[value] AS UNIQUEIDENTIFIER) = [ContactGUID]) """; From f5a7ac7b39c0fe334bd97a2db1c8f2debddab794 Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:09:34 -0400 Subject: [PATCH 2/7] build(devops): build pipeline node dep --- .azuredevops/pipelines/build-and-release.yaml | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.azuredevops/pipelines/build-and-release.yaml b/.azuredevops/pipelines/build-and-release.yaml index df9d076..0e1ffbc 100644 --- a/.azuredevops/pipelines/build-and-release.yaml +++ b/.azuredevops/pipelines/build-and-release.yaml @@ -39,10 +39,25 @@ stages: - name: Configuration value: Release - - name: ProjectPath + - name: ProjectFolder + value: src/Kentico.Xperience.Contacts.Importer + + - name: ProjectFilePath value: src/Kentico.Xperience.Contacts.Importer/Kentico.Xperience.Contacts.Importer.csproj steps: + - task: PowerShell@2 + displayName: Set Node.js version from package.json + inputs: + targetType: inline + script: | + Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION]"$(node -p "require('./src/${{ variables.ProjectFolder }}/Client/package.json').engines.node") + + - task: UseNode@1 + displayName: "Install Node.js from package.json version" + inputs: + version: "$(PACKAGE_JSON_NODE_VERSION)" + - task: UseDotNet@2 displayName: Select dotnet version inputs: @@ -53,7 +68,7 @@ stages: displayName: Restore dependencies inputs: command: restore - projects: ${{ variables.ProjectPath }} + projects: ${{ variables.ProjectFilePath }} feedsToUse: select restoreArguments: --locked-mode @@ -61,7 +76,7 @@ stages: displayName: Build inputs: command: build - projects: ${{ variables.ProjectPath }} + projects: ${{ variables.ProjectFilePath }} configuration: ${{ variables.Configuration }} arguments: --no-restore @@ -69,7 +84,7 @@ stages: displayName: Create NuGet package inputs: command: pack - packagesToPack: ${{ variables.ProjectPath }} + packagesToPack: ${{ variables.ProjectFilePath }} configuration: ${{ variables.Configuration }} packDirectory: $(System.DefaultWorkingDirectory)/packages includesymbols: true From 0611759774b5bf3cdd4840cd60f189d5a7c9df36 Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:28:57 -0400 Subject: [PATCH 3/7] build(devops): use pwsh to extract node version range --- .azuredevops/pipelines/build-and-release.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.azuredevops/pipelines/build-and-release.yaml b/.azuredevops/pipelines/build-and-release.yaml index 0e1ffbc..11b1ca8 100644 --- a/.azuredevops/pipelines/build-and-release.yaml +++ b/.azuredevops/pipelines/build-and-release.yaml @@ -51,7 +51,12 @@ stages: inputs: targetType: inline script: | - Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION]"$(node -p "require('./src/${{ variables.ProjectFolder }}/Client/package.json').engines.node") + $fileContent = Get-Content -Path './src/${{ variables.ProjectFolder }}/Client/package.json' -Raw + $jsonObject = ConvertFrom-Json -InputObject $fileContent + + # Get the value of engines.node + $enginesNode = $jsonObject.engines.node + Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION;isOutput=true]"$enginesNode - task: UseNode@1 displayName: "Install Node.js from package.json version" From a40c1ef436f8b62b012ffcd3c6f3d399839c9a9e Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:31:47 -0400 Subject: [PATCH 4/7] build(devops): fix project path --- .azuredevops/pipelines/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azuredevops/pipelines/build-and-release.yaml b/.azuredevops/pipelines/build-and-release.yaml index 11b1ca8..8eae267 100644 --- a/.azuredevops/pipelines/build-and-release.yaml +++ b/.azuredevops/pipelines/build-and-release.yaml @@ -51,7 +51,7 @@ stages: inputs: targetType: inline script: | - $fileContent = Get-Content -Path './src/${{ variables.ProjectFolder }}/Client/package.json' -Raw + $fileContent = Get-Content -Path './${{ variables.ProjectFolder }}/Client/package.json' -Raw $jsonObject = ConvertFrom-Json -InputObject $fileContent # Get the value of engines.node From 1daf3ac0a68bd7b902c49795b2e6fa57fa7b60bd Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:35:36 -0400 Subject: [PATCH 5/7] build(devops): pipeline typo --- .azuredevops/pipelines/build-and-release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azuredevops/pipelines/build-and-release.yaml b/.azuredevops/pipelines/build-and-release.yaml index 8eae267..5cc1774 100644 --- a/.azuredevops/pipelines/build-and-release.yaml +++ b/.azuredevops/pipelines/build-and-release.yaml @@ -56,7 +56,8 @@ stages: # Get the value of engines.node $enginesNode = $jsonObject.engines.node - Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION;isOutput=true]"$enginesNode + Write-Host "Required Node version $enginesNode" + Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION;isOutput=true]$enginesNode" - task: UseNode@1 displayName: "Install Node.js from package.json version" From 1cfec1c4345df84572c9ea48f270aec37331720c Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:42:19 -0400 Subject: [PATCH 6/7] build(devops): remove isOutput for variable --- .azuredevops/pipelines/build-and-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azuredevops/pipelines/build-and-release.yaml b/.azuredevops/pipelines/build-and-release.yaml index 5cc1774..2f56927 100644 --- a/.azuredevops/pipelines/build-and-release.yaml +++ b/.azuredevops/pipelines/build-and-release.yaml @@ -57,12 +57,12 @@ stages: # Get the value of engines.node $enginesNode = $jsonObject.engines.node Write-Host "Required Node version $enginesNode" - Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION;isOutput=true]$enginesNode" + Write-Host "##vso[task.setvariable variable=PACKAGE_JSON_NODE_VERSION]$enginesNode" - task: UseNode@1 displayName: "Install Node.js from package.json version" inputs: - version: "$(PACKAGE_JSON_NODE_VERSION)" + version: $(PACKAGE_JSON_NODE_VERSION) - task: UseDotNet@2 displayName: Select dotnet version From d9cca429849db46aaf03337bbfa0737feeedc76b Mon Sep 17 00:00:00 2001 From: "Sean G. Wright" Date: Tue, 25 Jul 2023 14:46:12 -0400 Subject: [PATCH 7/7] fix(sln): logo path rename --- images/{icon.png => logo.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename images/{icon.png => logo.png} (100%) diff --git a/images/icon.png b/images/logo.png similarity index 100% rename from images/icon.png rename to images/logo.png