Skip to content

Commit

Permalink
Merge pull request #443 from microsoft/dotliquid
Browse files Browse the repository at this point in the history 
Release 5.1.0
  • Loading branch information
@sowu880
sowu880 authored Nov 25, 2022
2 parents 9e1c134 + a30d9ed commit e49b56f
Show file tree
Hide file tree
Showing 12 changed files with 370 additions and 77 deletions.
4 changes: 2 additions & 2 deletions release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ variables:
functionalTests: "**/*FunctionalTests/*.csproj"
buildConfiguration: 'Release'
major: 5
minor: 0
patch: 5
minor: 1
patch: 0
buildnum: $[counter(format('{0}.{1}.{2}',variables['major'],variables['minor'], variables['patch']), 1)]
version: $(major).$(minor).$(patch).$(buildnum)

Expand Down
61 changes: 55 additions & 6 deletions src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/OCIArtifactFunctionalTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using EnsureThat;
using Microsoft.Health.Fhir.TemplateManagement.Client;
using Microsoft.Health.Fhir.TemplateManagement.Exceptions;
using Microsoft.Health.Fhir.TemplateManagement.Models;
Expand All @@ -18,7 +17,6 @@ namespace Microsoft.Health.Fhir.TemplateManagement.FunctionalTests
{
public class OciArtifactFunctionalTests : IAsyncLifetime
{
private const string _orasCacheEnvironmentVariableName = "ORAS_CACHE";
private static readonly string _testTarGzPath = Path.Join("TestData", "TarGzFiles");
private readonly string _containerRegistryServer;
private readonly string _baseLayerTemplatePath = Path.Join(_testTarGzPath, "layerbase.tar.gz");
Expand All @@ -31,8 +29,12 @@ public class OciArtifactFunctionalTests : IAsyncLifetime
private readonly string _testMultiLayersWithValidSequenceNumberImageReference;
private readonly string _testMultiLayersWithInValidSequenceNumberImageReference;
private readonly string _testInvalidCompressedImageReference;
private string _testOneLayerImageDigest;
private string _testMultiLayerImageDigest;
private bool _isOrasValid = true;
private readonly string _orasErrorMessage = "Oras tool invalid.";
private const string _orasCacheEnvironmentVariableName = "ORAS_CACHE";
private const string _defaultOrasCacheEnvironmentVariable = ".oras/cache";

public OciArtifactFunctionalTests()
{
Expand All @@ -43,6 +45,11 @@ public OciArtifactFunctionalTests()
_testMultiLayersWithValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:multilayers_valid_sequence";
_testMultiLayersWithInValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:multilayers_invalid_sequence";
_testInvalidCompressedImageReference = _containerRegistryServer + "/templatetest:invalid_image";

if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(_orasCacheEnvironmentVariableName)))
{
Environment.SetEnvironmentVariable(_orasCacheEnvironmentVariableName, _defaultOrasCacheEnvironmentVariable);
}
}

public async Task InitializeAsync()
Expand All @@ -57,14 +64,13 @@ public async Task InitializeAsync()

public Task DisposeAsync()
{
DirectoryHelper.ClearFolder(Environment.GetEnvironmentVariable(_orasCacheEnvironmentVariableName));
return Task.CompletedTask;
}

private async Task PushOneLayerWithValidSequenceNumberAsync()
{
string command = $"push {_testOneLayerWithValidSequenceNumberImageReference} {_baseLayerTemplatePath}";
await ExecuteOrasCommandAsync(command);
_testOneLayerImageDigest = await ExecuteOrasCommandAsync(command);
}

private async Task PushOneLayerWithoutSequenceNumberAsync()
Expand All @@ -83,6 +89,7 @@ private async Task PushMultiLayersWithValidSequenceNumberAsync()
{
string command = $"push {_testMultiLayersWithValidSequenceNumberImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}";
await ExecuteOrasCommandAsync(command);
_testMultiLayerImageDigest = await ExecuteOrasCommandAsync(command);
}

private async Task PushMultiLayersWithInValidSequenceNumberAsync()
Expand All @@ -97,18 +104,32 @@ private async Task PushInvalidCompressedImageAsync()
await ExecuteOrasCommandAsync(command);
}

private async Task ExecuteOrasCommandAsync(string command)
private async Task<string> ExecuteOrasCommandAsync(string command)
{
try
{
await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory());
var output = await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory());
var digest = GetImageDigest(output);
return digest.Value;
}
catch
{
_isOrasValid = false;
return null;
}
}

private Digest GetImageDigest(string input)
{
var digests = Digest.GetDigest(input);
if (digests.Count == 0)
{
throw new OciClientException(TemplateManagementErrorCode.OrasProcessFailed, "Failed to parse image digest.");
}

return digests[0];
}

// Pull one layer image with valid sequence number, successfully pulled with base layer copied.
[Fact]
public async Task GivenOneLayerImage_WhenPulled_ArtifactsWillBePulledWithBaseLayerCopiedAsync()
Expand Down Expand Up @@ -143,6 +164,20 @@ public async Task GivenOneLayerImageWithoutSequenceNumber_WhenPulled_ArtifactsWi
DirectoryHelper.ClearFolder(outputFolder);
}

[Fact]
public async Task GivenOneLayerImage_WhenPulledUsingDigest_ArtifactsWillBePulledWithBaseLayerCopiedAsync()
{
Assert.True(_isOrasValid, _orasErrorMessage);
string outputFolder = "TestData/testOneLayerWithDigest";
DirectoryHelper.ClearFolder(outputFolder);

var testManager = new OciFileManager(_containerRegistryServer, outputFolder);
await testManager.PullOciImageAsync("templatetest", _testOneLayerImageDigest, true);
Assert.Equal(843, Directory.EnumerateFiles(outputFolder, "*.*", SearchOption.AllDirectories).Count());
Assert.Single(Directory.EnumerateFiles(Path.Combine(outputFolder, ".image", "base"), "*.tar.gz", SearchOption.AllDirectories));
DirectoryHelper.ClearFolder(outputFolder);
}

// Pull one layer image with invalid sequence number, successfully pulled with base layer copied.
[Fact]
public async Task GivenOneLayerImageWithInvalidSequenceNumber_WhenPulled_ArtifactsWillBePulledWithBaseLayerCopiedAsync()
Expand Down Expand Up @@ -194,6 +229,20 @@ public async Task GivenMultiLayersImageWithInvalidSequenceNumber_WhenPulled_Arti
DirectoryHelper.ClearFolder(outputFolder);
}

[Fact]
public async Task GivenMultiLayerImage_WhenPulledUsingDigest_ArtifactsWillBePulledWithBaseLayerCopiedAsync()
{
Assert.True(_isOrasValid, _orasErrorMessage);
string outputFolder = "TestData/testMultiLayerWithDigest";
DirectoryHelper.ClearFolder(outputFolder);

var testManager = new OciFileManager(_containerRegistryServer, outputFolder);
await testManager.PullOciImageAsync("templatetest", _testMultiLayerImageDigest, true);
Assert.Equal(10, Directory.EnumerateFiles(outputFolder, "*.*", SearchOption.AllDirectories).Count());
Assert.Single(Directory.EnumerateFiles(Path.Combine(outputFolder, ".image", "base"), "*.tar.gz", SearchOption.AllDirectories));
DirectoryHelper.ClearFolder(outputFolder);
}

// Pull invalid image, exception will be thrown.
[Fact]
public async Task GivenInvalidCompressedImage_WhenPulled_ExceptionWillBeThrownAsync()
Expand Down
125 changes: 125 additions & 0 deletions ...osoft.Health.Fhir.TemplateManagement.FunctionalTests/TemplateCollectionFunctionalTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
using Microsoft.Health.Fhir.Liquid.Converter.Models;
using Microsoft.Health.Fhir.Liquid.Converter.Processors;
using Microsoft.Health.Fhir.TemplateManagement.Client;
using Microsoft.Health.Fhir.TemplateManagement.Exceptions;
using Microsoft.Health.Fhir.TemplateManagement.Models;
using Microsoft.Health.Fhir.TemplateManagement.Utilities;
using Newtonsoft.Json.Linq;
using Xunit;

Expand All @@ -40,13 +42,24 @@ public class TemplateCollectionFunctionalTests : IAsyncLifetime
private readonly string _defaultStu3ToR4TemplateImageReference = "microsofthealth/stu3tor4templates:default";
private readonly string testOneLayerImageReference;
private readonly string testMultiLayerImageReference;
private readonly string testOneLayerOCIImageReference;
private readonly string testMultiLayerOCIImageReference;
private readonly string testInvalidImageReference;
private readonly string testInvalidTemplateImageReference;
private string testOneLayerImageDigest;
private string testMultiLayerImageDigest;
private readonly ContainerRegistry _containerRegistry = new ContainerRegistry();
private readonly ContainerRegistryInfo _containerRegistryInfo;
private static readonly string _templateDirectory = Path.Join("..", "..", "data", "Templates");
private static readonly string _sampleDataDirectory = Path.Join("..", "..", "data", "SampleData");
private static readonly string _testTarGzPath = Path.Join("TestData", "TarGzFiles");
private readonly string _baseLayerTemplatePath = Path.Join(_testTarGzPath, "layerbase.tar.gz");
private readonly string _userLayerTemplatePath = Path.Join(_testTarGzPath, "layer2.tar.gz");
private static readonly ProcessorSettings _processorSettings = new ProcessorSettings();
private bool _isOrasValid = true;
private readonly string _orasErrorMessage = "Oras tool invalid.";
private const string _orasCacheEnvironmentVariableName = "ORAS_CACHE";
private const string _defaultOrasCacheEnvironmentVariable = ".oras/cache";

public TemplateCollectionFunctionalTests()
{
Expand All @@ -60,7 +73,14 @@ public TemplateCollectionFunctionalTests()
testMultiLayerImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:multilayers";
testInvalidImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:invalidlayers";
testInvalidTemplateImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:invalidtemplateslayers";
testOneLayerOCIImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:ocionelayer";
testMultiLayerOCIImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:ocimultilayer";
token = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_containerRegistryInfo.ContainerRegistryUsername}:{_containerRegistryInfo.ContainerRegistryPassword}"));

if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(_orasCacheEnvironmentVariableName)))
{
Environment.SetEnvironmentVariable(_orasCacheEnvironmentVariableName, _defaultOrasCacheEnvironmentVariable);
}
}

public async Task InitializeAsync()
Expand All @@ -74,6 +94,10 @@ public async Task InitializeAsync()
await InitMultiLayerImageAsync();
await InitInvalidTarGzImageAsync();
await InitInvalidTemplateImageAsync();

await OrasLogin();
await PushOneLayerOCIImageAsync();
await PushMultiLayersOCIImageAsync();
}

public Task DisposeAsync()
Expand All @@ -87,6 +111,12 @@ public static IEnumerable<object[]> GetValidImageInfoWithTag()
yield return new object[] { new List<int> { 767, 838 }, "templatetest", "multilayers" };
}

public static IEnumerable<object[]> GetValidOCIImageInfoWithTag()
{
yield return new object[] { new List<int> { 838 }, "templatetest", "ocionelayer" };
yield return new object[] { new List<int> { 834, 838 }, "templatetest", "ocimultilayer" };
}

public static IEnumerable<object[]> GetHl7v2DataAndEntryTemplate()
{
var data = new List<object[]>
Expand Down Expand Up @@ -310,6 +340,50 @@ public async Task GiveImageReference_WhenGetTemplateCollection_ACorrectTemplateC
}
}

[Theory]
[MemberData(nameof(GetValidOCIImageInfoWithTag))]
public async Task GiveOCIImageReference_WhenGetTemplateCollection_ACorrectTemplateCollectionWillBeReturnedAsync(List<int> expectedTemplatesCounts, string imageName, string tag)
{
if (_containerRegistryInfo == null)
{
return;
}

Assert.True(_isOrasValid, _orasErrorMessage);

string imageReference = string.Format("{0}/{1}:{2}", _containerRegistryInfo.ContainerRegistryServer, imageName, tag);
TemplateCollectionProviderFactory factory = new TemplateCollectionProviderFactory(cache, Options.Create(_config));
var templateCollectionProvider = factory.CreateTemplateCollectionProvider(imageReference, token);
var templateCollection = await templateCollectionProvider.GetTemplateCollectionAsync();
Assert.Equal(expectedTemplatesCounts.Count(), templateCollection.Count());
for (var i = 0; i < expectedTemplatesCounts.Count(); i++)
{
Assert.Equal(expectedTemplatesCounts[i], templateCollection[i].Count());
}
}

[Fact]
public async Task GiveOCIImageReferenceWithDigest_WhenGetTemplateCollection_ACorrectTemplateCollectionWillBeReturnedAsync()
{
if (_containerRegistryInfo == null)
{
return;
}

Assert.True(_isOrasValid, _orasErrorMessage);

string imageReference = string.Format("{0}/{1}@{2}", _containerRegistryInfo.ContainerRegistryServer, "templatetest", testOneLayerImageDigest);
TemplateCollectionProviderFactory factory = new TemplateCollectionProviderFactory(cache, Options.Create(_config));
var templateCollectionProvider = factory.CreateTemplateCollectionProvider(imageReference, token);
var templateCollection = await templateCollectionProvider.GetTemplateCollectionAsync();
Assert.Single(templateCollection);

imageReference = string.Format("{0}/{1}@{2}", _containerRegistryInfo.ContainerRegistryServer, "templatetest", testMultiLayerImageDigest);
templateCollectionProvider = factory.CreateTemplateCollectionProvider(imageReference, token);
templateCollection = await templateCollectionProvider.GetTemplateCollectionAsync();
Assert.Equal(2, templateCollection.Count());
}

[Theory]
[MemberData(nameof(GetHl7v2DataAndEntryTemplate))]
public async Task GetTemplateCollectionFromAcr_WhenGivenHl7v2DataForConverting__ExpectedFhirResourceShouldBeReturnedAsync(string hl7v2Data, string entryTemplate)
Expand Down Expand Up @@ -486,5 +560,56 @@ private async Task InitInvalidTemplateImageAsync()
List<string> templateFiles = new List<string> { invalidTemplatePath };
await _containerRegistry.GenerateTemplateImageAsync(_containerRegistryInfo, testInvalidTemplateImageReference, templateFiles);
}

private async Task PushOneLayerOCIImageAsync()
{
string command = $"push {testOneLayerOCIImageReference} {_baseLayerTemplatePath}";
testOneLayerImageDigest = await ExecuteOrasCommandAsync(command);
}

private async Task PushMultiLayersOCIImageAsync()
{
string command = $"push {testMultiLayerOCIImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}";
testMultiLayerImageDigest = await ExecuteOrasCommandAsync(command);
}

private async Task OrasLogin()
{
try
{
var command = $"login {_containerRegistryInfo.ContainerRegistryServer} -u {_containerRegistryInfo.ContainerRegistryUsername} -p {_containerRegistryInfo.ContainerRegistryPassword}";
await OrasClient.OrasExecutionAsync(command);
}
catch
{
_isOrasValid = false;
}
}

private async Task<string> ExecuteOrasCommandAsync(string command)
{
try
{
var output = await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory());
var digest = GetImageDigest(output);
return digest.Value;
}
catch
{
_isOrasValid = false;
return null;
}
}

private Digest GetImageDigest(string input)
{
var digests = Digest.GetDigest(input);
if (digests.Count == 0)
{
throw new OciClientException(TemplateManagementErrorCode.OrasProcessFailed, "Failed to parse image digest.");
}

return digests[0];
}
}
}
Loading

0 comments on commit e49b56f

Please sign in to comment.