Expression validation (#311)

* frontend support for expression validation

* parse expression validation config

* parse config file

* evaluate expressions for validation

* add argv function

* migrate from newtonsoft and fix parsing

* improve error handling and fix numeric parse

* started making shared tests

* improved test runner

* fix list of resolved keys

* add tests and refactor jsondatamodel

* update settings

* add app-settings-rewriter to altinn-app-cli

* fix upgrade appsettings

* refactor JsonDataModel RemoveField and check deleteRow arg

* add source and throw exception

cli-tools/altinn-app-cli/Program.cs

@@ -1,5 +1,7 @@
 using System.CommandLine;
+using System.CommandLine.Invocation;
 using System.Reflection;
+using altinn_app_cli.v7Tov8.AppSettingsRewriter;
 using altinn_app_cli.v7Tov8.CodeRewriters;
 using altinn_app_cli.v7Tov8.ProcessRewriter;
 using altinn_app_cli.v7Tov8.ProjectChecks;
@@ -18,27 +20,42 @@ static async Task<int> Main(string[] args)
         var projectFolderOption = new Option<string>(name: "--folder", description: "The project folder to read", getDefaultValue: () => "CurrentDirectory");
         var projectFileOption = new Option<string>(name: "--project", description: "The project file to read relative to --folder", getDefaultValue: () => "App/App.csproj");
         var processFileOption = new Option<string>(name: "--process", description: "The process file to read relative to --folder", getDefaultValue: () => "App/config/process/process.bpmn");
+        var appSettingsFolderOption = new Option<string>(name: "--appsettings-folder", description: "The folder where the appsettings.*.json files are located", getDefaultValue: () => "App");
         var targetVersionOption = new Option<string>(name: "--target-version", description: "The target version to upgrade to", getDefaultValue: () => "8.0.0-preview.9");
         var skipCsprojUpgradeOption = new Option<bool>(name: "--skip-csproj-upgrade", description: "Skip csproj file upgrade", getDefaultValue: () => false);
         var skipCodeUpgradeOption = new Option<bool>(name: "--skip-code-upgrade", description: "Skip code upgrade", getDefaultValue: () => false);
         var skipProcessUpgradeOption = new Option<bool>(name: "--skip-process-upgrade", description: "Skip process file upgrade", getDefaultValue: () => false);
+        var skipAppSettingsUpgradeOption = new Option<bool>(name: "--skip-appsettings-upgrade", description: "Skip appsettings file upgrade", getDefaultValue: () => false);
         var rootCommand = new RootCommand("Command line interface for working with Altinn 3 Applications");
         var upgradeCommand = new Command("upgrade", "Upgrade an app from v7 to v8")
         {
             projectFolderOption,
             projectFileOption,
             processFileOption,
+            appSettingsFolderOption,
             targetVersionOption,
             skipCsprojUpgradeOption,
             skipCodeUpgradeOption,
             skipProcessUpgradeOption,
+            skipAppSettingsUpgradeOption,
         };
         rootCommand.AddCommand(upgradeCommand);
         var versionCommand = new Command("version", "Print version of altinn-app-cli");
         rootCommand.AddCommand(versionCommand);
 
-        upgradeCommand.SetHandler(async (projectFolder, projectFile, processFile, targetVersion, skipCodeUpgrade, skipProcessUpgrade, skipCsprojUpgrade) =>
+        upgradeCommand.SetHandler(
+            async (InvocationContext context) =>
             {
+                var projectFolder = context.ParseResult.GetValueForOption(projectFolderOption)!;
+                var projectFile = context.ParseResult.GetValueForOption(projectFileOption)!;
+                var processFile = context.ParseResult.GetValueForOption(processFileOption)!;
+                var appSettingsFolder = context.ParseResult.GetValueForOption(appSettingsFolderOption)!;
+                var targetVersion = context.ParseResult.GetValueForOption(targetVersionOption)!;
+                var skipCodeUpgrade = context.ParseResult.GetValueForOption(skipCodeUpgradeOption)!;
+                var skipProcessUpgrade = context.ParseResult.GetValueForOption(skipProcessUpgradeOption)!;
+                var skipCsprojUpgrade = context.ParseResult.GetValueForOption(skipCsprojUpgradeOption)!;
+                var skipAppSettingsUpgrade = context.ParseResult.GetValueForOption(skipAppSettingsUpgradeOption)!;
+
                 if (projectFolder == "CurrentDirectory")
                 {
                     projectFolder = Directory.GetCurrentDirectory();
@@ -63,11 +80,13 @@ static async Task<int> Main(string[] args)
                 {
                     projectFile = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, projectFile);
                     processFile = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, processFile);
+                    appSettingsFolder = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, appSettingsFolder);
                 }
                 else
                 {
                     projectFile = Path.Combine(projectFolder, projectFile);
                     processFile = Path.Combine(projectFolder, processFile);
+                    appSettingsFolder = Path.Combine(projectFolder, appSettingsFolder);
                 }
 
                 var projectChecks = new ProjectChecks(projectFile);
@@ -77,6 +96,7 @@ static async Task<int> Main(string[] args)
                     returnCode = 2;
                     return;
                 }
+
                 if (!skipCsprojUpgrade)
                 {
                     returnCode = await UpgradeNugetVersions(projectFile, targetVersion);
@@ -92,6 +112,11 @@ static async Task<int> Main(string[] args)
                     returnCode = await UpgradeProcess(processFile);
                 }
 
+                if (!skipAppSettingsUpgrade && returnCode == 0)
+                {
+                    returnCode = await UpgradeAppSettings(appSettingsFolder);
+                }
+
                 if (returnCode == 0)
                 {
                     Console.WriteLine("Upgrade completed without errors. Please verify that the application is still working as expected.");
@@ -100,8 +125,9 @@ static async Task<int> Main(string[] args)
                 {
                     Console.WriteLine("Upgrade completed with errors. Please check for errors in the log above.");
                 }
-            },
-            projectFolderOption, projectFileOption, processFileOption, targetVersionOption, skipCodeUpgradeOption, skipProcessUpgradeOption, skipCsprojUpgradeOption);
+            }
+        );
+
         versionCommand.SetHandler(() =>
         {
             var version = Assembly.GetEntryAssembly()?.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "Unknown";
@@ -188,4 +214,33 @@ static async Task<int> UpgradeProcess(string processFile)
 
         return 0;
     }
+
+    static async Task<int> UpgradeAppSettings(string appSettingsFolder)
+    {
+        if (!Directory.Exists(appSettingsFolder))
+        {
+            Console.WriteLine($"App settings folder {appSettingsFolder} does not exist. Please supply location with --appsettings-folder [path/to/appsettings]");
+            return 1;
+        }
+
+        if (Directory.GetFiles(appSettingsFolder, AppSettingsRewriter.APP_SETTINGS_FILE_PATTERN).Count() == 0)
+        {
+            Console.WriteLine($"No appsettings*.json files found in {appSettingsFolder}");
+            return 1;
+        }
+
+        Console.WriteLine("Trying to upgrade appsettings*.json files");
+        AppSettingsRewriter rewriter = new(appSettingsFolder);
+        rewriter.Upgrade();
+        await rewriter.Write();
+        var warnings = rewriter.GetWarnings();
+        foreach (var warning in warnings)
+        {
+            Console.WriteLine(warning);
+        }
+
+        Console.WriteLine(warnings.Any() ? "AppSettings files upgraded with warnings. Review the warnings above and make sure that the appsettings files are still valid." : "AppSettings files upgraded");
+
+        return 0;
+    }
 }

cli-tools/altinn-app-cli/v7Tov8/AppSettingsRewriter/AppSettingsRewriter.cs

@@ -0,0 +1,110 @@
+
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace altinn_app_cli.v7Tov8.AppSettingsRewriter;
+
+
+/// <summary>
+/// Rewrites the appsettings.*.json files
+/// </summary>
+public class AppSettingsRewriter
+{
+    /// <summary>
+    /// The pattern used to search for appsettings.*.json files
+    /// </summary>
+    public static readonly string APP_SETTINGS_FILE_PATTERN = "appsettings*.json";
+
+    private Dictionary<string, JsonObject> appSettingsJsonCollection;
+
+    private readonly IList<string> warnings = new List<string>();
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="AppSettingsRewriter"/> class.
+    /// </summary>
+    public AppSettingsRewriter(string appSettingsFolder)
+    {
+        appSettingsJsonCollection = new Dictionary<string, JsonObject>();
+        foreach (var file in Directory.GetFiles(appSettingsFolder, APP_SETTINGS_FILE_PATTERN))
+        {
+            var json = File.ReadAllText(file);
+            var appSettingsJson = JsonNode.Parse(json);
+            if (appSettingsJson is not JsonObject appSettingsJsonObject)
+            {
+                warnings.Add($"Unable to parse AppSettings file {file} as a json object, skipping");
+                continue;
+            }
+
+            this.appSettingsJsonCollection.Add(file, appSettingsJsonObject);
+        }
+    }
+
+    /// <summary>
+    /// Gets the warnings
+    /// </summary>
+    public IList<string> GetWarnings()
+    {
+        return warnings;
+    }
+
+    /// <summary>
+    /// Upgrades the appsettings.*.json files
+    /// </summary>
+    public void Upgrade()
+    {
+        foreach ((var fileName, var appSettingsJson) in appSettingsJsonCollection)
+        {
+            RewriteRemoveHiddenDataSetting(fileName, appSettingsJson);
+        }
+    }
+
+    /// <summary>
+    /// Writes the appsettings.*.json files
+    /// </summary>
+    public async Task Write()
+    {
+        var tasks = appSettingsJsonCollection.Select(async appSettingsFiles =>
+        {
+            appSettingsFiles.Deconstruct(out var fileName, out var appSettingsJson);
+
+            JsonSerializerOptions options = new JsonSerializerOptions
+            {
+                WriteIndented = true,
+            };
+            await File.WriteAllTextAsync(fileName, appSettingsJson.ToJsonString(options));
+        });
+
+        await Task.WhenAll(tasks);
+    }
+
+    private void RewriteRemoveHiddenDataSetting(string fileName, JsonObject settings)
+    {
+        // Look for "AppSettings" object
+        settings.TryGetPropertyValue("AppSettings", out var appSettingsNode);
+        if (appSettingsNode is not JsonObject appSettingsObject)
+        {
+            // No "AppSettings" object found, nothing to change
+            return;
+        }
+
+        // Look for "RemoveHiddenDataPreview" property
+        appSettingsObject.TryGetPropertyValue("RemoveHiddenDataPreview", out var removeHiddenDataPreviewNode);
+        if (removeHiddenDataPreviewNode is not JsonValue removeHiddenDataPreviewValue)
+        {
+            // No "RemoveHiddenDataPreview" property found, nothing to change
+            return;
+        }
+
+        // Get value of "RemoveHiddenDataPreview" property
+        if (!removeHiddenDataPreviewValue.TryGetValue<bool>(out var removeHiddenDataValue))
+        {
+            warnings.Add($"RemoveHiddenDataPreview has unexpected value {removeHiddenDataPreviewValue.ToJsonString()} in {fileName}, expected a boolean");
+            return;
+        }
+
+        appSettingsObject.Remove("RemoveHiddenDataPreview");
+        appSettingsObject.Add("RemoveHiddenData", removeHiddenDataValue);
+        appSettingsObject.Add("RequiredValidation", removeHiddenDataValue);
+
+    }
+}

src/Altinn.App.Api/Controllers/ResourceController.cs

@@ -265,8 +265,23 @@ public async Task<ActionResult> GetFooterLayout(string org, string app)
             {
                 return NoContent();
             }
-            
+
             return Ok(layout);
         }
+
+        /// <summary>
+        /// Get validation configuration file.
+        /// </summary>
+        /// <param name="org">The application owner short name</param>
+        /// <param name="app">The application name</param>
+        /// <param name="id">Unique identifier of the model to fetch validations for.</param>
+        /// <returns>The validation configuration file as json.</returns>
+        [HttpGet]
+        [Route("{org}/{app}/api/validationconfig/{id}")]
+        public ActionResult GetValidationConfiguration(string org, string app, string id)
+        {
+            string? validationConfiguration = _appResourceService.GetValidationConfiguration(id);
+            return Ok(validationConfiguration);
+        }
     }
 }

src/Altinn.App.Core/Configuration/AppSettings.cs

@@ -23,6 +23,11 @@ public class AppSettings
         /// </summary>
         public const string JSON_SCHEMA_FILENAME = "schema.json";
 
+        /// <summary>
+        /// Constant for the location of validation configuration file
+        /// </summary>
+        public const string VALIDATION_CONFIG_FILENAME = "validation.json";
+
         /// <summary>
         /// The app configuration baseUrl where files are stored in the container
         /// </summary>
@@ -83,7 +88,7 @@ public class AppSettings
         /// </summary>
         public string LayoutSetsFileName { get; set; } = "layout-sets.json";
 
-                /// <summary>
+        /// <summary>
         /// Gets or sets the name of the layout setting file name
         /// </summary>
         public string FooterFileName { get; set; } = "footer.json";
@@ -103,6 +108,11 @@ public class AppSettings
         /// </summary>
         public string JsonSchemaFileName { get; set; } = JSON_SCHEMA_FILENAME;
 
+        /// <summary>
+        /// Gets or sets The JSON schema file name
+        /// </summary>
+        public string ValidationConfigurationFileName { get; set; } = VALIDATION_CONFIG_FILENAME;
+
         /// <summary>
         /// Gets or sets the filename for application meta data
         /// </summary>
@@ -214,8 +224,18 @@ public string GetResourceFolder()
         public string AppVersion { get; set; }
 
         /// <summary>
-        /// Enable the preview functionality to load layout in backend and remove data from hidden components before validation and task completion
+        /// Enable the functionality to load layout in backend and remove data from hidden components before task completion
+        /// </summary>
+        public bool RemoveHiddenData { get; set; } = false;
+
+        /// <summary>
+        /// Enable the functionality to load layout in backend and validate required fields as defined in the layout
+        /// </summary>
+        public bool RequiredValidation { get; set; } = false;
+
+        /// <summary>
+        /// Enable the functionality to run expression validation in backend
         /// </summary>
-        public bool RemoveHiddenDataPreview { get; set; } = false;
+        public bool ExpressionValidation { get; set; } = false;
     }
 }