From 0c19e2f647e955875c39594bfaf7d0f00550326a Mon Sep 17 00:00:00 2001 From: Merijn van Aartrijk Date: Wed, 3 Jul 2024 17:13:23 +0200 Subject: [PATCH 1/4] Added bicep deploy files. --- .gitignore | 2 - .../deploy/appServiceModule.bicep | 357 ++++++++++++++++++ .../deploy/customDomainModule.bicep | 34 ++ .../deploy/main.bicep | 48 +++ .../deploy/resourceGroupModule.bicep | 9 + .../deploy/sslBindingModule.bicep | 11 + deploy/dataspaceAlertModule.bicep | 80 ++++ deploy/dataspaceWorkspaceModule.bicep | 35 ++ 8 files changed, 574 insertions(+), 2 deletions(-) create mode 100644 Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep create mode 100644 Poort8.Dataspace.CoreManager/deploy/customDomainModule.bicep create mode 100644 Poort8.Dataspace.CoreManager/deploy/main.bicep create mode 100644 Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep create mode 100644 Poort8.Dataspace.CoreManager/deploy/sslBindingModule.bicep create mode 100644 deploy/dataspaceAlertModule.bicep create mode 100644 deploy/dataspaceWorkspaceModule.bicep diff --git a/.gitignore b/.gitignore index 5b615b4..b0040ae 100644 --- a/.gitignore +++ b/.gitignore @@ -371,5 +371,3 @@ FodyWeavers.xsd **/Identity.db **/Identity.db-shm **/Identity.db-wal -**/deploy/ -.github/workflows/ diff --git a/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep b/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep new file mode 100644 index 0000000..8a0671e --- /dev/null +++ b/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep @@ -0,0 +1,357 @@ +@allowed([ + 'prod' + 'preview' +]) +param environment string = 'preview' + +param location string = resourceGroup().location + +@minLength(3) +param appName string + +param workspaceId string + +param alertEmailAdresses array + +param errorAlertName string + +param errorAlertShortName string + +param criticalAlertName string + +param criticalAlertShortName string + +param availabilityAlertShortName string + +param customDomain string = '' + +@secure() +param sqlAdminPassword string + +@secure() +param brevoApiKey string + +var environmentSettings = { + prod: { + } + preview: { + appServicePlan: { + sku: { + name: 'B1' + } + } + sqlDatabase: { + sku: { + name: 'Basic' + } + properties: {} + } + appService: { + UseCase: { + name: 'CoreManagerOptions:UseCase' + value: 'default' + } + JwtTokenAuthority: { + name: 'CoreManagerOptions:JwtTokenAuthority' + value: 'https://poort8.eu.auth0.com/' + } + JwtTokenAudience: { + name: 'CoreManagerOptions:JwtTokenAudience' + value: 'Poort8-Dataspace-CoreManager' + } + RegisterNewUsersDisabled: { + name: 'FeatureManagement:RegisterNewUsersDisabled' + value: true + } + AROrgAndRGPagesDisabled: { + name: 'FeatureManagement:AROrgAndRGPagesDisabled' + value: false + } + ApiDisabled: { + name: 'FeatureManagement:ApiDisabled' + value: false + } + IshareEnabled: { + name: 'FeatureManagement:IshareEnabled' + value: false + } + ClientId: { + name: 'IshareCoreOptions:ClientId' + value: 'ClientId' + } + SatelliteId: { + name: 'IshareCoreOptions:SatelliteId' + value: 'SatelliteId' + } + SatelliteUrl: { + name: 'IshareCoreOptions:SatelliteUrl' + value: 'https://SatelliteUrl.nl' + } + ManageEntitiesApi: { + name: 'Roles:ManageEntitiesApi' + value: 'your@email.nl' + } + CanSetPolicyIssuer: { + name: 'Roles:CanSetPolicyIssuer' + value: 'your@email.nl' + } + } + } +} + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: '${appName}-asp' + location: location + sku: environmentSettings[environment].appServicePlan.sku +} + +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: '${appName}-ai' + location: location + kind: 'web' + properties: { + Application_Type: 'web' + Request_Source: 'rest' + IngestionMode: 'LogAnalytics' + WorkspaceResourceId: workspaceId + } +} + +resource applicationInsightsAudit 'Microsoft.Insights/components@2020-02-02' = { + name: '${appName}-audit' + location: location + kind: 'web' + properties: { + Application_Type: 'web' + Request_Source: 'rest' + IngestionMode: 'LogAnalytics' + WorkspaceResourceId: workspaceId + } +} + +module errorAlert '../../deploy/dataspaceAlertModule.bicep' = { + name: 'ErrorAlert' + params: { + environment: environment + location: location + alertName: errorAlertName + actionGroupShortName: errorAlertShortName + emailAdresses: alertEmailAdresses + evaluationFrequency: 'P1D' + query: 'AppTraces\n| where Message has "P8.err"\n| project TimeGenerated, Message, AppRoleName' + severity: 1 + workspaceId: workspaceId + } +} + +module criticalAlert '../../deploy/dataspaceAlertModule.bicep' = { + name: 'CriticalAlert' + params: { + environment: environment + location: location + alertName: criticalAlertName + actionGroupShortName: criticalAlertShortName + emailAdresses: alertEmailAdresses + evaluationFrequency: 'PT5M' + query: 'AppTraces\n| where Message has "P8.crit"' + severity: 0 + workspaceId: workspaceId + } +} + +resource actionGroup 'Microsoft.Insights/actionGroups@2023-01-01' = { + name: '${appName}-action-group' + location: 'global' + properties: { + enabled: true + groupShortName: availabilityAlertShortName + emailReceivers: [for emailAdress in alertEmailAdresses: { + name: emailAdress + emailAddress: emailAdress + useCommonAlertSchema: true + }] + } +} + +resource availabilityTest 'Microsoft.Insights/webtests@2022-06-15' = { + name: '${appName}-availability-test' + location: location + tags: { + 'hidden-link:${applicationInsights.id}': 'Resource' + } + properties: { + Configuration: { + WebTest: ' ' + } + Enabled: true + Kind: 'ping' + RetryEnabled: true + Frequency: 300 + Timeout: 120 + Locations: [ + { Id: 'emea-fr-pra-edge'} + { Id: 'emea-gb-db3-azr' } + { Id: 'emea-se-sto-edge' } + { Id: 'emea-nl-ams-azr' } + { Id: 'emea-ru-msa-edge' } + ] + Name: '${appName}-health-test' + SyntheticMonitorId: '${appName}-availability-test' + } +} + +resource pingAlertRule 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: '${appName}-avail-alert' + location: 'global' + properties: { + actions: [ + { + actionGroupId: actionGroup.id + } + ] + criteria: { + 'odata.type': 'Microsoft.Azure.Monitor.WebtestLocationAvailabilityCriteria' + webTestId: availabilityTest.id + componentId: applicationInsights.id + failedLocationCount: 5 + } + enabled: true + evaluationFrequency: 'PT5M' + scopes: [ + availabilityTest.id + applicationInsights.id + ] + severity: 1 + windowSize: 'PT5M' + } +} + +resource sqlServer 'Microsoft.Sql/servers@2023-05-01-preview' = { + name: '${toLower(appName)}-sql-server' + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + administratorLogin: 'Poort8' + administratorLoginPassword: sqlAdminPassword + minimalTlsVersion: '1.2' + } +} + +resource sqlAllowAllWindowsAzureIps 'Microsoft.Sql/servers/firewallRules@2023-05-01-preview' = { + name: 'AllowAllWindowsAzureIps' + parent: sqlServer + properties: { + startIpAddress: '0.0.0.0' + endIpAddress: '0.0.0.0' + } +} + +resource sqlDB 'Microsoft.Sql/servers/databases@2023-05-01-preview' = { + parent: sqlServer + name: toLower(appName) + location: location + sku: environmentSettings[environment].sqlDatabase.sku + properties: environmentSettings[environment].sqlDatabase.properties +} + +resource appService 'Microsoft.Web/sites@2022-09-01' = { + name: appName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: appServicePlan.id + vnetRouteAllEnabled: true + httpsOnly: true + clientAffinityEnabled: true + siteConfig: { + appSettings: [ + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: applicationInsights.properties.InstrumentationKey + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsights.properties.ConnectionString + } + { + name: 'AUDIT_APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsightsAudit.properties.ConnectionString + } + { + name: 'ApplicationInsightsAgent_EXTENSION_VERSION' + value: '~2' + } + { + name: 'XDT_MicrosoftApplicationInsights_Mode' + value: 'Recommended' + } + { + name: 'WEBSITE_TIME_ZONE' + value: 'W. Europe Standard Time' + } + { + name: 'WEBSITE_LOAD_USER_PROFILE' + value: '1' + } + { + name: 'OrganizationRegistry:ConnectionString' + value: 'Data Source=tcp:${sqlServer.properties.fullyQualifiedDomainName},1433;Initial Catalog=${sqlDB.name};User Id=${sqlServer.properties.administratorLogin}@${sqlServer.properties.fullyQualifiedDomainName};Password=${sqlAdminPassword};' + } + { + name: 'AuthorizationRegistry:ConnectionString' + value: 'Data Source=tcp:${sqlServer.properties.fullyQualifiedDomainName},1433;Initial Catalog=${sqlDB.name};User Id=${sqlServer.properties.administratorLogin}@${sqlServer.properties.fullyQualifiedDomainName};Password=${sqlAdminPassword};' + } + { + name: 'Identity:ConnectionString' + value: 'Data Source=tcp:${sqlServer.properties.fullyQualifiedDomainName},1433;Initial Catalog=${sqlDB.name};User Id=${sqlServer.properties.administratorLogin}@${sqlServer.properties.fullyQualifiedDomainName};Password=${sqlAdminPassword};' + } + environmentSettings[environment].appService.RegisterNewUsersDisabled + environmentSettings[environment].appService.AROrgAndRGPagesDisabled + environmentSettings[environment].appService.ApiDisabled + environmentSettings[environment].appService.IshareEnabled + environmentSettings[environment].appService.ClientId + environmentSettings[environment].appService.SatelliteId + environmentSettings[environment].appService.SatelliteUrl + environmentSettings[environment].appService.UseCase + environmentSettings[environment].appService.ManageEntitiesApi + environmentSettings[environment].appService.CanSetPolicyIssuer + environmentSettings[environment].appService.JwtTokenAuthority + environmentSettings[environment].appService.JwtTokenAudience + { + name: 'BrevoApiKey' + value: brevoApiKey + } + ] + phpVersion: 'OFF' + netFrameworkVersion: 'v8.0' + alwaysOn: true + webSocketsEnabled: true + ipSecurityRestrictionsDefaultAction: 'Allow' + ipSecurityRestrictions: [ + { + ipAddress: 'Any' + action: 'Allow' + priority: 2147483647 + name: 'Allow all' + description: 'Allow all access' + } + ] + } + } +} + +module customDomainModule 'customDomainModule.bicep' = if (length(customDomain) > 0) { + name: 'CustomDomainModule' + params: { + location: location + appName: appService.name + customDomain: customDomain + appServicePlanId: appServicePlan.id + } +} + +output appServiceName string = appService.name diff --git a/Poort8.Dataspace.CoreManager/deploy/customDomainModule.bicep b/Poort8.Dataspace.CoreManager/deploy/customDomainModule.bicep new file mode 100644 index 0000000..761ccc9 --- /dev/null +++ b/Poort8.Dataspace.CoreManager/deploy/customDomainModule.bicep @@ -0,0 +1,34 @@ +param appName string +param customDomain string +param appServicePlanId string +param location string + +resource appService 'Microsoft.Web/sites@2022-09-01' existing = { + name: appName +} + +resource appServiceHostNameBinding 'Microsoft.Web/sites/hostNameBindings@2022-03-01' = { + parent: appService + name: customDomain +} + +resource sslCertificate 'Microsoft.Web/certificates@2022-03-01' = { + name: '${customDomain}-${appName}' + location: location + dependsOn: [ + appServiceHostNameBinding + ] + properties: { + serverFarmId: appServicePlanId + canonicalName: customDomain + } +} + +module sslBinding 'sslBindingModule.bicep' = { + name: 'SSLBindingModule' + params: { + appName: appService.name + customDomain: appServiceHostNameBinding.name + certificateThumbprint: sslCertificate.properties.thumbprint + } +} diff --git a/Poort8.Dataspace.CoreManager/deploy/main.bicep b/Poort8.Dataspace.CoreManager/deploy/main.bicep new file mode 100644 index 0000000..4c65a2c --- /dev/null +++ b/Poort8.Dataspace.CoreManager/deploy/main.bicep @@ -0,0 +1,48 @@ +@allowed([ + 'prod' + 'preview' +]) +param environment string = 'preview' + +param location string = resourceGroup().location + +var isProd = environment == 'prod' +var nameAddition = (isProd ? '' : '-${environment}') + +var environmentAppName = 'Poort8-Dataspace-CoreManager${nameAddition}' + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: 'YourVaultName' + scope: resourceGroup('ResourceGroupGuid', 'ResourceGroupName') +} + +module workspace '../../deploy/dataspaceWorkspaceModule.bicep' = { + name: 'WorkspaceModule' + params: { + environment: environment + location: location + appName: environmentAppName + } +} + +module appService 'appServiceModule.bicep' = { + name: 'AppServiceModule' + params: { + environment: environment + location: location + appName: environmentAppName + workspaceId: workspace.outputs.workspaceId + alertEmailAdresses: [ + 'your@email.nl' + ] + errorAlertName: 'Dataspace-CoreManager-Error' + errorAlertShortName: 'DCME${nameAddition}' + criticalAlertName: 'Dataspace-CoreManager-Critical' + criticalAlertShortName: 'DCMC${nameAddition}' + availabilityAlertShortName: 'DCM${nameAddition}' + sqlAdminPassword: keyVault.getSecret('Poort8DataspaceSqlAdminPassword-${environment}') + brevoApiKey: keyVault.getSecret('BrevoApiKey') + } +} + +output appServiceName string = appService.outputs.appServiceName diff --git a/Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep b/Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep new file mode 100644 index 0000000..766cb93 --- /dev/null +++ b/Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep @@ -0,0 +1,9 @@ +targetScope = 'subscription' + +param resourceGroupName string +param resourceGroupLocation string + +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceGroupLocation +} diff --git a/Poort8.Dataspace.CoreManager/deploy/sslBindingModule.bicep b/Poort8.Dataspace.CoreManager/deploy/sslBindingModule.bicep new file mode 100644 index 0000000..0628322 --- /dev/null +++ b/Poort8.Dataspace.CoreManager/deploy/sslBindingModule.bicep @@ -0,0 +1,11 @@ +param appName string +param customDomain string +param certificateThumbprint string + +resource sslBindingEnable 'Microsoft.Web/sites/hostNameBindings@2020-06-01' = { + name: '${appName}/${customDomain}' + properties: { + sslState: 'SniEnabled' + thumbprint: certificateThumbprint + } +} diff --git a/deploy/dataspaceAlertModule.bicep b/deploy/dataspaceAlertModule.bicep new file mode 100644 index 0000000..065c780 --- /dev/null +++ b/deploy/dataspaceAlertModule.bicep @@ -0,0 +1,80 @@ +@allowed([ + 'prod' + 'preview' +]) +param environment string = 'preview' + +param location string = resourceGroup().location + +param alertName string + +@minLength(1) +param emailAdresses array + +@maxLength(12) +param actionGroupShortName string + +@minValue(0) +@maxValue(4) +param severity int + +param evaluationFrequency string + +param workspaceId string + +param query string + +var isProd = environment == 'prod' +var nameAddition = (isProd ? '' : '-${environment}') + +resource actionGroup 'Microsoft.Insights/actionGroups@2023-01-01' = { + name: '${alertName}${nameAddition}-action-group' + location: 'Global' + properties: { + enabled: true + groupShortName: actionGroupShortName + emailReceivers: [for email in emailAdresses: { + name: email + emailAddress: email + useCommonAlertSchema: false + }] + } +} + +resource alert 'Microsoft.Insights/scheduledQueryRules@2022-08-01-preview' = { + name: '${alertName}${nameAddition}' + location: location + properties: { + severity: severity + enabled: true + evaluationFrequency: evaluationFrequency + scopes: [ + workspaceId + ] + targetResourceTypes: [ + 'Microsoft.OperationalInsights/workspaces' + ] + windowSize: evaluationFrequency + criteria: { + allOf: [ + { + query: query + timeAggregation: 'Count' + dimensions: [] + operator: 'GreaterThan' + threshold: 0 + failingPeriods: { + numberOfEvaluationPeriods: 1 + minFailingPeriodsToAlert: 1 + } + } + ] + } + actions: { + actionGroups: [ + actionGroup.id + ] + } + autoMitigate: false + } +} diff --git a/deploy/dataspaceWorkspaceModule.bicep b/deploy/dataspaceWorkspaceModule.bicep new file mode 100644 index 0000000..f1c6761 --- /dev/null +++ b/deploy/dataspaceWorkspaceModule.bicep @@ -0,0 +1,35 @@ +@allowed([ + 'prod' + 'preview' +]) +param environment string = 'preview' + +param location string = resourceGroup().location + +@minLength(3) +param appName string + +var environmentSettings = { + prod: { + retentionInDays: 730 + } + preview: { + retentionInDays: 90 + } +} + +resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: '${appName}-ws' + location: location +} + +resource appTracesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + parent: workspace + name: 'AppTraces' + properties: { + totalRetentionInDays: environmentSettings[environment].retentionInDays + retentionInDays: environmentSettings[environment].retentionInDays + } +} + +output workspaceId string = workspace.id From e23b3d68ef0bdda47731a542058b4392d699ff40 Mon Sep 17 00:00:00 2001 From: Merijn van Aartrijk Date: Wed, 3 Jul 2024 20:23:21 +0200 Subject: [PATCH 2/4] Added first 6 chapters of docs. --- README.md | 2 + {Poort8.Dataspace.Wiki => docs}/APIdocs.md | 0 docs/Chapter 1 - Introduction.md | 78 ++++++++++ docs/Chapter 2 - Dataspace Concepts.md | 121 +++++++++++++++ docs/Chapter 3 - Customer Journeys.md | 51 +++++++ ...ter 4 - NoodleBar Implementation Stages.md | 49 +++++++ docs/Chapter 5 - Tech Stack.md | 67 +++++++++ docs/Chapter 6 - Stage 1 Deployment Guide.md | 138 ++++++++++++++++++ .../Poort8.NoodleBar.postman_collection.json | 0 .../architecture.md | 2 +- .../customerjourneys.md | 0 .../databasemigrations.md | 2 +- 12 files changed, 508 insertions(+), 2 deletions(-) rename {Poort8.Dataspace.Wiki => docs}/APIdocs.md (100%) create mode 100644 docs/Chapter 1 - Introduction.md create mode 100644 docs/Chapter 2 - Dataspace Concepts.md create mode 100644 docs/Chapter 3 - Customer Journeys.md create mode 100644 docs/Chapter 4 - NoodleBar Implementation Stages.md create mode 100644 docs/Chapter 5 - Tech Stack.md create mode 100644 docs/Chapter 6 - Stage 1 Deployment Guide.md rename {Poort8.Dataspace.Wiki => docs}/Poort8.NoodleBar.postman_collection.json (100%) rename {Poort8.Dataspace.Wiki => docs}/architecture.md (99%) rename {Poort8.Dataspace.Wiki => docs}/customerjourneys.md (100%) rename {Poort8.Dataspace.Wiki => docs}/databasemigrations.md (88%) diff --git a/README.md b/README.md index 571681b..460abbd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Poort8 Dataspace NoodleBar +## TODO: local dev getting started + ### Context The project is under the Basis Data Infrastructuur (BDI) umbrella, pending it's ongoing development. diff --git a/Poort8.Dataspace.Wiki/APIdocs.md b/docs/APIdocs.md similarity index 100% rename from Poort8.Dataspace.Wiki/APIdocs.md rename to docs/APIdocs.md diff --git a/docs/Chapter 1 - Introduction.md b/docs/Chapter 1 - Introduction.md new file mode 100644 index 0000000..43a16c2 --- /dev/null +++ b/docs/Chapter 1 - Introduction.md @@ -0,0 +1,78 @@ +# Chapter 1: Introduction + +### 1.1 Dataspaces and Federated Data Sharing + +Dataspaces represent a paradigm shift in how organizations share and manage data. Unlike traditional centralized data storage systems, dataspaces allow data to remain with its original owner while enabling secure, controlled access and sharing. This approach is built on trust frameworks that define agreements on identification, authentication, and authorization to ensure data integrity and security. + +Federated data sharing facilitates collaboration across different organizations by allowing them to access and use data without physically transferring it. In federated data sharing, data can be shared via a service provider, but the owner always remains in control. This method not only enhances data privacy and security but also ensures compliance with various legal and regulatory standards. The key principles of federated data sharing include: + +- **Data Sovereignty**: Data owners retain control over their data at all times. +- **Interoperability**: Seamless integration across different systems and organizations. +- **Scalability**: Ability to scale data sharing across various sectors and regions. +- **Trust and Security**: Robust mechanisms for identification, authentication, and authorization. + +By leveraging these principles, federated data sharing enables more efficient and effective use of data, fostering innovation and collaboration while maintaining strict security and privacy standards. + +### 1.2 Overview of NoodleBar + +NoodleBar, developed by Poort8, is a cutting-edge dataspace solution designed to facilitate secure, controlled, and efficient data sharing among businesses. NoodleBar is known for its "dataspace in a day" capability, providing all the necessary building blocks to get a dataspace up and running quickly. The main building blocks are an organization register and an authorization register, which are the core components of every dataspace. These components can be integrated with other systems to make the dataspace more federated. Inspired by the iSHARE Trust Framework and the principles of federated data sharing, NoodleBar adopts the same dataspace concepts and roles, ensuring seamless integration with iSHARE-compliant systems. NoodleBar offers a comprehensive platform for creating and managing dataspaces, with robust tools for data transformation and integration. These features aim to streamline data workflows, enhance data accessibility, and support informed decision-making. + +### 1.3 About Poort8 + +Poort8 was the first to implement an iSHARE-compliant authorization register and has been pioneering in the field of dataspaces across multiple sectors. We are innovators, creating solutions quickly and efficiently, moving from pilot to production with ease. Poort8 focuses on delivering tangible results—no lengthy PowerPoint presentations, just working pilots. We excel at translating business requirements into functional dataspaces, establishing ourselves as authorities in the field of dataspaces. + +### 1.4 Purpose and Scope + +This documentation aims to provide a comprehensive guide to understanding, deploying, and using NoodleBar. The scope of this documentation includes: + +- Basic concepts and core functionalities of NoodleBar. +- Step-by-step deployment instructions and system requirements. +- User guides for navigating and utilizing the platform. +- Detailed descriptions of advanced features and customization options. +- Administrative and troubleshooting tips to ensure smooth operation. + +### 1.5 Audience + +This documentation is intended for: +- **IT Administrators**: For deployment, management, and maintenance of NoodleBar in their infrastructure. +- **Developers**: For extending and integrating NoodleBar with other systems through APIs and custom plugins. +- **Business Stakeholders**: To understand the benefits and applications of NoodleBar in driving data-driven decision-making within their organizations. + +Prerequisites for the audience include a basic understanding of data management principles and IT infrastructure. + +### 1.6 Integration with iSHARE and Federated Data Sharing + +NoodleBar seamlessly integrates with the iSHARE Trust Framework, leveraging its robust identification, authentication, and authorization mechanisms. This integration ensures that data exchanges are secure, transparent, and controlled, aligning with the highest standards of data governance. By adopting iSHARE's dataspace concepts and roles, NoodleBar provides a familiar and compliant environment for organizations already using or planning to use iSHARE. + +Inspired by the principles of federated data sharing, NoodleBar ensures that data remains at its source while enabling controlled access and sharing across different sectors and organizations. This approach enhances data accessibility and trust, promoting responsible data usage, privacy, and transparency. + +For more information on iSHARE, visit the [iSHARE Trust Framework website](https://framework.ishare.eu/is/). + +### 1.7 Context and Objective + +The project is under the Basis Data Infrastructuur (BDI) umbrella, pending its ongoing development. The objective is to facilitate setting up dataspaces that follow certain principles, serving as an initial platform for data providers, apps, and data consumers. + +### 1.8 Roles + +- **Data Providers**: Organizations that either offer a data source with raw data or an app with processed data. In all cases, access conditions are set by the data owner. +- **App Providers**: Organizations that act as intermediaries, adding value to raw data. They act as a Data Consumer on behalf of their end users, and as a Data Provider for their end users. +- **Data Consumers**: Organizations that use data via Service Providers or directly. +- **Dataspace Initiators**: Organizations that setup and manage the dataspace. + +### 1.9 Principles + +- **Data Sovereignty**: Data owners (issuers) can issue access to their data, even if through federated apps. +- **Data Localization**: Data stays at its source unless caching or staging is essential. +- **Identity Flexibility**: Data consumers choose their identity providers. + +### 1.10 Customer Journeys + +The wiki describes the following Customer Journeys in more detail: + +- **Initiating Dataspace Core** +- **Onboarding Data Sources** +- **Onboarding Data Owners and Consumers** +- **Data Sources Becoming Independent** +- **Adding Providers and Apps** + +The first three journeys comprise the launch of a first (prototype) of a dataspace. Journeys 4 and 5 allow data sources and Service Providers to become independent contributors to the dataspace. diff --git a/docs/Chapter 2 - Dataspace Concepts.md b/docs/Chapter 2 - Dataspace Concepts.md new file mode 100644 index 0000000..b21107f --- /dev/null +++ b/docs/Chapter 2 - Dataspace Concepts.md @@ -0,0 +1,121 @@ +# Chapter 2: Dataspace Concepts + +### 2.1 Dataspace Architecture + +NoodleBar's architecture is designed to facilitate secure and efficient data sharing through the implementation of core dataspace principles. It supports interoperability, scalability, and compliance with the iSHARE Trust Framework. + +#### Key Concepts: + +1. **Data Providers (Sources)**: Data providers connect to the dataspace core for adherence, identification, authentication, and authorization. They supply data to other parties in the dataspace, ensuring it meets the required standards and protocols for secure sharing. + +2. **Data Consumers**: Data consumers also connect to the dataspace core. Using the principles of the dataspace, they can access and retrieve data from the sources. This access is controlled and regulated to ensure data integrity and security. + +3. **Organization Register**: This component acts as a central repository for managing organizational identities within the dataspace. It ensures that all participating entities are properly identified and authenticated. + +4. **Authorization Register**: This register manages access control by defining and enforcing policies for data access. It ensures that only authorized entities can access specific data resources, maintaining data sovereignty and security. + +### 2.2 Building Blocks of NoodleBar + +NoodleBar offers a modular approach to setting up a dataspace, allowing for flexibility and integration with existing systems. The core building blocks include: + +1. **Organization Register**: This component manages organizational identities within the dataspace, ensuring that all participating entities are properly identified and authenticated. + +2. **Authorization Register**: This tool manages access permissions, ensuring that only authorized entities can access specific data resources. + +3. **Dataspace Adapters**: These adapters make it possible for legacy APIs to connect to a dataspace. They ensure that data from non-BDI sources can be integrated into the dataspace by converting them into RESTful APIs and mapping the data to the dataspace schema. + +4. **Dataspace (Prototype) Apps**: Applications that interact with the dataspace to retrieve data, authenticate users, enforce policies, and provide various user interfaces for data interaction. These prototype apps can be customized and extended as needed. + +### 2.3 Deployment Models + +NoodleBar provides different deployment models to suit various organizational needs. The key models are: + +1. **Local Identity Server**: This is the most straightforward deployment, functioning as a standalone system. It is ideal for prototypes and a "dataspace in a day." + +2. **OAuth Identity Server**: This version supports federated capabilities using the OAuth standard. It allows for standalone operation while enabling integration with external systems, providing a flexible and scalable solution. + +3. **iSHARE**: Fully iSHARE-compliant, this deployment ensures the highest level of security and interoperability. It uses X.509 certificates for authentication and iSHARE-compliant authorization mechanisms, making it suitable for organizations with stringent compliance requirements. + +### 2.4 Roles in the iSHARE Trust Framework + +The iSHARE Trust Framework consists of several roles that interact based on specific agreements. These roles include: + +- **Service Consumer**: A legal entity that consumes a service provided by a Service Provider. +- **Service Provider**: A legal entity that provides a service for consumption by a Service Consumer. +- **Entitled Party**: A legal entity that holds the rights to a service provided by a Service Provider. +- **Identity Provider**: Provides identification and authentication services. +- **Identity Broker**: Facilitates connections between different Identity Providers and Service Providers. +- **Authorization Registry**: Manages storage and validation of delegation and authorization information. + +These roles ensure that all parties involved in data sharing are properly authenticated, authorized, and compliant with the iSHARE terms of use. + +### 2.5 Advantages of NoodleBar + +- **Quick Deployment**: With the "dataspace in a day" capability, NoodleBar provides all the essential building blocks to get a dataspace up and running quickly. +- **Modularity**: The components can be customized and integrated with other systems, making it a flexible solution for various organizational needs. +- **Scalability**: NoodleBar is designed to scale, supporting growing data sharing needs as organizations expand. +- **Compliance**: Fully aligned with the iSHARE Trust Framework, NoodleBar ensures compliance with stringent data sharing standards. +- **Versatility**: NoodleBar can be used to quickly set up a dataspace in a day and is also being used in fully compliant federated dataspaces. + +### 2.6 Dataspace Adapters (In Development) + +Dataspace Adapters play a critical role in integrating legacy systems into the dataspace. They ensure that data from non-BDI sources can be converted into a format compatible with the dataspace. + +#### Key Objectives: + +1. **Conversion to RESTful APIs**: Transform legacy data sources into RESTful services. +2. **Data Mapping**: Map raw data to the dataspace schema (e.g., DCSA, P4). +3. **iSHARE Authentication**: Implement authentication mechanisms based on the Organization Register. +4. **Policy Enforcement**: Enforce access control policies through the Authorization Register. +5. **Event and Pub/Sub Support**: Enable real-time data interaction through events and Pub/Sub mechanisms. + +#### Key Components: + +- **API Gateway**: Facilitates the conversion of legacy data into RESTful services. +- **Data Mapper**: Maps data to conform to dataspace standards. +- **Authentication Service**: Supports iSHARE and other authentication methods. +- **Authorization Service**: Enforces policies using the Authorization Register. +- **Event Service**: Manages real-time events and Pub/Sub operations. + +### 2.7 Dataspace Apps (In Development) + +Dataspace Apps provide interfaces and functionalities for interacting with the dataspace. They include tools for data retrieval, user authentication, and policy enforcement. + +#### Key Features: + +1. **Data Retrieval**: Access data from both BDI and non-BDI sources. +2. **User Authentication**: Authenticate users and machines. +3. **Policy Enforcement Point (PEP)**: Enforce authorization policies. +4. **Data Storage/Caching**: Store and cache data locally for efficient access. +5. **Logic Integration**: Add custom logic to data interactions. +6. **Multiple Frontends**: Support various user interfaces like web apps, chat interfaces, and email clients. + +#### Deployment Options: + +- Integrated within monolithic applications. +- As separate backend systems with REST interfaces. + +### 2.8 Administrative Tools (In Development) + +NoodleBar includes administrative tools to manage the dataspace efficiently. + +#### Core Components: + +- **Core Manager**: A web application for managing the Organization Register and Authorization Register. +- **Dashboard**: Provides an overview of data sources, applications, and usage statistics. +- **Self-Service Portals**: Allows organizations to manage their registrations and authorizations independently. + +### 2.9 Disambiguation + +In the developing realm of federated dataspace schemes, different—sometimes ambiguous—terminology is used. If possible, the Poort8 dataspace follows [schema.org](https://schema.org/) for terminologies and data models. For authorizations, Poort8 uses terminology in line with [casbin.org](https://casbin.org/). In the table below, we provide a non-exhaustive overview of how typical terminology is mapped to our Dataspace software. + +| Poort8.Dataspace | equivalent to ... | +|-------------------|-----------------------------------------| +| organisation | party | +| issuer | entitledParty, data owner | +| subject | accessSubject, data consumer, data service consumer | +| provider | service provider, data service provider | +| federated app | application of a service provider | +| product, service | resourceType | +| features | attributes | +| policy | authorization, permission | diff --git a/docs/Chapter 3 - Customer Journeys.md b/docs/Chapter 3 - Customer Journeys.md new file mode 100644 index 0000000..1555800 --- /dev/null +++ b/docs/Chapter 3 - Customer Journeys.md @@ -0,0 +1,51 @@ +# Chapter 3: Customer Journeys + +NoodleBar's implementation follows several customer journeys to ensure a seamless and structured approach to setting up and managing a dataspace. These journeys guide users through the process of initiating a dataspace, onboarding data sources, and enabling data providers and consumers to become independent contributors. + +### 3.1 Initiating Dataspace Core + +The first step in setting up a dataspace is to initiate the Dataspace Core. This involves: + +- **Setting Up the Organization Register**: Establishing a central repository for managing organizational identities. +- **Configuring the Authorization Register**: Defining and enforcing access control policies. +- **Launching the Dataspace Manager**: A web application for managing dataspace standards, including authentication, onboarding requirements, and data models. + +### 3.2 Onboarding Data Sources + +After setting up the core, the next step is to onboard data sources: + +- **Registering Data Providers**: Adding organizations that will provide data. +- **Integrating Data Sources**: Using Dataspace Adapters to convert legacy APIs and ensure data compatibility with the dataspace schema. +- **Defining Access Policies**: Setting up rules and permissions for data access through the Authorization Register. + +### 3.3 Onboarding Data Owners and Consumers + +To fully operationalize the dataspace, onboard data owners and consumers: + +- **Adding Data Owners**: Organizations that control data sources and define access conditions. +- **Registering Data Consumers**: Entities that will access the data provided. +- **Configuring Access Controls**: Ensuring proper authentication and authorization mechanisms are in place. + +### 3.4 Data Sources Becoming Independent + +As the dataspace matures, data sources can become independent: + +- **Developing Independent Solutions**: Data providers can create their own solutions, reducing reliance on Dataspace Adapters. +- **Enhancing Integration**: Continued support for interoperability and compliance with dataspace standards. + +### 3.5 Adding Providers and Apps + +Finally, expand the dataspace by adding more providers and apps: + +- **Onboarding New Service Providers**: Integrating additional data sources and services. +- **Launching New Applications**: Developing and deploying new apps that leverage the dataspace for enhanced functionality and user experience. + +### Customer Journeys Summary + +- **Initiating Dataspace Core**: Set up foundational components and management tools. +- **Onboarding Data Sources**: Register and integrate data providers and sources. +- **Onboarding Data Owners and Consumers**: Add and configure entities for data access and control. +- **Data Sources Becoming Independent**: Enable data providers to develop independent, compliant solutions. +- **Adding Providers and Apps**: Expand the dataspace with new services and applications. + +These customer journeys provide a clear roadmap for setting up and managing a dataspace, ensuring that all participants can collaborate effectively while maintaining data sovereignty and compliance. diff --git a/docs/Chapter 4 - NoodleBar Implementation Stages.md b/docs/Chapter 4 - NoodleBar Implementation Stages.md new file mode 100644 index 0000000..fc8c3f5 --- /dev/null +++ b/docs/Chapter 4 - NoodleBar Implementation Stages.md @@ -0,0 +1,49 @@ +# Chapter 4: NoodleBar Implementation Stages + +Poort8 has been the frontrunner in creating solutions for dataspaces. We believe it is important to have a running prototype as soon as possible, which is why we created NoodleBar. Beginning with the iSHARE framework from the start has proven to be far too complex, and frankly, most use cases don't need this level of security and governance. With NoodleBar, we set out to create a solution that makes it possible to set up a dataspace prototype in a day. This prototype can be integrated with existing data sources or synthetic data to use the dataspace from day one. + +Another hurdle with iSHARE is that it does not use any existing standards. This means there are no packages, middleware, or even code examples available for developers to use. Developers need to first learn this complex iSHARE standard and then implement everything from scratch. To address this, we added the de-facto standard to NoodleBar: OAuth. + +OAuth is an open standard for access delegation, commonly used for token-based authentication and authorization. It allows third-party services to exchange information without exposing user credentials. The added value of OAuth includes: + +- **Ease of Use**: OAuth is widely recognized and supported, making it easier for developers to integrate and use. +- **Security**: OAuth provides a secure way to authorize access without sharing credentials, reducing the risk of credential theft. +- **Interoperability**: OAuth supports integration with various services and platforms, enhancing the flexibility and scalability of the dataspace. + +### Stage 1: Deploy NoodleBar Using the Local Identity Server + +The first stage involves deploying NoodleBar using the Local Identity Server. This setup provides the essential components to get started quickly: + +- **Organization Register**: Manages organizational identities, ensuring proper identification and authentication of all participating entities. +- **Authorization Register**: Manages access control, allowing the definition and enforcement of authorization policies for organizations. +- **NoodleBar Web App**: A user-friendly interface for managing organizations and adding authorization policies. + +Using this setup, authentication and tokens are managed by the Organization Register. Data providers can modify their existing APIs to integrate with the NoodleBar system. This involves using the enforce API of the Authorization Register to ensure that data is only returned if the enforce API returns an allowed response. + +### Stage 2: Deploy NoodleBar Using an OAuth Server + +The second stage leverages the OAuth protocol for enhanced federated capabilities: + +- **OAuth Identity Server**: Supports OAuth-based authentication and authorization, providing a flexible and scalable solution. +- **Organization and Authorization Registers**: Continue to manage identities and access control. + +This stage maintains the same core functionalities as Stage 1 but enhances security and interoperability by using the OAuth standard. This allows for seamless integration with external systems, making it suitable for more complex and distributed environments. + +### Stage 3: Deploy NoodleBar Using iSHARE + +For use cases that demand the highest standards of security and governance, iSHARE can be utilized: + +- **iSHARE Satellite**: Replaces the NoodleBar Organization Register. The iSHARE Satellite, which uses X.509 certificates for robust authentication, must be deployed and maintained by experienced developers. For more information, visit the [iSHARE Satellite GitHub repository](https://github.com/iSHAREScheme/iSHARESatellite). +- **iSHARE Authorization Register**: Ensures compliance with stringent security and governance standards. + +Deploying NoodleBar using iSHARE ensures that all data exchanges are conducted with the highest level of trust and security. The NoodleBar Authorization Register can be configured to be fully iSHARE compliant. However, it's important to note that this stage has proven to be complex, costly, and time-consuming. For production environments, eIDAS certificates must be purchased by every party in the dataspace, further increasing the cost, complexity, and maintainability demands. + +### Advantages of NoodleBar + +- **Quick Deployment**: NoodleBar can be deployed in a day, providing all the essential components for a fully functional pilot. +- **Versatility**: NoodleBar supports various deployment models, from simple local identity management to advanced OAuth and iSHARE compliant setups. +- **Scalability**: Designed to grow with organizational needs, supporting increasing data sharing requirements. +- **Compliance**: Aligns with the iSHARE Trust Framework, ensuring adherence to high security and governance standards. +- **Flexibility**: NoodleBar can be used to quickly set up a dataspace in a day and is also being used in fully compliant federated dataspaces. + +By following these stages, organizations can leverage NoodleBar’s full potential, starting from a basic setup to a highly secure and compliant dataspace solution. diff --git a/docs/Chapter 5 - Tech Stack.md b/docs/Chapter 5 - Tech Stack.md new file mode 100644 index 0000000..4364eaa --- /dev/null +++ b/docs/Chapter 5 - Tech Stack.md @@ -0,0 +1,67 @@ +# Chapter 5: Tech Stack + +NoodleBar is built on a robust and scalable .NET tech stack deployed on Azure. This choice caters to the needs of current partners, stakeholders, and customers, leveraging Azure's extensive capabilities and the versatility of the .NET ecosystem. + +Stage 1 is aimed towards easy deployment and working prototypes. The OAuth variant can be used in production environments. + +### 5.1 Infrastructure as Code with Bicep + +To streamline deployment and configuration, and to minimize errors, NoodleBar uses Bicep for infrastructure as code. Bicep is a domain-specific language (DSL) designed for declaratively deploying Azure resources. It simplifies infrastructure management by providing a more readable and modular approach compared to traditional JSON templates. + +#### Key Benefits of Bicep: + +- **Declarative Syntax**: Bicep's syntax is concise and easy to read, making infrastructure definitions clear and manageable. +- **Modularization**: Supports the creation of reusable modules, enabling consistent and efficient infrastructure deployment. +- **Seamless Azure Integration**: Bicep integrates directly with Azure Resource Manager (ARM), ensuring smooth deployment and management of Azure resources. + +### 5.2 Deployment and Authentication Options + +#### Stage 1: Local Identity Server + +**Requirements**: +- NoodleBar fork on GitHub +- Azure Subscription + +Stage 1 involves deploying NoodleBar using the Local Identity Server. This setup provides the essential components to get started quickly, utilizing ASP.NET Core Identity to handle authentication and issue bearer tokens. These tokens are not OAuth compatible, but they provide a straightforward and secure method for managing user authentication. The self-hosted variant of Stage 1 is designed for ease of deployment, requiring no prerequisites other than forking the code, building it, and deploying it to Azure. This variant supports two database options: SQLite and SQL Server, with additional databases planned based on demand. + +#### Stage 2: OAuth Server Integration + +**Requirements**: +- OAuth server, for example, Auth0 + +In Stage 2, an OAuth-compatible authorization server is required. Examples include Keycloak, IdentityServer, or any Identity-as-a-Service (IaaS) solution. This guide will explain how Auth0 can be configured to act as the dataspace authentication server. NoodleBar can integrate with these services using standard middleware, which simplifies the setup and enhances security and interoperability. + +#### Stage 3: iSHARE Compliance + +**Requirements**: +- iSHARE Satellite +- eIDAS Certificates +- iSHARE Conformance Test + +For use cases that demand the highest standards of security and governance, iSHARE can be utilized. This stage involves replacing the NoodleBar Organization Register with the iSHARE Satellite, which uses X.509 certificates for robust authentication. Deploying NoodleBar using iSHARE ensures that all data exchanges are conducted with the highest level of trust and security. The NoodleBar Authorization Register can be configured to be fully iSHARE compliant. However, it's important to note that this stage has proven to be complex, costly, and time-consuming. For production environments, eIDAS certificates must be purchased by each party in the dataspace, ensuring compliance with stringent security and governance standards. + +### 5.3 Technologies and Tools Used + +**Build Using**: +- ASP.NET +- Blazor +- Entity Framework +- Microsoft Fluent UI Blazor components +- OpenTelemetry +- Casbin.NET +- FastEndpoints +- Poort8.Ishare.Core +- xUnit +- FluentAssertions +- Snapshooter + +**Deployed On**: +- Azure App Service +- Azure SQL Server +- Application Insights + +### 5.4 Future Deployment Options + +Deployment using Docker containers or .NET Aspire to target other cloud providers is also on our roadmap. This will provide flexibility and enable NoodleBar to be deployed in diverse cloud environments, catering to a broader range of organizational needs. + +By leveraging these technologies and deployment strategies, NoodleBar ensures a flexible, secure, and scalable solution for creating and managing dataspaces. diff --git a/docs/Chapter 6 - Stage 1 Deployment Guide.md b/docs/Chapter 6 - Stage 1 Deployment Guide.md new file mode 100644 index 0000000..2cb00e9 --- /dev/null +++ b/docs/Chapter 6 - Stage 1 Deployment Guide.md @@ -0,0 +1,138 @@ +# Chapter 6: Stage 1 Deployment Guide + +The Stage 1 deployment of NoodleBar involves setting up the core components necessary for a functional dataspace prototype. This stage focuses on deploying the Local Identity Server, which handles authentication and authorization, and the NoodleBar Web App for managing organizations and policies. + +### 6.1 Prerequisites + +Before you begin the deployment, ensure that you have the following prerequisites in place: + +- **NoodleBar fork on GitHub**: Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account. +- **Azure Subscription**: Ensure you have an active Azure subscription to deploy the necessary resources. + +### 6.2 Setting Up the Environment + +1. **Fork the NoodleBar Repository**: + Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account: + ```sh + git clone https://github.com/YourAccount/Poort8.Dataspace.NoodleBar.git + cd Poort8.Dataspace.NoodleBar + ``` + +2. **Modify Bicep Files for Deployment**: + The repository includes Bicep files for deployment. Modify these files where applicable to fit your specific environment and requirements. The main Bicep files are: + - `main.bicep`: This is the main entry point for the deployment and references other Bicep modules. + - `dataspaceAlertModule.bicep`: Deploys alert rules for monitoring the resources. + - `dataspaceWorkspaceModule.bicep`: Sets up the Azure Log Analytics workspace. + - `customDomainModule.bicep`: Configures custom domain settings. + - `resourceGroupModule.bicep`: Creates the resource group for all resources. + - `sslBindingModule.bicep`: Sets up SSL bindings for secure connections. + - `appServiceModule.bicep`: Deploys the Azure App Service to host the NoodleBar application. + +3. **Set Configuration and Secrets**: + Configure environment variables and secrets needed for the deployment. Secrets must be configured in the GitHub repository settings under "Settings" -> "Secrets and variables" -> "Actions". Add the following secrets: + - `AZURE_CLIENT_ID` + - `AZURE_TENANT_ID` + - `AZURE_SUBSCRIPTION_ID` + - Any other secrets required for your specific setup, such as database connection strings. + +4. **Set Up GitHub Workflow to Deploy NoodleBar**: + Configure a GitHub workflow to automate the deployment of NoodleBar. Ensure that the workflow uses the modified Bicep files and secrets for deployment. + + Here's an example of a GitHub Actions workflow file (`.github/workflows/deploy.yml`): + + ```yaml + name: Deploy NoodleBar + + on: + workflow_dispatch: + push: + branches: + - master + paths: + - 'Poort8.Dataspace.CoreManager/**' + - 'Poort8.Dataspace.AuthorizationRegistry/**' + - 'Poort8.Dataspace.OrganizationRegistry/**' + - 'Poort8.Dataspace.API/**' + + permissions: + id-token: write + contents: read + + jobs: + deploy-preview: + runs-on: ubuntu-latest + env: + resourceGroupName: 'NoodleBar-Preview' + resourceGroupLocation: 'westeurope' + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + allow-no-subscriptions: true + + - name: Publish + run: dotnet publish ./Poort8.Dataspace.CoreManager/Poort8.Dataspace.CoreManager.csproj -c Release -o publish/app + + - name: Deploy resource group + uses: azure/arm-deploy@v2 + with: + scope: subscription + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + region: ${{ env.resourceGroupLocation }} + template: ./Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep + parameters: 'resourceGroupName=${{ env.resourceGroupName }} resourceGroupLocation=${{ env.resourceGroupLocation }}' + + - name: Deploy preview bicep + uses: azure/arm-deploy@v2 + with: + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + resourceGroupName: ${{ env.resourceGroupName }} + template: ./Poort8.Dataspace.CoreManager/deploy/main.bicep + parameters: 'environment=preview' + + - name: Deploy app to preview + uses: azure/webapps-deploy@v3 + with: + app-name: NoodleBar-Preview + package: publish/app + ``` + +### 6.3 Deploying the NoodleBar Application + +All components of NoodleBar, including the Organization Register, Authorization Register, and the NoodleBar Web App, are deployed as a single application. Use the GitHub Actions workflow to deploy these components using the modified Bicep files. + +1. **Run the Deployment Workflow**: + Trigger the GitHub Actions workflow to deploy the NoodleBar application. This workflow will use the Azure CLI and ARM templates to set up the necessary resources and deploy the application. + +2. **Verify the Deployment**: + Ensure the deployment is successful by accessing the service endpoint: + ```sh + curl http:///health + ``` + +### 6.4 Configuring the NoodleBar System + +1. **Register as a New User**: + Go to `http://` to register as a new user. + +2. **Add Organizations**: + Use the NoodleBar web app to add new organizations to the Organization Register. + +3. **Define Authorization Policies**: + Set up authorization policies for the organizations using the Authorization Register. + +4. **Integrate Data Providers**: + Modify existing APIs to use the enforce API of the Authorization Register to control data access. + +By following these steps, you can successfully deploy NoodleBar using the Local Identity Server, setting up a fully functional dataspace prototype quickly and efficiently. \ No newline at end of file diff --git a/Poort8.Dataspace.Wiki/Poort8.NoodleBar.postman_collection.json b/docs/Poort8.NoodleBar.postman_collection.json similarity index 100% rename from Poort8.Dataspace.Wiki/Poort8.NoodleBar.postman_collection.json rename to docs/Poort8.NoodleBar.postman_collection.json diff --git a/Poort8.Dataspace.Wiki/architecture.md b/docs/architecture.md similarity index 99% rename from Poort8.Dataspace.Wiki/architecture.md rename to docs/architecture.md index c95f0c9..0efcf40 100644 --- a/Poort8.Dataspace.Wiki/architecture.md +++ b/docs/architecture.md @@ -139,7 +139,7 @@ Onboarding-->Prototype 2. **Authorization Register (AuthRegister)** - Entails: - Permissions - - Resource groups and resources + - Products and features - Organizations and users - Requirements: - iSHARE endpoints diff --git a/Poort8.Dataspace.Wiki/customerjourneys.md b/docs/customerjourneys.md similarity index 100% rename from Poort8.Dataspace.Wiki/customerjourneys.md rename to docs/customerjourneys.md diff --git a/Poort8.Dataspace.Wiki/databasemigrations.md b/docs/databasemigrations.md similarity index 88% rename from Poort8.Dataspace.Wiki/databasemigrations.md rename to docs/databasemigrations.md index e3a8d6b..5e83d21 100644 --- a/Poort8.Dataspace.Wiki/databasemigrations.md +++ b/docs/databasemigrations.md @@ -1,4 +1,4 @@ -1. Copy the migrations folder from Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations to Poort8.Dataspace.CoreManager, depending on whether you changed Organization registry or Authorization registry data models. +1. Copy the migrations folder from Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations, depending on whether you changed Organization registry or Authorization registry data models. 2. In the AddOrganizationRegistrySqlite method in the Poort8.Dataspace.(_Organization/Authorization_)Registry.Extensions.DefaultExtension class, change the MigrationsAssembly from 'Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations' to 'Poort8.Dataspace.CoreManager'. This points the startup process to migration files of step 1. 3. Execute entity framework command to generate the migration script: * Package Manager Console: From 7eab4ff57f7e2babd22b594b114ff40640e5ed9d Mon Sep 17 00:00:00 2001 From: Merijn van Aartrijk Date: Thu, 4 Jul 2024 14:46:41 +0200 Subject: [PATCH 3/4] Review fixes. --- Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep | 4 ++-- Poort8.Dataspace.CoreManager/deploy/main.bicep | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep b/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep index 8a0671e..0baedf3 100644 --- a/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep +++ b/Poort8.Dataspace.CoreManager/deploy/appServiceModule.bicep @@ -53,7 +53,7 @@ var environmentSettings = { } JwtTokenAuthority: { name: 'CoreManagerOptions:JwtTokenAuthority' - value: 'https://poort8.eu.auth0.com/' + value: 'https://xxx.auth0.com/' } JwtTokenAudience: { name: 'CoreManagerOptions:JwtTokenAudience' @@ -233,7 +233,7 @@ resource sqlServer 'Microsoft.Sql/servers@2023-05-01-preview' = { type: 'SystemAssigned' } properties: { - administratorLogin: 'Poort8' + administratorLogin: 'YourUserName' administratorLoginPassword: sqlAdminPassword minimalTlsVersion: '1.2' } diff --git a/Poort8.Dataspace.CoreManager/deploy/main.bicep b/Poort8.Dataspace.CoreManager/deploy/main.bicep index 4c65a2c..06a61d5 100644 --- a/Poort8.Dataspace.CoreManager/deploy/main.bicep +++ b/Poort8.Dataspace.CoreManager/deploy/main.bicep @@ -13,7 +13,7 @@ var environmentAppName = 'Poort8-Dataspace-CoreManager${nameAddition}' resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { name: 'YourVaultName' - scope: resourceGroup('ResourceGroupGuid', 'ResourceGroupName') + scope: resourceGroup('SubscriptionId', 'ResourceGroupName') } module workspace '../../deploy/dataspaceWorkspaceModule.bicep' = { @@ -40,7 +40,7 @@ module appService 'appServiceModule.bicep' = { criticalAlertName: 'Dataspace-CoreManager-Critical' criticalAlertShortName: 'DCMC${nameAddition}' availabilityAlertShortName: 'DCM${nameAddition}' - sqlAdminPassword: keyVault.getSecret('Poort8DataspaceSqlAdminPassword-${environment}') + sqlAdminPassword: keyVault.getSecret('DataspaceSqlAdminPassword-${environment}') brevoApiKey: keyVault.getSecret('BrevoApiKey') } } From 8e745743228bb5872c317974d7f4125e0a7f42ca Mon Sep 17 00:00:00 2001 From: Merijn van Aartrijk Date: Thu, 4 Jul 2024 14:48:05 +0200 Subject: [PATCH 4/4] Docs working version. --- README.md | 90 ++++++---- ... - Introduction.md => 1 - Introduction.md} | 2 +- ... Concepts.md => 2 - Dataspace Concepts.md} | 77 +++++++- ...r Journeys.md => 3 - Customer Journeys.md} | 2 +- ...=> 4 - NoodleBar Implementation Stages.md} | 14 +- ...er 5 - Tech Stack.md => 5 - Tech Stack.md} | 2 +- ...ployment Using a Local Identity Server.md} | 2 +- docs/7 - Deployment Using OAuth Server.md | 162 +++++++++++++++++ docs/8 - Deployment Using iSHARE.md | 164 +++++++++++++++++ ...grations.md => 9 - Database Migrations.md} | 6 +- docs/architecture.md | 169 ------------------ docs/customerjourneys.md | 86 --------- 12 files changed, 471 insertions(+), 305 deletions(-) rename docs/{Chapter 1 - Introduction.md => 1 - Introduction.md} (99%) rename docs/{Chapter 2 - Dataspace Concepts.md => 2 - Dataspace Concepts.md} (79%) rename docs/{Chapter 3 - Customer Journeys.md => 3 - Customer Journeys.md} (98%) rename docs/{Chapter 4 - NoodleBar Implementation Stages.md => 4 - NoodleBar Implementation Stages.md} (86%) rename docs/{Chapter 5 - Tech Stack.md => 5 - Tech Stack.md} (99%) rename docs/{Chapter 6 - Stage 1 Deployment Guide.md => 6 - Deployment Using a Local Identity Server.md} (99%) create mode 100644 docs/7 - Deployment Using OAuth Server.md create mode 100644 docs/8 - Deployment Using iSHARE.md rename docs/{databasemigrations.md => 9 - Database Migrations.md} (82%) delete mode 100644 docs/architecture.md delete mode 100644 docs/customerjourneys.md diff --git a/README.md b/README.md index 460abbd..785d5a4 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,85 @@ # Poort8 Dataspace NoodleBar -## TODO: local dev getting started +## Overview -### Context +NoodleBar, developed by Poort8, is a cutting-edge dataspace solution designed to facilitate secure, controlled, and efficient data sharing among businesses. Known for its "dataspace in a day" capability, NoodleBar provides all the necessary building blocks to get a dataspace up and running quickly. Inspired by the iSHARE Trust Framework, NoodleBar adopts the same dataspace concepts and roles, ensuring seamless integration with iSHARE-compliant systems. -The project is under the Basis Data Infrastructuur (BDI) umbrella, pending it's ongoing development. +## Table of Contents -### Objective +1. [Introduction](docs/1-Introduction.md) +2. [Dataspace Concepts](docs/2-Dataspace-Concepts.md) +3. [Customer Journeys](docs/3-Customer-Journeys.md) +4. [NoodleBar Implementation Stages](docs/4-NoodleBar-Implementation-Stages.md) +5. [Tech Stack](docs/5-Tech-Stack.md) +6. [Deployment Using a Local Identity Server](docs/6-Deployment-Using-a-Local-Identity-Server.md) +7. [Deployment Using OAuth Server](docs/7-Deployment-Using-OAuth-Server.md) +8. [Deployment Using iSHARE](docs/8-Deployment-Using-iSHARE.md) +9. [Database Migrations](docs/9-Database-Migrations.md) + +## Context + +The project is under the Basis Data Infrastructuur (BDI) umbrella, pending its ongoing development. + +## Objective To facilitate setting up dataspaces that follow certain principles, serving as an initial platform for data providers, apps, and data consumers. -### Roles +## Roles -- Data Providers: Organizations that either offer a data source with raw data or an app with processed data. In all cases, access conditions are set by the data owner. -- App Providers: Organizations that act as intermediaries, adding value to raw data. They act as a Data Consumer on behalf of their end users, and as a Data Provider for their end users. -- Data Consumers: Organizations that use data via Service Providers or directly. -- Dataspace initiators: Organizations that setup and manage the dataspace. +- **Data Providers**: Organizations that either offer a data source with raw data or an app with processed data. In all cases, access conditions are set by the data owner. +- **App Providers**: Organizations that act as intermediaries, adding value to raw data. They act as a Data Consumer on behalf of their end users, and as a Data Provider for their end users. +- **Data Consumers**: Organizations that use data via Service Providers or directly. +- **Dataspace Initiators**: Organizations that setup and manage the dataspace. -### Principles +## Principles 1. Data owners (issuers) can issue access to their data, even if through federated apps. 2. Data stays at its source unless caching or staging is essential. 3. Data consumers choose their identity providers. -### Customer Journeys +## Customer Journeys -The wiki describes the following [Customer Journeys](/Poort8.Dataspace.Wiki/customerjourneys.md) in more detail: +The wiki describes the following [Customer Journeys](docs/3-Customer-Journeys.md) in more detail: - Initiating Dataspace Core - Onboarding Data Sources - Onboarding Data Owners and Consumers - Data Sources Becoming Independent - Adding providers and apps - + The first 3 journeys comprise the launch of a first (prototype) of a dataspace. Journeys 4 and 5 allow data sources and Service Providers to become independent contributors to the dataspace. -### Functionalities +## Functionalities The Dataspace Core provides services for the organization registry, the organization onboarding process, the dataspace manager and (optionally) an authorization registry. With the dataspace manager the dataspace standards can be managed, such as the requirements for authentication and onboarding, and the definition of a dataspace data model. Secondly (and optionally), the dataspace initiator can provide Dataspace Adapters to Data Providers, with services to support them with mapping to the dataspace data model, and Identification, Authentication and Authorisation (IAA) according to the Dataspace standards. Dataspace Adapters are expected to be made redundant as Data Providers create independent solutions for this. Thirdly (and optionally), the dataspace initiator may choose to launch the dataspace with a prototype app, using the Dataspace Prototype services for logic, IAA, and multiple front-end channels for the end user. Such a Dataspace Prototype app can be removed when additional apps are added to the dataspace. -See the [architectural](/Poort8.Dataspace.Wiki/architecture.md) outline of these functions for more detail. +See the [architectural outline](docs/2-Dataspace-Concepts.md) of these functions for more detail. -### Challenges +## Challenges To make the software modular and customizable so as to allow stakeholders to evolve independently but in compliance with the principles and regulations of the dataspace. -### Tech Stack +## Tech Stack + +For the current partners, stakeholders, and customers, the .NET tech stack deployed on Azure was chosen for NoodleBar. -- No strong preferences for technologies. -- Independent data sources must align with the dataspace data models and Identification, Authentication and Authorisation requirements. +- **Built With**: + - .NET 8 + - Casbin.NET + - xUnit + - Entity Framework + - Blazor + - Microsoft Fluent UI Blazor components + - OpenTelemetry + - FastEndpoints + - Poort8.Ishare.Core + - Snapshooter + +- **Deployed On**: + - Azure App Service + - SQL Server + - Application Insights ### Support for New Apps @@ -57,7 +87,7 @@ New apps can utilize the dataspace data model, authorization registry, and organ ### Disambiguation -In the developing realm of federated dataspace schemes, different - sometimes ambiguous - terminology is used. If possible, the Poort8 dataspace follows [schema.org](https://schema.org) for terminologie and data models. For authorizations, Poort8 uses terminologie in line with [casbin.org](https://casbin.org). In the table below we provide a non-exhaustive overview of how typical terminologie is mapped to our Dataspace software. +In the developing realm of federated dataspace schemes, different - sometimes ambiguous - terminology is used. If possible, the Poort8 dataspace follows [schema.org](https://schema.org) for terminology and data models. For authorizations, Poort8 uses terminology in line with [casbin.org](https://casbin.org). In the table below we provide a non-exhaustive overview of how typical terminology is mapped to our Dataspace software. | Poort8.Dataspace | equivalent to ... | | ----------------------- | --------------------------------------------------- | @@ -66,33 +96,19 @@ In the developing realm of federated dataspace schemes, different - sometimes am | subject | accessSubject, data consumer, data service consumer | | provider | service provider, data service provider | | federated app | application of a service provider | -| resource group,service | resourceType | +| resource group, service | resourceType | | resources | attributes | | policy | authorization, permission | ## Getting Started -### Built With - -- .NET 8 - The core framework used -- Casbin.NET - For authorization -- xUnit - For testing -- Entity Framework - For ORM (Sqlite and SQL Server supported for now) - ### Prerequisites -Install .NET8. +Install .NET 8. ### Installation 1. Clone the repo: ```sh git clone https://github.com/POORT8/Poort8.Dataspace.NoodleBar.git - ``` -2. Build and start Poort8.Dataspace.CoreManager from you favourite IDE or CLI. The CoreManager web app including an instance of Poort8.Dataspace.AuthorizationRegistry and Poort8.Dataspace.OrganizationRegistry will be started. - -### Running the Tests - -Run the tests in: -- Poort8.Dataspace.AuthorizationRegistry.Tests -- Poort8.Dataspace.OrganizationRegistry.Tests +2. Build and start Poort8.Dataspace.CoreManager from your favourite IDE or CLI. The CoreManager web app including an instance of Poort8.Dataspace.AuthorizationRegistry and Poort8.Dataspace.OrganizationRegistry will be started. \ No newline at end of file diff --git a/docs/Chapter 1 - Introduction.md b/docs/1 - Introduction.md similarity index 99% rename from docs/Chapter 1 - Introduction.md rename to docs/1 - Introduction.md index 43a16c2..a5b18ee 100644 --- a/docs/Chapter 1 - Introduction.md +++ b/docs/1 - Introduction.md @@ -1,4 +1,4 @@ -# Chapter 1: Introduction +# 1: Introduction ### 1.1 Dataspaces and Federated Data Sharing diff --git a/docs/Chapter 2 - Dataspace Concepts.md b/docs/2 - Dataspace Concepts.md similarity index 79% rename from docs/Chapter 2 - Dataspace Concepts.md rename to docs/2 - Dataspace Concepts.md index b21107f..573d1b4 100644 --- a/docs/Chapter 2 - Dataspace Concepts.md +++ b/docs/2 - Dataspace Concepts.md @@ -1,4 +1,4 @@ -# Chapter 2: Dataspace Concepts +# 2: Dataspace Concepts ### 2.1 Dataspace Architecture @@ -119,3 +119,78 @@ In the developing realm of federated dataspace schemes, different—sometimes am | product, service | resourceType | | features | attributes | | policy | authorization, permission | + +### 2.10 Architectural Overview + +```mermaid +graph TD + +%% Dataspace Core +style Core fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; +subgraph Core[Dataspace Core] +subgraph Admin[Administration] + CoreManager[Core Manager] + organizationRegister[Organization Register] +end +AuthRegister[Authorization Register] +end + +%% BDI Source +style BDIServiceExt fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; +subgraph BDIServiceExt[BDI Source] + subgraph Dataspace_Adapter[Dataspace Adapter for non-BDI sources] + DataFetcher[Data fetcher] + DataMapper[Data Mapper] + AuthService[Authentication Service] + AuthzService[Authorization Service] + end + BDIServiceEndpoint[BDI Service] +end + +%% Dataspace Apps +style Dataspace_Apps fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; +subgraph Dataspace_Apps[Dataspace Apps] + subgraph Onboarding[Dataspace Onboarding App] + RegisteringUsers[Registering Users] + AuthUsers2[Authenticate Users] + AuthMan[Registering Authorizations] + end + subgraph Prototype[Dataspace Prototype App] + AuthUsers[Authenticate Users] + PEP[Authorize Users] + RESTSrc[Retrieve Data from BDI-services] + DataStore[Data Storage/Caching] + Logic[Add Logic] + FrontEnds[Multiple Frontends] + end +end + +%% Connections within Dataspace Adapter +DataFetcher-->DataMapper +DataMapper-->AuthService +AuthService-->AuthzService + +%% Connections within Dataspace Core +CoreManager-->organizationRegister +CoreManager-->AuthRegister + +%% Dependencies +RegisteringUsers-->|Registers in|organizationRegister +AuthRegister-->|Checked By|PEP +AuthService-->|Checks|organizationRegister +organizationRegister-->|Checked By|AuthUsers +AuthzService-->BDIServiceEndpoint +BDIServiceEndpoint-->RESTSrc +AuthMan-->|Registers in|AuthRegister +AuthzService-->|Checks|AuthRegister + +%% Connections within Dataspace Apps +AuthUsers2-->RegisteringUsers +RegisteringUsers-->AuthMan +RESTSrc-->DataStore +AuthUsers-->PEP +PEP-->RESTSrc +DataStore-->Logic +Logic-->FrontEnds +Onboarding-->Prototype +``` \ No newline at end of file diff --git a/docs/Chapter 3 - Customer Journeys.md b/docs/3 - Customer Journeys.md similarity index 98% rename from docs/Chapter 3 - Customer Journeys.md rename to docs/3 - Customer Journeys.md index 1555800..cb58a2c 100644 --- a/docs/Chapter 3 - Customer Journeys.md +++ b/docs/3 - Customer Journeys.md @@ -1,4 +1,4 @@ -# Chapter 3: Customer Journeys +# 3: Customer Journeys NoodleBar's implementation follows several customer journeys to ensure a seamless and structured approach to setting up and managing a dataspace. These journeys guide users through the process of initiating a dataspace, onboarding data sources, and enabling data providers and consumers to become independent contributors. diff --git a/docs/Chapter 4 - NoodleBar Implementation Stages.md b/docs/4 - NoodleBar Implementation Stages.md similarity index 86% rename from docs/Chapter 4 - NoodleBar Implementation Stages.md rename to docs/4 - NoodleBar Implementation Stages.md index fc8c3f5..711cd55 100644 --- a/docs/Chapter 4 - NoodleBar Implementation Stages.md +++ b/docs/4 - NoodleBar Implementation Stages.md @@ -1,4 +1,4 @@ -# Chapter 4: NoodleBar Implementation Stages +# 4: NoodleBar Implementation Options Poort8 has been the frontrunner in creating solutions for dataspaces. We believe it is important to have a running prototype as soon as possible, which is why we created NoodleBar. Beginning with the iSHARE framework from the start has proven to be far too complex, and frankly, most use cases don't need this level of security and governance. With NoodleBar, we set out to create a solution that makes it possible to set up a dataspace prototype in a day. This prototype can be integrated with existing data sources or synthetic data to use the dataspace from day one. @@ -10,9 +10,9 @@ OAuth is an open standard for access delegation, commonly used for token-based a - **Security**: OAuth provides a secure way to authorize access without sharing credentials, reducing the risk of credential theft. - **Interoperability**: OAuth supports integration with various services and platforms, enhancing the flexibility and scalability of the dataspace. -### Stage 1: Deploy NoodleBar Using the Local Identity Server +### Option 1: Deploy NoodleBar Using the Local Identity Server -The first stage involves deploying NoodleBar using the Local Identity Server. This setup provides the essential components to get started quickly: +The first option involves deploying NoodleBar using the Local Identity Server. This setup provides the essential components to get started quickly: - **Organization Register**: Manages organizational identities, ensuring proper identification and authentication of all participating entities. - **Authorization Register**: Manages access control, allowing the definition and enforcement of authorization policies for organizations. @@ -20,16 +20,16 @@ The first stage involves deploying NoodleBar using the Local Identity Server. Th Using this setup, authentication and tokens are managed by the Organization Register. Data providers can modify their existing APIs to integrate with the NoodleBar system. This involves using the enforce API of the Authorization Register to ensure that data is only returned if the enforce API returns an allowed response. -### Stage 2: Deploy NoodleBar Using an OAuth Server +### Option 2: Deploy NoodleBar Using an OAuth Server -The second stage leverages the OAuth protocol for enhanced federated capabilities: +This option leverages the OAuth protocol for enhanced federated capabilities: - **OAuth Identity Server**: Supports OAuth-based authentication and authorization, providing a flexible and scalable solution. - **Organization and Authorization Registers**: Continue to manage identities and access control. -This stage maintains the same core functionalities as Stage 1 but enhances security and interoperability by using the OAuth standard. This allows for seamless integration with external systems, making it suitable for more complex and distributed environments. +This stage maintains the same core functionalities as option 1 but enhances security and interoperability by using the OAuth standard. This allows for seamless integration with external systems, making it suitable for more complex and distributed environments. -### Stage 3: Deploy NoodleBar Using iSHARE +### Option 3: Deploy NoodleBar Using iSHARE For use cases that demand the highest standards of security and governance, iSHARE can be utilized: diff --git a/docs/Chapter 5 - Tech Stack.md b/docs/5 - Tech Stack.md similarity index 99% rename from docs/Chapter 5 - Tech Stack.md rename to docs/5 - Tech Stack.md index 4364eaa..eafc577 100644 --- a/docs/Chapter 5 - Tech Stack.md +++ b/docs/5 - Tech Stack.md @@ -1,4 +1,4 @@ -# Chapter 5: Tech Stack +# 5: Tech Stack NoodleBar is built on a robust and scalable .NET tech stack deployed on Azure. This choice caters to the needs of current partners, stakeholders, and customers, leveraging Azure's extensive capabilities and the versatility of the .NET ecosystem. diff --git a/docs/Chapter 6 - Stage 1 Deployment Guide.md b/docs/6 - Deployment Using a Local Identity Server.md similarity index 99% rename from docs/Chapter 6 - Stage 1 Deployment Guide.md rename to docs/6 - Deployment Using a Local Identity Server.md index 2cb00e9..730a81b 100644 --- a/docs/Chapter 6 - Stage 1 Deployment Guide.md +++ b/docs/6 - Deployment Using a Local Identity Server.md @@ -1,4 +1,4 @@ -# Chapter 6: Stage 1 Deployment Guide +# 6: Deployment Using a Local Identity Server The Stage 1 deployment of NoodleBar involves setting up the core components necessary for a functional dataspace prototype. This stage focuses on deploying the Local Identity Server, which handles authentication and authorization, and the NoodleBar Web App for managing organizations and policies. diff --git a/docs/7 - Deployment Using OAuth Server.md b/docs/7 - Deployment Using OAuth Server.md new file mode 100644 index 0000000..bc8b498 --- /dev/null +++ b/docs/7 - Deployment Using OAuth Server.md @@ -0,0 +1,162 @@ +# 7: Deployment Using OAuth Server + +The Stage 2 deployment of NoodleBar involves setting up the core components necessary for a functional dataspace prototype using an OAuth-compatible authorization server. This stage enhances security and interoperability by using the OAuth standard, allowing for seamless integration with external systems. + +### 7.1 Prerequisites + +Before you begin the deployment, ensure that you have the following prerequisites in place: + +- **NoodleBar fork on GitHub**: Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account. +- **Azure Subscription**: Ensure you have an active Azure subscription to deploy the necessary resources. +- **OAuth Server**: Set up an OAuth-compatible authorization server (e.g., Auth0, Keycloak, IdentityServer). + +### 7.2 Setting Up the Environment + +1. **Fork the NoodleBar Repository**: + Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account: + ```sh + git clone https://github.com/YourAccount/Poort8.Dataspace.NoodleBar.git + cd Poort8.Dataspace.NoodleBar + ``` + +2. **Modify Bicep Files for Deployment**: + The repository includes Bicep files for deployment. Modify these files where applicable to fit your specific environment and requirements. The main Bicep files are: + - `main.bicep`: This is the main entry point for the deployment and references other Bicep modules. + - `dataspaceAlertModule.bicep`: Deploys alert rules for monitoring the resources. + - `dataspaceWorkspaceModule.bicep`: Sets up the Azure Log Analytics workspace. + - `customDomainModule.bicep`: Configures custom domain settings. + - `resourceGroupModule.bicep`: Creates the resource group for all resources. + - `sslBindingModule.bicep`: Sets up SSL bindings for secure connections. + - `appServiceModule.bicep`: Deploys the Azure App Service to host the NoodleBar application. + +3. **Set Configuration and Secrets**: + Configure environment variables and secrets needed for the deployment. Secrets must be configured in the GitHub repository settings under "Settings" -> "Secrets and variables" -> "Actions". Add the following secrets: + - `AZURE_CLIENT_ID` + - `AZURE_TENANT_ID` + - `AZURE_SUBSCRIPTION_ID` + - `OAUTH_CLIENT_ID` + - `OAUTH_CLIENT_SECRET` + - `OAUTH_AUTHORITY` + - Any other secrets required for your specific setup, such as database connection strings. + +4. **Set Up GitHub Workflow to Deploy NoodleBar**: + Configure a GitHub workflow to automate the deployment of NoodleBar. Ensure that the workflow uses the modified Bicep files and secrets for deployment. + + Here's an example of a GitHub Actions workflow file (`.github/workflows/deploy.yml`): + + ```yaml + name: Deploy NoodleBar + + on: + workflow_dispatch: + push: + branches: + - master + paths: + - 'Poort8.Dataspace.CoreManager/**' + - 'Poort8.Dataspace.AuthorizationRegistry/**' + - 'Poort8.Dataspace.OrganizationRegistry/**' + - 'Poort8.Dataspace.API/**' + + permissions: + id-token: write + contents: read + + jobs: + deploy-preview: + runs-on: ubuntu-latest + env: + resourceGroupName: 'NoodleBar-Preview' + resourceGroupLocation: 'westeurope' + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + allow-no-subscriptions: true + + - name: Publish + run: dotnet publish ./Poort8.Dataspace.CoreManager/Poort8.Dataspace.CoreManager.csproj -c Release -o publish/app + + - name: Deploy resource group + uses: azure/arm-deploy@v2 + with: + scope: subscription + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + region: ${{ env.resourceGroupLocation }} + template: ./Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep + parameters: 'resourceGroupName=${{ env.resourceGroupName }} resourceGroupLocation=${{ env.resourceGroupLocation }}' + + - name: Deploy preview bicep + uses: azure/arm-deploy@v2 + with: + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + resourceGroupName: ${{ env.resourceGroupName }} + template: ./Poort8.Dataspace.CoreManager/deploy/main.bicep + parameters: 'environment=preview' + + - name: Deploy app to preview + uses: azure/webapps-deploy@v3 + with: + app-name: NoodleBar-Preview + package: publish/app + ``` + +### 7.3 Configuring the OAuth Server + +1. **Register a New Application**: + Register a new application in your OAuth server (e.g., Auth0, Keycloak). Configure the following settings: + - **Redirect URIs**: Add the redirect URI for your NoodleBar instance (e.g., `http:///signin-oidc`). + - **Client ID and Secret**: Obtain the client ID and secret for the application. + - **Authority URL**: Get the authority URL for the OAuth server. + +2. **Update Configuration in NoodleBar**: + Update the NoodleBar configuration to use the OAuth server for authentication. Modify the `appsettings.json` or environment variables to include the OAuth settings: + ```json + "Authentication": { + "OAuth": { + "Authority": "https:///", + "ClientId": "", + "ClientSecret": "" + } + } + ``` + +### 7.4 Deploying the NoodleBar Application + +All components of NoodleBar, including the Organization Register, Authorization Register, and the NoodleBar Web App, are deployed as a single application. Use the GitHub Actions workflow to deploy these components using the modified Bicep files. + +1. **Run the Deployment Workflow**: + Trigger the GitHub Actions workflow to deploy the NoodleBar application. This workflow will use the Azure CLI and ARM templates to set up the necessary resources and deploy the application. + +2. **Verify the Deployment**: + Ensure the deployment is successful by accessing the service endpoint: + ```sh + curl http:///health + ``` + +### 7.5 Configuring the NoodleBar System + +1. **Register as a New User**: + Go to `http://` to register as a new user using the OAuth provider. + +2. **Add Organizations**: + Use the NoodleBar web app to add new organizations to the Organization Register. + +3. **Define Authorization Policies**: + Set up authorization policies for the organizations using the Authorization Register. + +4. **Integrate Data Providers**: + Modify existing APIs to use the enforce API of the Authorization Register to control data access. + +By following these steps, you can successfully deploy NoodleBar using an OAuth server, setting up a fully functional dataspace prototype with enhanced security and interoperability. diff --git a/docs/8 - Deployment Using iSHARE.md b/docs/8 - Deployment Using iSHARE.md new file mode 100644 index 0000000..59a7b38 --- /dev/null +++ b/docs/8 - Deployment Using iSHARE.md @@ -0,0 +1,164 @@ +# 8: Deployment Using iSHARE + +The Stage 3 deployment of NoodleBar involves setting up the core components necessary for a highly secure and compliant dataspace prototype using the iSHARE standard. This stage offers the highest level of security and governance, making it suitable for organizations with stringent compliance requirements. + +### 8.1 Prerequisites + +Before you begin the deployment, ensure that you have the following prerequisites in place: + +- **NoodleBar fork on GitHub**: Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account. +- **Azure Subscription**: Ensure you have an active Azure subscription to deploy the necessary resources. +- **iSHARE Satellite**: Set up the iSHARE Satellite for managing organizational identities. +- **eIDAS Certificates**: Obtain eIDAS certificates for authentication and secure communication. + +### 8.2 Setting Up the Environment + +1. **Fork the NoodleBar Repository**: + Fork the Poort8.Dataspace.NoodleBar repository to your GitHub account: + ```sh + git clone https://github.com/YourAccount/Poort8.Dataspace.NoodleBar.git + cd Poort8.Dataspace.NoodleBar + ``` + +2. **Modify Bicep Files for Deployment**: + The repository includes Bicep files for deployment. Modify these files where applicable to fit your specific environment and requirements. The main Bicep files are: + - `main.bicep`: This is the main entry point for the deployment and references other Bicep modules. + - `dataspaceAlertModule.bicep`: Deploys alert rules for monitoring the resources. + - `dataspaceWorkspaceModule.bicep`: Sets up the Azure Log Analytics workspace. + - `customDomainModule.bicep`: Configures custom domain settings. + - `resourceGroupModule.bicep`: Creates the resource group for all resources. + - `sslBindingModule.bicep`: Sets up SSL bindings for secure connections. + - `appServiceModule.bicep`: Deploys the Azure App Service to host the NoodleBar application. + +3. **Set Configuration and Secrets**: + Configure environment variables and secrets needed for the deployment. Secrets must be configured in the GitHub repository settings under "Settings" -> "Secrets and variables" -> "Actions". Add the following secrets: + - `AZURE_CLIENT_ID` + - `AZURE_TENANT_ID` + - `AZURE_SUBSCRIPTION_ID` + - `ISHARE_CLIENT_ID` + - `ISHARE_CLIENT_SECRET` + - `ISHARE_CERTIFICATE` + - Any other secrets required for your specific setup, such as database connection strings. + +4. **Set Up GitHub Workflow to Deploy NoodleBar**: + Configure a GitHub workflow to automate the deployment of NoodleBar. Ensure that the workflow uses the modified Bicep files and secrets for deployment. + + Here's an example of a GitHub Actions workflow file (`.github/workflows/deploy.yml`): + + ```yaml + name: Deploy NoodleBar + + on: + workflow_dispatch: + push: + branches: + - master + paths: + - 'Poort8.Dataspace.CoreManager/**' + - 'Poort8.Dataspace.AuthorizationRegistry/**' + - 'Poort8.Dataspace.OrganizationRegistry/**' + - 'Poort8.Dataspace.API/**' + + permissions: + id-token: write + contents: read + + jobs: + deploy-preview: + runs-on: ubuntu-latest + env: + resourceGroupName: 'NoodleBar-Preview' + resourceGroupLocation: 'westeurope' + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + allow-no-subscriptions: true + + - name: Publish + run: dotnet publish ./Poort8.Dataspace.CoreManager/Poort8.Dataspace.CoreManager.csproj -c Release -o publish/app + + - name: Deploy resource group + uses: azure/arm-deploy@v2 + with: + scope: subscription + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + region: ${{ env.resourceGroupLocation }} + template: ./Poort8.Dataspace.CoreManager/deploy/resourceGroupModule.bicep + parameters: 'resourceGroupName=${{ env.resourceGroupName }} resourceGroupLocation=${{ env.resourceGroupLocation }}' + + - name: Deploy preview bicep + uses: azure/arm-deploy@v2 + with: + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + resourceGroupName: ${{ env.resourceGroupName }} + template: ./Poort8.Dataspace.CoreManager/deploy/main.bicep + parameters: 'environment=preview' + + - name: Deploy app to preview + uses: azure/webapps-deploy@v3 + with: + app-name: NoodleBar-Preview + package: publish/app + ``` + +### 8.3 Configuring the iSHARE Satellite + +1. **Deploy the iSHARE Satellite**: + Set up the iSHARE Satellite for managing organizational identities. Refer to the [iSHARE Satellite GitHub repository](https://github.com/iSHAREScheme/iSHARESatellite) for deployment instructions. + +2. **Obtain eIDAS Certificates**: + Obtain eIDAS certificates for authentication and secure communication. These certificates are mandatory for production environments and ensure secure data exchanges. + +3. **Update Configuration in NoodleBar**: + Update the NoodleBar configuration to use the iSHARE Satellite for authentication. Modify the `appsettings.json` or environment variables to include the iSHARE settings: + ```json + "Authentication": { + "iSHARE": { + "Authority": "https:///", + "ClientId": "", + "ClientSecret": "", + "Certificate": "" + } + } + ``` + +### 8.4 Deploying the NoodleBar Application + +All components of NoodleBar, including the Organization Register, Authorization Register, and the NoodleBar Web App, are deployed as a single application. Use the GitHub Actions workflow to deploy these components using the modified Bicep files. + +1. **Run the Deployment Workflow**: + Trigger the GitHub Actions workflow to deploy the NoodleBar application. This workflow will use the Azure CLI and ARM templates to set up the necessary resources and deploy the application. + +2. **Verify the Deployment**: + Ensure the deployment is successful by accessing the service endpoint: + ```sh + curl http:///health + ``` + +### 8.5 Configuring the NoodleBar System + +1. **Register as a New User**: + Go to `http://` to register as a new user using the iSHARE provider. + +2. **Add Organizations**: + Use the NoodleBar web app to add new organizations to the Organization Register. + +3. **Define Authorization Policies**: + Set up authorization policies for the organizations using the Authorization Register. + +4. **Integrate Data Providers**: + Modify existing APIs to use the enforce API of the Authorization Register to control data access. + +By following these steps, you can successfully deploy NoodleBar using the iSHARE standard, setting up a highly secure and compliant dataspace prototype. diff --git a/docs/databasemigrations.md b/docs/9 - Database Migrations.md similarity index 82% rename from docs/databasemigrations.md rename to docs/9 - Database Migrations.md index 5e83d21..4184af5 100644 --- a/docs/databasemigrations.md +++ b/docs/9 - Database Migrations.md @@ -1,4 +1,8 @@ -1. Copy the migrations folder from Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations, depending on whether you changed Organization registry or Authorization registry data models. +# 9: Database Migrations + +### 9.1 Running Migrations + +1. 1. Copy the migrations folder from Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations, depending on whether you changed Organization registry or Authorization registry data models. 2. In the AddOrganizationRegistrySqlite method in the Poort8.Dataspace.(_Organization/Authorization_)Registry.Extensions.DefaultExtension class, change the MigrationsAssembly from 'Poort8.Dataspace.(_Organization/Authorization_)Registry.SqliteMigrations' to 'Poort8.Dataspace.CoreManager'. This points the startup process to migration files of step 1. 3. Execute entity framework command to generate the migration script: * Package Manager Console: diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 0efcf40..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,169 +0,0 @@ -# Architecture -```mermaid -graph TD - -%% Dataspace Core -style Core fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; -subgraph Core[Dataspace Core] -subgraph Admin[Administration] - CoreManager[Core Manager] - organizationRegister[Organization Register] -end -AuthRegister[Authorization Register] -end - -%% BDI Source -style BDIServiceExt fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; -subgraph BDIServiceExt[BDI Source] - subgraph Dataspace_Adapter[Dataspace Adapter for non-BDI sources] - DataFetcher[Data fetcher] - DataMapper[Data Mapper] - AuthService[Authentication Service] - AuthzService[Authorization Service] - end - BDIServiceEndpoint[BDI Service] -end - - - -%% Dataspace Apps -style Dataspace_Apps fill:transparent,stroke:#333,stroke-width:2px,stroke-dasharray: 5, 5; -subgraph Dataspace_Apps[Dataspace Apps] - subgraph Onboarding[Dataspace Onboarding App] - RegisteringUsers[Registering Users] - AuthUsers2[Authenticate Users] - AuthMan[Registering Authorizations] - end - subgraph Prototype[Dataspace Prototype App] - AuthUsers[Authenticate Users] - PEP[Authorize Users] - RESTSrc[Retrieve Data from BDI-services] - DataStore[Data Storage/Caching] - Logic[Add Logic] - FrontEnds[Multiple Frontends] - end -end - - - -%% Connections within Dataspace Adapter -DataFetcher-->DataMapper -DataMapper-->AuthService -AuthService-->AuthzService - -%% Connections within Dataspace Core -CoreManager-->organizationRegister -CoreManager-->AuthRegister - -%% Dependencies -RegisteringUsers-->|Registers in|organizationRegister -AuthRegister-->|Checked By|PEP -AuthService-->|Checks|organizationRegister -organizationRegister-->|Checked By|AuthUsers -AuthzService-->BDIServiceEndpoint -BDIServiceEndpoint-->RESTSrc -AuthMan-->|Registers in|AuthRegister -AuthzService-->|Checks|AuthRegister - - -%% Connections within Dataspace Apps -AuthUsers2-->RegisteringUsers -RegisteringUsers-->AuthMan -RESTSrc-->DataStore -AuthUsers-->PEP -PEP-->RESTSrc -DataStore-->Logic -Logic-->FrontEnds -Onboarding-->Prototype - - - -``` - -## Dataspace Adapter - -### Objectives - -1. Convert non-BDI data sources into RESTful APIs. -2. Map data to Dataspace Schema (DCSA, P4, etc.). -3. Implement iSHARE authentication based on the organization Register. -4. Other authentication mechanisms like API-key. -5. Policy Enforcement Point (PEP) based on the Authorization Register. -6. Support for events and Pub/Sub mechanisms. - -### Components - -1. **API Gateway** - - Converts non-BDI data into RESTful services. - -2. **Data Mapper** - - Maps raw data to dataspace schema (DCSA, P4, etc.). - -3. **Authentication Service** - - Implements iSHARE authentication based on the organization Register. - - Supports other authentication methods like API keys. - -4. **Authorization Service** - - PEP implementation based on the Authorization Register. - -5. **Event Service** - - Supports real-time events and Pub/Sub mechanisms. - -### Deployment Options - -1. Integrated within a monolithic application. -2. Docker Containers (Service and Common). -3. Azure Functions. -4. Low-code platforms. - -### Considerations - -1. The adapter should be easily convertible to a BDI source. - ---- - -## Dataspace Core - -### Components - -1. **Organization Register** - - Entails: - - Organizations - - EP Creation? - - Trusted List? - - Requirements: - - iSHARE endpoints - - REST - - Independent database? - -2. **Authorization Register (AuthRegister)** - - Entails: - - Permissions - - Products and features - - Organizations and users - - Requirements: - - iSHARE endpoints - - REST - - Independent database? - -3. **Core Manager** - - Web app for managing the Organization Register and Authorization Register. - - IAA of the app? - - Use of Keyper? - - The Core manager also hosts a 'overview page' of available data sources and apps. - ---- - -## Dataspace Apps - -1. Retrieve data from REST sources (both non-BDI and BDI). -2. Authenticate users or machines. -3. PEP based on Authorization Register. -4. Data storage/caching. -5. Can add logic and combine sources. -6. Multiple frontends (chat, web app, email, etc.). - -### Deployment Options - -1. Integrated within a monolithic application with REST-like interfaces. -2. Separate backend system. diff --git a/docs/customerjourneys.md b/docs/customerjourneys.md deleted file mode 100644 index c04aa5e..0000000 --- a/docs/customerjourneys.md +++ /dev/null @@ -1,86 +0,0 @@ -# Dataspace Noodle Bar -Create your own dataspace in 6 steps. - -## 1. Deploy the base for your dataspace -The base layer of any dataspace must be organized by a dataspace initiator. - -1. **Dataspace Initiator Engagement**: The dataspace initiator goes through the incubator program to align with the principles and governance of a dataspace. -2. **Authentication Decision**: Decide what modes of authentication will be allowed for companies and employees. The following authentication mechanisms are supported: - - Username en passwords via ASP.NET Core Identity - - Magic link via email - - eHerkenning (_Requires implementation with external service provider_) - - Office 365 (_Requires implementation with external service provider_) -4. **Organization Register Setup**: Define what data needs to be collected for users in the dataspace organization register. -``` -deploy -Organization Register -Dataspace Manager -``` -4. **Registering participants**: Use the Dataspace Manager to register all participating organisations in the dataspace, or allow them to use self-service onboarding via the Dataspace Manager (customer journey 3). - -## 2. Create a landing zone for data sources - -The dataspace initiator can decide how much support the dataspace provides to data sources. -1. **Data Source Selection**: Identify potential data sources that align with the dataspace objectives. -2. **Register Data Sources**: Verify compliancy and register Data Sources in the organization register. -3. **Data model**: The dataspace initiator can define data model(s) that are used in the dataspace. If not, data sources can define their own data models and must organise authorizations accordingly. -4. **Authorization registry setup**: Optionally, the Dataspace initiator can establish an authorization registry for permissions and access controls in the dataspace. - - _Authorization usecase_: Based on the access control rules, an authorization usecase and authorization rules can be selected from existing models or a new one must be developed. Out-of-the-box, the following usecases are available:_ - - Access Control Lists - - ABAC (basic iSHARE) - - Subscription-based Access Control - - Or a custom ABAC usecase can be created for non-standard authorization rules. -5. **Dataspace Adapter**: Optionally, the Dataspace initiator can decide to provide a Dataspace Adapter to kickstart Data sources in the Dataspace. Each of these usecases can be applied in accordance with BDI standards for IAA. - - _Data Mapping_: Data sources specify how their data should be mapped to the dataspace data model. - - _Data Provisioning_: Data sources provide raw data in one of the formats supported by the dataspace (API, Data dump, Custom, etc.) - - _Data Standardization_: The Dataspace Adapter is configured to map the raw data to the Dataspace data model and provide a BDI service for it, including IAA. - -``` -deploy -Authorization Register -Authorization usecase -Dataspace adapter(s) -``` - -## 3. Registering Data Owners and Consumers -Allow Data Owners and Consumers to use self-service onboarding via the Dataspace Manager. -1. **Authentication and registration**: A representative of the organization uses an accepted Identity Provider to authenticate themselves as formal representation, and may provide additional details. -2. **Onboarding**: Organizations use the Dataspace onboarding service(s) to become a member of the Dataspace and be registered. _(Uses Organisation Register 1.3)_ -3. **Access Control**: Data owners define conditions under which their data can be accessed by data consumers. _(Uses Authorization Registry from 2.4)_ - -## 4. Use Keyper for easy onboarding -Use SaaS service with implemented authentication solutions to start immediately with high trust levels. -1. **Connect dataspace**: connect Keyper to your dataspace and select allowed authentication providers. -2. **Use Keyper**: Keyper provides seamless onboarding and management processes for BDI dataspaces, equivalent to those of journey 4, but including eHerkenning and Office 365 implementations. - -``` -configure -Keyper -``` - -## 5. Add a Prototype App -Use SaaS service to facilitate multi-channel access to prototype commands that combine data sources into valuable tools for users. -1. **Dataspace Prototype**: The Dataspace initiator can decide to connect a prototype app to kickstart usage of the dataspace. -2. **Initial App Design**: Decide on functionalities and commands to transform the raw data into valuable insights. _(Uses 2.2 and 2.3)_ -3. **Demonstration Setup**: Develop a demonstration flow to guide users through the features of the app. - -``` -configure -Prototype -``` -## 6. Seasoning with non-standardized data -Create temporary solutions to include non-standardized data in your dataspace. - -``` -develop -Data ETL -``` - -## 7. Extensions: Adding Providers and Apps to the Dataspace - -1. **New App Development**: Organizations design and build new apps in alignment with the dataspace. _(Uses 2.2 and 2.3)_ -2. **Dataspace Registration**: New apps register themselves with the dataspace as both data service consumers and providers. -3. **Usage Terms**: Set conditions under which the prototype app can be used. _(Can use Authorization Registry from 1.4)_ -4. **User Onboarding**: Registered users can now use these new apps to interact with data sources. - -