Implement Data Source Registration per ADR-0006

.github/workflows/1-pr.yaml

@@ -11,7 +11,7 @@ env:
   DLL_FILE_PATH: "./bin/Release/net6.0/CarbonAware.WebApi.dll"
   DOTNET_SRC_DIR: "./src"
   # console app packages
-  DOTNET_SOLUTION: "src/GSF.CarbonAware/src/GSF.CarbonAware.csproj"
+  DOTNET_SOLUTION: "src/CarbonAwareSDK.sln"
   OUTPUT_DIR: "packages"
   CONSOLE_APP: "samples/lib-integration/ConsoleApp/ConsoleApp.csproj"
   CREATE_PKGS: "scripts/package/create_packages.ps1"

.github/workflows/verify-azure-function-with-packages.yaml

@@ -30,23 +30,28 @@ jobs:
       - name: Docker Target Final
         run: |
           docker build --no-cache . -f ${{ env.DOCKERFILE_PATH }} -t ${{ env.CONTAINER_IMAGE_NAME }}
-          docker images -qf dangling=true | xargs --no-run-if-empty docker rmi
+          docker image prune -f
 
       - name: Docker Run Container
-        run: docker run -d --name ${{ env.CONTAINER_RUNTIME_NAME }} -p 8080:80 ${{ env.CONTAINER_IMAGE_NAME }}
+        run: |
+          docker run -d --name ${{ env.CONTAINER_RUNTIME_NAME }} -p 8080:80 ${{ env.CONTAINER_IMAGE_NAME }}
+          timeout 30 bash -c -- 'while [ -z "$(docker ps -q)" ]; do echo "waiting for container to start" && sleep 5s; done'
+          docker ps
 
       # Request fails with authentication error. Expected
       - name: Get Average Carbon Intensity
         run: |
           set +e
-          ret_code=$(wget -S -t 5 --waitretry=5 "http://0.0.0.0:8080/api/GetAverageCarbonIntensity?startDate=2022-03-01T15:30:00Z&endDate=2022-03-01T18:30:00Z&location=eastus" 2>&1 | grep "HTTP/" | awk '{print $2}')
+          ret_code=$(wget --server-response "http://0.0.0.0:8080/api/GetAverageCarbonIntensity?startDate=2022-03-01T15:30:00Z&endDate=2022-03-01T18:30:00Z&location=eastus" 2>&1 | awk '/^  HTTP/{print $2}')
+          echo "Return Code: $ret_code"
           set -e
-          [ "401" == $ret_code ]
+          [ "401" = "$ret_code" ] 
 
       # Request fails with authentication error. Expected
       - name: Get Current Forecast
         run: |
           set +e
-          ret_code=$(wget -S -t 5 --waitretry=5 "http://0.0.0.0:8080/api/GetCurrentForecast" --header "Content-Type: application/json" --post-data '{"startDate":"2022-11-02T15:30:00Z","endDate":"2022-11-02T18:30:00Z","location":"eastus","duration":"15"}' 2>&1 | grep "HTTP/" | awk '{print $2}')
+          ret_code=$(wget --server-response "http://0.0.0.0:8080/api/GetCurrentForecast" --header "Content-Type: application/json" --post-data '{"startDate":"2022-11-02T15:30:00Z","endDate":"2022-11-02T18:30:00Z","location":"eastus","duration":"15"}' 2>&1 | awk '/^  HTTP/{print $2}')
+          echo "Return Code: $ret_code"
           set -e
-          [ "401" == $ret_code ]
+          [ "401" = "$ret_code" ] 

.gitignore

@@ -33,3 +33,6 @@ src/data/location-sources/custom-azure-zones.json
 
 # exclude artifacts on java-client example
 samples/java-client/target
+
+# exclude artifacts from default package builder
+/packages

.vscode/launch.json

@@ -37,7 +37,8 @@
         "uriFormat": "%s/swagger"
       },
       "env": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "DOTNET_HOSTBUILDER__RELOADCONFIGONCHANGE": "false"
       },
       "sourceFileMap": {
         "/Views": "${workspaceFolder}/Views"

docs/architecture/data-sources.md

@@ -100,56 +100,28 @@ public class MyNewDataSource: IEmissionsDataSource
 
 ### Add Dependency Injection Configuration
 
-The SDK uses dependency injection to load registered data sources based on set
-environment variables. For a data source to be registered, it need to have a
-Service Collection Extension defined. To do so, add a `Configuration` directory
-in your data source project and create a new ServiceCollectionExtensions file.
-We have provided a command snippet below:
-
-```sh
-cd src/CarbonAware.DataSources/CarbonAware.DataSources.MyNewDataSource/src
-mkdir Configuration
-touch Configuration\ServiceCollectionExtensions.cs
-```
-
-Using the skeleton below, add the data source specific configuration and
-implementation instances to the service collection.
+The SDK uses dependency injection to load registered data sources based on configuration. For a data source to be registered, it needs to have a
+static method `ConfigureDI<T>` defined. We have provided an example below:
 
 ```csharp
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.DependencyInjection.Extensions;
-namespace CarbonAware.DataSources.MyNewDataSource.Configuration;
-public static class ServiceCollectionExtensions
+public static IServiceCollection ConfigureDI<T>(IServiceCollection services, DataSourcesConfiguration dataSourcesConfig)
+    where T : IDataSource
 {
-    public static void AddMyNewDataSource(this IServiceCollection services)
+    var configSection = dataSourcesConfig.ConfigurationSection<T>();
+    AddMyNewDataSourceClient(services, configSection);
+    services.AddScoped<ISomeInterface, MyDataSourceDependency>();
+    try
+    {
+        services.TryAddSingleton(typeof(T), typeof(MyNewDataSource));
+    } catch (Exception ex)
     {
-        // ... register your data source with the IServiceCollection instance
+        throw new ArgumentException($"MyNewDataSource is not a supported {typeof(T).Name} data source.", ex);
     }
+    return services;
 }
 ```
 
-### Register the New Data Source
-
-Once the data source's ServiceCollectionExtensions is configured, it can be
-registered as an available data source for the SDK by adding to the switch
-statement found in the AddDataSourceService function of
-[this file](../../src/CarbonAware.DataSources/CarbonAware.DataSources.Registration\Configuration\ServiceCollectionExtensions.cs).
-Note you will need to add a new enum type to the `DataSourceType`
-[enum file](../../src/CarbonAware.DataSources/CarbonAware.DataSources.Registration/Configuration/DataSourceType.cs)
-to reference in the switch statement.
-
-```csharp
-    switch (dataSourceType)
-    {
-        ...
-        case DataSourceType.MyNewDataSourceEnum:
-        {
-            services.AddMyNewDataSource();
-            break;
-        }
-        ...
-    }
-```
+This function will be called at runtime to configure your data source like `MyNewDataSource.ConfigureDI<IEmissionsDataSource>(services, config);`. For more examples, check out the [implementations of the existing data sources](/src/CarbonAware.DataSources/).
 
 ### Adding Tests
 
@@ -180,7 +152,8 @@ setting:
 
 ```bash
 DataSources__EmissionsDataSource="MyNewDataSource"
-DataSources__Configurations__MyNewDataSource__Proxy__UseProxy=true
+DataSources__Configurations__MyNewDataSource__Username="MyNewDataSourceUser123"
+DataSources__Configurations__MyNewDataSource__Password="MyNewDataSourceP@ssword!"
 ```
 
 Both the WebAPI and the CLI read the env variables in so once set, you can spin

docs/architecture/overview.md

@@ -84,19 +84,14 @@ result.
 
 See the [data source README](./data-sources.md) for more detailed information.
 
-## Dependency Registration
-
-The SDK uses dependency injection to load the data sources based on set
-environment variables. To register a new dependency, a new
-ServiceCollectionExtension method must be defined. These dependencies are loaded
-in a hierarchical structure such that:
-
-1. Each data source defines a `ServiceCollectionExtension` method.
-2. All available data sources are registered in the `DataSource.Registration`
-   project.
-3. The GSF library defines a `ServiceCollectionExtension` method where it
-   registers the data sources for the handlers to use.
-4. The `Program.cs` file registers the GSF library classes at startup
+### Dependency Registration
+
+The SDK uses dependency injection to load the data sources based on configuration. To register a new dependency, the data source musr define a static method `ConfigureDI<T>`. These dependencies are then loaded in the following manner:
+
+1. Each data source defines a `ConfigureDI<T>` method.
+2. The GSF library defines a `ServiceCollectionExtension` method where it
+   uses the configuration settings to dynamically load and configure the user-specified data sources for the handlers to use.
+3. The `Program.cs` file registers the GSF library classes at startup
 
 ## Example Call Flow
 

docs/packaging.md

@@ -18,22 +18,21 @@ showing how the package can be consumed.
   - [Console App Sample](#console-app-sample)
     - [Run the Sample Console App](#run-the-sample-console-app)
 
-## Included Projects
+## Included Packages
 
-The current package include 8 projects from the SDK:
+The current solution creates 5 packages from the SDK projects:
 
 1. "GSF.CarbonAware"
-2. "CarbonAware"
-3. "CarbonAware.DataSources.ElectricityMapsFree"
-4. "CarbonAware.DataSources.ElectricityMaps"
-5. "CarbonAware.DataSources.Json"
-6. "CarbonAware.DataSources.Registration"
-7. "CarbonAware.DataSources.WattTime"
-8. "CarbonAware.LocationSources"
-
-These 8 projects enable users of the library to consume the current endpoints
-exposed by the library. The package that needs to be added to a new C# project
-is `GSF.CarbonAware`.
+  1. "CarbonAware"
+  2. "CarbonAware.LocationSources"
+2. "CarbonAware.DataSources.ElectricityMapsFree"
+3. "CarbonAware.DataSources.ElectricityMaps"
+4. "CarbonAware.DataSources.Json"
+5. "CarbonAware.DataSources.WattTime"
+
+Adding the `GSF.CarbonAware` project to your C# projects gives you access to all
+CarbonAware SDK interfaces.
+Additionally, operators must add any data source packages they plan to use.
 
 ## Included Scripts
 

samples/azure/azure-function/Dockerfile

@@ -10,7 +10,9 @@ COPY ["scripts/", "data/scripts/"]
 COPY ["samples/", "data/samples/"]
 
 WORKDIR /data
-RUN pwsh -c scripts/package/create_packages.ps1 src/GSF.CarbonAware/src/GSF.CarbonAware.csproj /packages && \
+RUN pwsh -c scripts/package/create_packages.ps1 src/CarbonAwareSDK.sln /packages && \
+    dotnet nuget locals all --clear && \
+    dotnet nuget add source /packages -n carbonaware && \
     dotnet restore "samples/azure/azure-function/function.csproj" && \
     pwsh -c scripts/package/add_packages.ps1 samples/azure/azure-function/function.csproj /packages && \
     dotnet build "samples/azure/azure-function/function.csproj" -c Release -o /app/build
@@ -20,4 +22,6 @@ RUN dotnet publish "samples/azure/azure-function/function.csproj" -c Release -o
 
 FROM base AS final
 WORKDIR /home/site/wwwroot
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true
 COPY --from=publish /app/publish .

samples/azure/azure-function/function.csproj

@@ -4,6 +4,9 @@
     <AzureFunctionsVersion>v4</AzureFunctionsVersion>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="CarbonAware.DataSources.Json" Version="1.0.0" />
+    <PackageReference Include="CarbonAware.DataSources.WattTime" Version="1.0.0" />
+    <PackageReference Include="GSF.CarbonAware" Version="1.0.0-beta" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />

samples/azure/azure-function/host.json

@@ -6,6 +6,10 @@
                 "isEnabled": true,
                 "excludedTypes": "Request"
             }
+        }, 
+        "logLevel": {
+            "default": "Information"
         }
-    }
+    },
+    "tracing": {"consoleLevel": "verbose"}
 }

scripts/package/add_packages.ps1

@@ -13,6 +13,10 @@ if ([string]::IsNullOrEmpty($DOTNET_PROJECT) -or [string]::IsNullOrEmpty($PACKAG
 
 $packages = @(
     "GSF.CarbonAware"
+    "CarbonAware.DataSources.Json"
+    "CarbonAware.DataSources.WattTime"
+    "CarbonAware.DataSources.ElectricityMaps"
+    "CarbonAware.DataSources.ElectricityMapsFree"
 )
 
 # Remove packages from project
@@ -21,6 +25,7 @@ foreach ($pname in $packages) {
 }
 
 Remove-Item -Recurse -Force ~/.nuget/packages/gsf.carbonaware*
+Remove-Item -Recurse -Force ~/.nuget/packages/carbonaware.datasources*
 
 # Add packages to project
 foreach ($pname in $packages) {

scripts/package/create_packages.ps1

@@ -14,7 +14,7 @@ if ([string]::IsNullOrEmpty($DOTNET_SOLUTION) -or [string]::IsNullOrEmpty($DEST_
 $REVISION = $(git rev-parse HEAD)
 $BRANCH = "dev"
 
-New-Item -Path $DEST_PACKAGES -ItemType Directory
+New-Item -Path $DEST_PACKAGES -ItemType Directory -Force | Out-Null
 
 # Remove existing packages
 Get-ChildItem -Path $path -Recurse -Include *.nupkg, *.snupkg | Remove-Item

src/CarbonAware.CLI/src/CarbonAware.CLI.csproj

@@ -27,6 +27,14 @@
     <ProjectReference Include="..\..\CarbonAware\src\CarbonAware.csproj" />
     <ProjectReference Include="..\..\CarbonAware.LocationSources\src\CarbonAware.LocationSources.csproj" />
     <ProjectReference Include="..\..\GSF.CarbonAware\src\GSF.CarbonAware.csproj" />
+    <ProjectReference
+      Include="..\..\CarbonAware.DataSources\CarbonAware.DataSources.Json\src\CarbonAware.DataSources.Json.csproj" />
+    <ProjectReference
+      Include="..\..\CarbonAware.DataSources\CarbonAware.DataSources.WattTime\src\CarbonAware.DataSources.WattTime.csproj" />
+    <ProjectReference
+      Include="..\..\CarbonAware.DataSources\CarbonAware.DataSources.ElectricityMaps\src\CarbonAware.DataSources.ElectricityMaps.csproj" />
+    <ProjectReference
+      Include="..\..\CarbonAware.DataSources\CarbonAware.DataSources.ElectricityMapsFree\src\CarbonAware.DataSources.ElectricityMapsFree.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/CarbonAware.CLI/test/integrationTests/CarbonAware.CLI.IntegrationTests.csproj

@@ -15,8 +15,6 @@
   <ItemGroup>
     <ProjectReference
       Include="..\..\..\CarbonAware.DataSources\CarbonAware.DataSources.Json\mock\CarbonAware.DataSources.Json.Mocks.csproj" />
-    <ProjectReference
-      Include="..\..\..\CarbonAware.DataSources\CarbonAware.DataSources.Registration\CarbonAware.DataSources.Registration.csproj" />
     <ProjectReference
       Include="..\..\..\CarbonAware.DataSources\CarbonAware.DataSources.WattTime\mock\CarbonAware.DataSources.WattTime.Mocks.csproj" />
     <ProjectReference

src/CarbonAware.CLI/test/integrationTests/Commands/Emissions/EmissionsCommandTests.cs

@@ -1,4 +1,3 @@
-using CarbonAware.DataSources.Configuration;
 using NUnit.Framework;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;

src/CarbonAware.CLI/test/integrationTests/Commands/EmissionsForecasts/EmissionsForecastsCommandTests.cs

@@ -1,5 +1,4 @@
-using CarbonAware.DataSources.Configuration;
-using NUnit.Framework;
+using NUnit.Framework;
 using System.Text.Json.Nodes;
 
 namespace CarbonAware.CLI.IntegrationTests.Commands.EmissionsForecasts;

src/CarbonAware.CLI/test/integrationTests/Commands/Location/LocationCommandTests.cs

@@ -1,5 +1,4 @@
-using CarbonAware.DataSources.Configuration;
-using NUnit.Framework;
+using NUnit.Framework;
 using System.Text.Json;
 
 namespace CarbonAware.CLI.IntegrationTests.Commands.Location;

src/CarbonAware.CLI/test/integrationTests/IntegrationTestingBase.cs

@@ -1,5 +1,4 @@
-using CarbonAware.DataSources.Configuration;
-using CarbonAware.Interfaces;
+using CarbonAware.Interfaces;
 using CarbonAware.DataSources.ElectricityMaps.Mocks;
 using CarbonAware.DataSources.ElectricityMapsFree.Mocks;
 using CarbonAware.DataSources.Json.Mocks;
@@ -11,6 +10,15 @@
 
 namespace CarbonAware.CLI.IntegrationTests;
 
+public enum DataSourceType
+{
+    None,
+    WattTime,
+    JSON,
+    ElectricityMaps,
+    ElectricityMapsFree,
+}
+
 /// <summary>
 /// A base class that does all the common setup for the Integration Testing
 /// Overrides WebAPI factory by switching out different configurations via _datasource

src/CarbonAware.DataSources/CarbonAware.DataSources.ElectricityMaps/mock/CarbonAware.DataSources.ElectricityMaps.Mocks.csproj

@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\src\CarbonAware.DataSources.ElectricityMaps.csproj" />
+    <ProjectReference Include="..\..\..\CarbonAware\src\CarbonAware.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/CarbonAware.DataSources/CarbonAware.DataSources.ElectricityMaps/src/CarbonAware.DataSources.ElectricityMaps.csproj

@@ -10,15 +10,25 @@
   <ItemGroup>
     <InternalsVisibleTo Include="CarbonAware.DataSources.ElectricityMaps.Mocks" />
     <InternalsVisibleTo Include="CarbonAware.DataSources.ElectricityMaps.Tests" />
-    <InternalsVisibleTo Include="CarbonAware.DataSources.Registration" />
     <InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\..\CarbonAware\src\CarbonAware.csproj" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
+    <ItemGroup>
+    <ProjectReference Include="..\..\..\CarbonAware\src\CarbonAware.csproj" PrivateAssets="all" />
+  </ItemGroup>
+
+  <!-- Add files required to create a package -->
+  <ItemGroup Label="FilesToCopy">
+    <Content Include="$(AssemblyName).targets" Pack="true"
+      PackagePath="build/$(AssemblyName).targets" />
+    <Content Include="$(TargetDir)/CarbonAware.dll" Pack="true" PackagePath="lib/net6.0/" />
+  </ItemGroup>
+
+  <ItemGroup Label="IncludeSymbols" Condition="$(IncludeSymbols) == true">
+    <Content Include="$(TargetDir)/CarbonAware.pdb" Pack="true" PackagePath="lib/net6.0/" />
   </ItemGroup>
 </Project>

src/CarbonAware.DataSources/CarbonAware.DataSources.ElectricityMaps/src/CarbonAware.DataSources.ElectricityMaps.targets

@@ -0,0 +1,8 @@
+<Project>
+    <ItemGroup>
+        <DllFiles Include="$(MSBuildThisFileDirectory)/../lib/net6.0/*.dll" />
+    </ItemGroup>
+    <Target Name="CopyDllFiles" BeforeTargets="BeforeBuild">
+        <Copy SourceFiles="@(DllFiles)" DestinationFolder="$(TargetDir)" />
+    </Target>
+</Project>