Merge pull request #2 from dominikus1993/rewrite

Add Reddit Support
dominikus1993 · Apr 4, 2021 · 664619f · 664619f
2 parents 802ae81 + 24f2917
commit 664619f
Show file tree

Hide file tree

Showing 18 changed files with 153 additions and 59 deletions.
diff --git a/.ionide/symbolCache.db b/.ionide/symbolCache.db
diff --git a/paket.dependencies b/paket.dependencies
@@ -16,6 +16,7 @@ nuget Discord.Net.Webhook
 nuget Microsoft.Extensions.DependencyInjection.Abstractions
 nuget Microsoft.Extensions.Configuration.Abstractions
 nuget Microsoft.Extensions.Configuration
+nuget Microsoft.Extensions.Configuration.Binder 
 nuget Microsoft.Extensions.Configuration.EnvironmentVariables
 nuget Microsoft.Extensions.Configuration.Json
 nuget Microsoft.Extensions.Configuration.UserSecrets

diff --git a/paket.lock b/paket.lock
@@ -44,20 +44,10 @@ NUGET
       System.Reflection.Emit.Lightweight (>= 4.3)
       System.Runtime.Serialization.Formatters (>= 4.3)
       System.ValueTuple (>= 4.5)
-    Microsoft.AspNetCore.Hosting.Abstractions (2.2)
-      Microsoft.AspNetCore.Hosting.Server.Abstractions (>= 2.2)
-      Microsoft.AspNetCore.Http.Abstractions (>= 2.2)
-      Microsoft.Extensions.Hosting.Abstractions (>= 2.2)
-    Microsoft.AspNetCore.Hosting.Server.Abstractions (2.2)
-      Microsoft.AspNetCore.Http.Features (>= 2.2)
-      Microsoft.Extensions.Configuration.Abstractions (>= 2.2)
-    Microsoft.AspNetCore.Http.Abstractions (2.2)
-      Microsoft.AspNetCore.Http.Features (>= 2.2)
-      System.Text.Encodings.Web (>= 4.5)
     Microsoft.AspNetCore.Http.Features (5.0.4)
       Microsoft.Extensions.Primitives (>= 5.0)
       System.IO.Pipelines (>= 5.0.1)
-    Microsoft.CodeCoverage (16.9.1)
+    Microsoft.CodeCoverage (16.9.4)
     Microsoft.CSharp (4.7)
     Microsoft.Extensions.Configuration (5.0)
       Microsoft.Extensions.Configuration.Abstractions (>= 5.0)
@@ -179,16 +169,16 @@ NUGET
       Microsoft.Extensions.Options (>= 5.0)
       Microsoft.Extensions.Primitives (>= 5.0)
     Microsoft.Extensions.Primitives (5.0)
-    Microsoft.NET.Test.Sdk (16.9.1)
-      Microsoft.CodeCoverage (>= 16.9.1)
-      Microsoft.TestPlatform.TestHost (>= 16.9.1)
+    Microsoft.NET.Test.Sdk (16.9.4)
+      Microsoft.CodeCoverage (>= 16.9.4)
+      Microsoft.TestPlatform.TestHost (>= 16.9.4)
     Microsoft.NETCore.Platforms (5.0.1)
     Microsoft.NETCore.Targets (5.0)
-    Microsoft.TestPlatform.ObjectModel (16.9.1)
+    Microsoft.TestPlatform.ObjectModel (16.9.4)
       NuGet.Frameworks (>= 5.0)
       System.Reflection.Metadata (>= 1.6)
-    Microsoft.TestPlatform.TestHost (16.9.1)
-      Microsoft.TestPlatform.ObjectModel (>= 16.9.1)
+    Microsoft.TestPlatform.TestHost (16.9.4)
+      Microsoft.TestPlatform.ObjectModel (>= 16.9.4)
       Newtonsoft.Json (>= 9.0.1)
     Microsoft.Win32.Primitives (4.3)
       Microsoft.NETCore.Platforms (>= 1.1)
@@ -224,10 +214,9 @@ NUGET
       Microsoft.NETCore.Platforms (>= 1.1.1)
       Microsoft.NETCore.Targets (>= 1.1.3)
     Serilog (2.10)
-    Serilog.AspNetCore (4.0)
-      Microsoft.AspNetCore.Hosting.Abstractions (>= 2.2)
-      Microsoft.Extensions.DependencyInjection (>= 3.1.8)
-      Microsoft.Extensions.Logging (>= 3.1.8)
+    Serilog.AspNetCore (4.1)
+      Microsoft.Extensions.DependencyInjection (>= 5.0)
+      Microsoft.Extensions.Logging (>= 5.0)
       Serilog (>= 2.10)
       Serilog.Extensions.Hosting (>= 4.1.2)
       Serilog.Formatting.Compact (>= 1.1)
@@ -516,7 +505,6 @@ NUGET
       Microsoft.NETCore.Targets (>= 1.1)
       System.Runtime (>= 4.3)
       System.Text.Encoding (>= 4.3)
-    System.Text.Encodings.Web (5.0.1)
     System.Text.Json (5.0.1)
     System.Threading (4.3)
       System.Runtime (>= 4.3)

diff --git a/src/DevNews.Cli/DevNews.Cli.csproj b/src/DevNews.Cli/DevNews.Cli.csproj
@@ -1,9 +1,10 @@
-<?xml version="1.0" encoding="utf-8"?>
+
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net5.0</TargetFramework>
     <AnalysisLevel>latest</AnalysisLevel>
+    <UserSecretsId>34d628c9-a3dc-4fd4-9842-3a74e7396cf9</UserSecretsId>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\DevNews.Core\DevNews.Core.csproj" />

diff --git a/src/DevNews.Cli/Program.cs b/src/DevNews.Cli/Program.cs
@@ -25,9 +25,10 @@ await CoconaApp.Create()
                 .UseLogger("DevNews.Cli")
                 .ConfigureServices((ctx, services) =>
                 {
-                    services.AddPersistence(ctx.Configuration);
-                    services.AddNotifiers(ctx.Configuration);
-                    services.AddParsers();
+                    var configuration = ctx.Configuration;
+                    services.AddPersistence(configuration);
+                    services.AddNotifiers(configuration);
+                    services.AddParsers(configuration);
                     services.AddCore();
                 })
                 .ConfigureAppConfiguration(builder =>

diff --git a/src/DevNews.Cli/appsettings.json b/src/DevNews.Cli/appsettings.json
@@ -11,5 +11,8 @@
         "Articles": "mongodb://root:rootpassword@127.0.0.1:27017",
         "Discord": "",
         "Teams": ""
+    },
+    "Reddit": {
+        "SubReddits": ["dotnet", "fsharp", "golang", "python", "node"]
     }
 }
diff --git a/src/DevNews.Core/Model/Aggregates.cs b/src/DevNews.Core/Model/Aggregates.cs
@@ -13,7 +13,8 @@ public Article(string title, string link) : this(title, null, link)
 
         public bool IsValidArticle()
         {
-            return Uri.TryCreate(Link, UriKind.Absolute, out var result)
+            var contentIsValid = Content is null || Content.Length < 2048;
+            return contentIsValid && Uri.TryCreate(Link, UriKind.Absolute, out var result)
                    && (result.Scheme == Uri.UriSchemeHttp || result.Scheme == Uri.UriSchemeHttps);
         }
     }

diff --git a/src/DevNews.Infrastructure.Notifications/Discord/DiscordConfiguration.cs b/src/DevNews.Infrastructure.Notifications/Discord/DiscordConfiguration.cs
@@ -1,7 +1,9 @@
+using System.Diagnostics.CodeAnalysis;
+
 namespace DevNews.Infrastructure.Notifications.Discord
 {
     public class DiscordConfiguration
     {
-        public string? WebHookUrl { get; init; }
+       public string WebHookUrl { get; set; } = "";
     }
 }
diff --git a/src/DevNews.Infrastructure.Parsers/DependencyInjection/Extensions.cs b/src/DevNews.Infrastructure.Parsers/DependencyInjection/Extensions.cs
@@ -1,17 +1,22 @@
+using System;
 using DevNews.Core.Abstractions;
 using DevNews.Infrastructure.Parsers.Dotnetomaniak;
 using DevNews.Infrastructure.Parsers.HackerNews;
+using DevNews.Infrastructure.Parsers.Reddit;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace DevNews.Infrastructure.Parsers.DependencyInjection
 {
     public static class Extensions
     {
-        public static IServiceCollection AddParsers(this IServiceCollection services)
+        public static IServiceCollection AddParsers(this IServiceCollection services, IConfiguration configuration)
         {
             services.AddTransient<IArticlesParser, HackerNewsArticlesParser>();
             services.AddTransient<IArticlesParser, DotnetomaniakArticlesParser>();
+            services.AddReddit(configuration);
             return services;
         }
+
     }
 }
diff --git a/src/DevNews.Infrastructure.Parsers/Dotnetomaniak/DotnetomaniakArticlesParser.cs b/src/DevNews.Infrastructure.Parsers/Dotnetomaniak/DotnetomaniakArticlesParser.cs
@@ -1,13 +1,15 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using DevNews.Core.Abstractions;
 using DevNews.Core.Model;
 using HtmlAgilityPack;
 
+[assembly: InternalsVisibleTo("DevNews.Infrastructure.Parsers.UnitTests")]
 namespace DevNews.Infrastructure.Parsers.Dotnetomaniak
 {
-    public class DotnetomaniakArticlesParser : IArticlesParser
+    internal class DotnetomaniakArticlesParser : IArticlesParser
     {
         private const string DotnetoManiakUrl = "https://dotnetomaniak.pl/";
         private static readonly Uri DotnetoManiakUri = new(DotnetoManiakUrl);
@@ -18,11 +20,11 @@ public async IAsyncEnumerable<Article> Parse()
             var document = await html.LoadFromWebAsync(DotnetoManiakUrl);
 
             var nodes = document.DocumentNode.SelectNodes("//*[@class=\"article\"]")
-                .Select(x => x.ChildNodes).Select(static n => CreateArticle(n));
+                .Select(x => x.ChildNodes);
 
-            foreach (var article in nodes)
+            foreach (var node in nodes)
             {
-                yield return article;
+                yield return CreateArticle(node);
             }
         }
 
@@ -31,7 +33,7 @@ private static Article CreateArticle(HtmlNodeCollection nodes)
             var titleNode = nodes.First(div => div.HasClass("title"));
             var href = titleNode.ChildNodes.FindFirst("a").GetAttributeValue("href", null);
             var link = new Uri(DotnetoManiakUri, href).AbsoluteUri;
-            var description = nodes.First(div => div.HasClass("description")).ChildNodes.FindFirst("span").InnerText;;
+            var description = nodes.First(div => div.HasClass("description")).ChildNodes.FindFirst("span").InnerText;
             var title = titleNode.InnerText;
             return new Article(title, description, link);
         }

diff --git a/src/DevNews.Infrastructure.Parsers/HackerNews/HackerNewsArticleParser.cs b/src/DevNews.Infrastructure.Parsers/HackerNews/HackerNewsArticleParser.cs
@@ -1,26 +1,26 @@
 using System.Collections.Generic;
-using System.Linq;
+using System.Runtime.CompilerServices;
 using DevNews.Core.Abstractions;
 using DevNews.Core.Model;
 using HtmlAgilityPack;
 
+[assembly: InternalsVisibleTo("DevNews.Infrastructure.Parsers.UnitTests")]
 namespace DevNews.Infrastructure.Parsers.HackerNews
 {
-    public class HackerNewsArticlesParser : IArticlesParser
+    internal class HackerNewsArticlesParser : IArticlesParser
     {
         private const string HackerNewsUrl = "https://news.ycombinator.com/";
 
         public async IAsyncEnumerable<Article> Parse()
         {
             var html = new HtmlWeb();
             var document = await html.LoadFromWebAsync(HackerNewsUrl);
-
-            var nodes = document.DocumentNode.SelectNodes("//*[@class=\"storylink\"]")
-                .Select(static node => new Article(node.InnerText, node.GetAttributeValue("href", null)));
-
-            foreach (var article in nodes)
+
+            var nodes = document.DocumentNode.SelectNodes("//*[@class=\"storylink\"]");
+
+            foreach (var node in nodes)
             {
-                yield return article;
+                yield return new Article(node.InnerText, node.GetAttributeValue("href", null));
             }
         }
     }

diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/Configuration.cs b/src/DevNews.Infrastructure.Parsers/Reddit/Configuration.cs
@@ -1,7 +1,7 @@
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
-    public class RedditConfiguration
+    internal class RedditConfiguration
     {
-        public string[]? SubReddits { get; init; } 
+        public string[]? SubReddits { get; set; }
     }
 }
diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/Extensions.cs b/src/DevNews.Infrastructure.Parsers/Reddit/Extensions.cs
@@ -0,0 +1,20 @@
+using System;
+using DevNews.Core.Abstractions;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace DevNews.Infrastructure.Parsers.Reddit
+{
+    internal static class Extensions
+    {
+        internal static void AddReddit(this IServiceCollection services, IConfiguration configuration)
+        {
+            services.AddSingleton(configuration.GetSection("Reddit").Get<RedditConfiguration>());
+            services.AddHttpClient<SubRedditParser>(client =>
+            {
+                client.BaseAddress = new Uri("https://www.reddit.com/");
+            });
+            services.AddTransient<IArticlesParser, RedditParser>();
+        }
+    }
+}
diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/Model.cs b/src/DevNews.Infrastructure.Parsers/Reddit/Model.cs
@@ -1,7 +1,31 @@
+using System.Text.Json.Serialization;
+
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
-    public class Model
+    internal class Data
     {
-
+        [JsonPropertyName("children")] 
+        public PostData[]? Posts { get; init; }
     }
+
+    internal record PostData
+    {
+        [JsonPropertyName("data")] 
+        public Post? Post { get; init; }
+    }
+
+    public class Post
+    {
+        [JsonPropertyName("title")] 
+        public string? Title { get; init; }   
+        [JsonPropertyName("url")] 
+        public string? Url { get; init; } 
+        [JsonPropertyName("selftext")] 
+        public string? Content { get; set; }
+    }
+    internal class Subreddit
+    {
+        [JsonPropertyName("data")]
+        public Data? Data { get; init; }
+    };
 }
diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/RedditParser.cs b/src/DevNews.Infrastructure.Parsers/Reddit/RedditParser.cs
@@ -1,11 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Threading.Tasks;
+using System.Runtime.CompilerServices;
 using DevNews.Core.Abstractions;
 using DevNews.Core.Model;
+using LanguageExt;
 using Open.ChannelExtensions;
 
+[assembly: InternalsVisibleTo("DevNews.Infrastructure.Parsers.UnitTests")]
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
     internal class RedditParser : IArticlesParser
@@ -26,9 +28,10 @@ public async IAsyncEnumerable<Article> Parse()
             {
                 throw new AggregateException(nameof(_redditConfiguration.SubReddits));
             }
-            
+
             var subRedditPostsChannel = subs.ToChannel()
                 .TaskPipeAsync(Environment.ProcessorCount, async sub => await _subRedditParser.Parse(sub))
+                .Pipe(static articlesOption => articlesOption.IfNone(static () => Array.Empty<Article>()))
                 .Join();
 
             await foreach (var article in subRedditPostsChannel.ReadAllAsync())

diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/SubRedditParser.cs b/src/DevNews.Infrastructure.Parsers/Reddit/SubRedditParser.cs
@@ -1,23 +1,37 @@
 using System;
 using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Json;
 using System.Threading.Tasks;
 using DevNews.Core.Model;
-using HtmlAgilityPack;
+using LanguageExt;
+using static LanguageExt.Prelude;
 
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
-    public class SubRedditParser
+    internal sealed class SubRedditParser
     {
+        private const string PostToDownload = "10";
+        private readonly HttpClient _client;
 
-        public Task<Article[]> Parse(string name)
+        public SubRedditParser(HttpClient client)
         {
-            var url = $"https://www.reddit.com/r/{name}";
-            var html = new HtmlWeb();
-            var document = await html.LoadFromWebAsync(url);
-
-            var nodes = document.DocumentNode.SelectNodes("//*[@class=\"storylink\"]")
-                .Select(static node => new Article(node.InnerText, node.GetAttributeValue("href", null)));
-
+            _client = client;
+        }
+
+        public async Task<Option<Article[]>> Parse(string name)
+        {
+            var url = $"r/{name}/top/.json?limit={PostToDownload}";
+            var result = await _client.GetFromJsonAsync<Subreddit>(url);
+            if (result?.Data?.Posts is null)
+            {
+                return None;
+            }
+
+            return result.Data.Posts
+                .Where(x => x.Post is not null)
+                .Select(x => new Article(x.Post.Title, x.Post.Content, x.Post.Url))
+                .ToArray();
         }
     }
 }
diff --git a/src/DevNews.Infrastructure.Parsers/paket.references b/src/DevNews.Infrastructure.Parsers/paket.references
@@ -1,2 +1,5 @@
 HtmlAgilityPack
-Microsoft.Extensions.DependencyInjection.Abstractions
+Microsoft.Extensions.DependencyInjection.Abstractions
+Microsoft.Extensions.Configuration.Abstractions
+Microsoft.Extensions.Configuration.Binder
+Microsoft.Extensions.Http