small changes

dominikus1993 · Apr 1, 2021 · 802ae81 · 802ae81
1 parent 5f0ac7b
commit 802ae81
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 6 deletions.
diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/RedditParser.cs b/src/DevNews.Infrastructure.Parsers/Reddit/RedditParser.cs
@@ -4,13 +4,14 @@
 using System.Threading.Tasks;
 using DevNews.Core.Abstractions;
 using DevNews.Core.Model;
+using Open.ChannelExtensions;
 
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
     internal class RedditParser : IArticlesParser
     {
-        private RedditConfiguration _redditConfiguration;
-        private SubRedditParser _subRedditParser;
+        private readonly RedditConfiguration _redditConfiguration;
+        private readonly SubRedditParser _subRedditParser;
 
         public RedditParser(RedditConfiguration redditConfiguration, SubRedditParser subRedditParser)
         {
@@ -25,10 +26,12 @@ public async IAsyncEnumerable<Article> Parse()
             {
                 throw new AggregateException(nameof(_redditConfiguration.SubReddits));
             }
+
+            var subRedditPostsChannel = subs.ToChannel()
+                .TaskPipeAsync(Environment.ProcessorCount, async sub => await _subRedditParser.Parse(sub))
+                .Join();
 
-            var articles = await Task.WhenAll(subs.Select(sub => _subRedditParser.Parse(sub)));
-
-            foreach (var article in articles.SelectMany(x => x))
+            await foreach (var article in subRedditPostsChannel.ReadAllAsync())
             {
                 yield return article;
             }

diff --git a/src/DevNews.Infrastructure.Parsers/Reddit/SubRedditParser.cs b/src/DevNews.Infrastructure.Parsers/Reddit/SubRedditParser.cs
@@ -1,6 +1,8 @@
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 using DevNews.Core.Model;
+using HtmlAgilityPack;
 
 namespace DevNews.Infrastructure.Parsers.Reddit
 {
@@ -9,7 +11,13 @@ public class SubRedditParser
 
         public Task<Article[]> Parse(string name)
         {
-            return Task.FromResult(Array.Empty<Article>());
+            var url = $"https://www.reddit.com/r/{name}";
+            var html = new HtmlWeb();
+            var document = await html.LoadFromWebAsync(url);
+
+            var nodes = document.DocumentNode.SelectNodes("//*[@class=\"storylink\"]")
+                .Select(static node => new Article(node.InnerText, node.GetAttributeValue("href", null)));
+
         }
     }
 }