diff --git a/MyApp.ServiceInterface/App/ImportQuestionCommand.cs b/MyApp.ServiceInterface/App/ImportQuestionCommand.cs index 1bf835f..2e2de3a 100644 --- a/MyApp.ServiceInterface/App/ImportQuestionCommand.cs +++ b/MyApp.ServiceInterface/App/ImportQuestionCommand.cs @@ -165,42 +165,52 @@ public async Task ExecuteAsync(ImportQuestion request) throw new Exception("Import failed"); } - private static async Task GetJsonFromRedditAsync(string url) + private async Task GetJsonFromRedditAsync(string url) { - // C# HttpClient requests are getting blocked - // var json = await url.GetJsonFromUrlAsync(requestFilter: c => - // { - // c.AddHeader(HttpHeaders.UserAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0"); - // c.AddHeader(HttpHeaders.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"); - // c.AddHeader(HttpHeaders.AcceptLanguage, "en-US,en;q=0.9"); - // c.AddHeader(HttpHeaders.CacheControl, "max-age=0"); - // }); - // return json; - - // Using curl Instead: - var args = new[] - { - $"curl -s '{url}'", - "-H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'", - "-H 'accept-language: en-US,en;q=0.9'", - "-H 'cache-control: max-age=0'", - "-H 'dnt: 1'", - "-H 'priority: u=0, i'", - "-H 'upgrade-insecure-requests: 1'", - "-H 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'" - }.ToList(); - if (Env.IsWindows) + url = url.Replace("www.reddit.com", "oauth.reddit.com"); + if (appConfig.RedditAccessToken != null) { - args = args.Map(x => x.Replace('\'', '"')); + try + { + return await url.GetJsonFromUrlAsync(requestFilter: req => { + req.AddBearerToken(appConfig.RedditAccessToken); + req.AddHeader("User-Agent", "pvq.app"); + }); + } + catch (Exception e) + { + log.LogWarning("Failed to fetch Reddit API: {Message}\nRetrieving new access_token...", e.Message); + appConfig.RedditAccessToken = null; + } } + + appConfig.RedditAccessToken = await FetchNewRedditAccessTokenAsync(); - var argsString = string.Join(" ", args); - var sb = StringBuilderCache.Allocate(); - await ProcessUtils.RunShellAsync(argsString, onOut:line => sb.AppendLine(line)); - var json = StringBuilderCache.ReturnAndFree(sb); + var json = await url.GetJsonFromUrlAsync(requestFilter: req => { + req.AddBearerToken(appConfig.RedditAccessToken); + req.AddHeader("User-Agent", "pvq.app"); + }); return json; } + private async Task FetchNewRedditAccessTokenAsync() + { + Dictionary postData = new() + { + ["grant_type"] = "client_credentials", + ["device_id"] = Guid.NewGuid().ToString("N"), + }; + var response = await "https://www.reddit.com/api/v1/access_token".PostToUrlAsync(postData, requestFilter: req => + { + req.AddBasicAuth( + appConfig.RedditClient ?? throw new ArgumentNullException(nameof(appConfig.RedditClient)), + appConfig.RedditSecret ?? throw new ArgumentNullException(nameof(appConfig.RedditSecret))); + req.AddHeader("User-Agent", "pvq.app"); + }); + var obj = (Dictionary)JSON.parse(response); + return (string)obj["access_token"]; + } + public static AskQuestion? CreateFromStackOverflowInlineEdit(string html) { var span = html.AsSpan(); diff --git a/MyApp.ServiceInterface/Data/AppConfig.cs b/MyApp.ServiceInterface/Data/AppConfig.cs index 6efc520..47d0f55 100644 --- a/MyApp.ServiceInterface/Data/AppConfig.cs +++ b/MyApp.ServiceInterface/Data/AppConfig.cs @@ -20,6 +20,9 @@ public class AppConfig public string AiServerBaseUrl { get; set; } public string AiServerApiKey { get; set; } + public string? RedditClient { get; set; } + public string? RedditSecret { get; set; } + public string? RedditAccessToken { get; set; } public JsonApiClient CreateAiServerClient() => new(AiServerBaseUrl) { BearerToken = AiServerApiKey }; public string CacheDir { get; set; } diff --git a/MyApp.Tests/ImportTests.cs b/MyApp.Tests/ImportTests.cs index bb3add3..b314291 100644 --- a/MyApp.Tests/ImportTests.cs +++ b/MyApp.Tests/ImportTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using MyApp.Data; using MyApp.ServiceInterface.App; using MyApp.ServiceModel; @@ -17,7 +18,11 @@ private static AppConfig CreateAppConfig() var hostDir = TestUtils.GetHostDir(); var allTagsFile = new FileInfo(Path.GetFullPath(Path.Combine(hostDir, "wwwroot", "data", "tags.txt"))); - var to = new AppConfig(); + var to = new AppConfig + { + RedditClient = Environment.GetEnvironmentVariable("REDDIT_CLIENT"), + RedditSecret = Environment.GetEnvironmentVariable("REDDIT_SECRET") + }; to.LoadTags(allTagsFile); return to; } @@ -27,13 +32,7 @@ public ImportTests() appConfig = CreateAppConfig(); } - class MockLogger : ILogger - { - public IDisposable? BeginScope(TState state) where TState : notnull => null; - public bool IsEnabled(LogLevel logLevel) => false; - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) {} - } - private ImportQuestionCommand CreateCommand() => new (new MockLogger(), appConfig); + private ImportQuestionCommand CreateCommand() => new(new NullLogger(), appConfig); [Test] public async Task Can_import_from_discourse_url() @@ -174,6 +173,7 @@ record RedditTest(string Url, string Title, string BodyPrefix, string[] Tags, st ) ]; + [Explicit("Requires REDDIT_CLIENT and REDDIT_SECRET env vars")] [Test] public async Task Can_import_from_reddit() { @@ -195,6 +195,36 @@ await command.ExecuteAsync(new ImportQuestion } } + [Explicit("Requires REDDIT_CLIENT and REDDIT_SECRET env vars")] + [Test] + public async Task Can_request_using_OAuth() + { + var redditOAuthUrl = "https://www.reddit.com/api/v1/access_token"; + var uuid = Guid.NewGuid().ToString("N"); + Dictionary postData = new() + { + ["grant_type"] = "client_credentials", + ["device_id"] = uuid, + }; + var response = await redditOAuthUrl.PostToUrlAsync(postData, requestFilter: req => + { + req.AddBasicAuth(Environment.GetEnvironmentVariable("REDDIT_CLIENT")!, Environment.GetEnvironmentVariable("REDDIT_SECRET")!); + req.AddHeader("User-Agent", "pvq.app"); + }); + + var obj = (Dictionary)JSON.parse(response); + obj.PrintDump(); + var accessToken = (string)obj["access_token"]; + var json = await "https://oauth.reddit.com/r/dotnet/comments/1byolum/all_the_net_tech_i_use_what_else_is_out_there.json" + .GetJsonFromUrlAsync(requestFilter: req => + { + req.AddBearerToken(accessToken); + req.AddHeader("User-Agent", "pvq.app"); + }); + + json.Print(); + } + [Explicit("Requires curl")] [Test] public async Task Can_call_curl_to_get_url() diff --git a/MyApp/Configure.AppHost.cs b/MyApp/Configure.AppHost.cs index eef9568..197c611 100644 --- a/MyApp/Configure.AppHost.cs +++ b/MyApp/Configure.AppHost.cs @@ -24,6 +24,9 @@ public void Configure(IWebHostBuilder builder) => builder services.AddSingleton(); + AppConfig.Instance.RedditClient ??= Environment.GetEnvironmentVariable("REDDIT_CLIENT"); + AppConfig.Instance.RedditSecret ??= Environment.GetEnvironmentVariable("REDDIT_SECRET"); + var r2Bucket = context.Configuration.GetValue("R2Bucket", "pvq-dev"); var r2AccountId = context.Configuration.GetValue("R2AccountId", Environment.GetEnvironmentVariable("R2_ACCOUNT_ID")); var r2AccessId = context.Configuration.GetValue("R2AccessKeyId", Environment.GetEnvironmentVariable("R2_ACCESS_KEY_ID"));