Skip to content

Commit

Permalink
Add proper support for Reddit Import
Browse files Browse the repository at this point in the history
  • Loading branch information
@mythz
mythz committed May 3, 2024
1 parent 67cc182 commit a4f0fbc
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 37 deletions.
68 changes: 39 additions & 29 deletions MyApp.ServiceInterface/App/ImportQuestionCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,42 +165,52 @@ public async Task ExecuteAsync(ImportQuestion request)
throw new Exception("Import failed");
}

private static async Task<string> GetJsonFromRedditAsync(string url)
private async Task<string> GetJsonFromRedditAsync(string url)
{
// C# HttpClient requests are getting blocked
// var json = await url.GetJsonFromUrlAsync(requestFilter: c =>
// {
// c.AddHeader(HttpHeaders.UserAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0");
// c.AddHeader(HttpHeaders.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
// c.AddHeader(HttpHeaders.AcceptLanguage, "en-US,en;q=0.9");
// c.AddHeader(HttpHeaders.CacheControl, "max-age=0");
// });
// return json;

// Using curl Instead:
var args = new[]
{
$"curl -s '{url}'",
"-H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'",
"-H 'accept-language: en-US,en;q=0.9'",
"-H 'cache-control: max-age=0'",
"-H 'dnt: 1'",
"-H 'priority: u=0, i'",
"-H 'upgrade-insecure-requests: 1'",
"-H 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'"
}.ToList();
if (Env.IsWindows)
url = url.Replace("www.reddit.com", "oauth.reddit.com");
if (appConfig.RedditAccessToken != null)
{
args = args.Map(x => x.Replace('\'', '"'));
try
{
return await url.GetJsonFromUrlAsync(requestFilter: req => {
req.AddBearerToken(appConfig.RedditAccessToken);
req.AddHeader("User-Agent", "pvq.app");
});
}
catch (Exception e)
{
log.LogWarning("Failed to fetch Reddit API: {Message}\nRetrieving new access_token...", e.Message);
appConfig.RedditAccessToken = null;
}
}

appConfig.RedditAccessToken = await FetchNewRedditAccessTokenAsync();

var argsString = string.Join(" ", args);
var sb = StringBuilderCache.Allocate();
await ProcessUtils.RunShellAsync(argsString, onOut:line => sb.AppendLine(line));
var json = StringBuilderCache.ReturnAndFree(sb);
var json = await url.GetJsonFromUrlAsync(requestFilter: req => {
req.AddBearerToken(appConfig.RedditAccessToken);
req.AddHeader("User-Agent", "pvq.app");
});
return json;
}

private async Task<string> FetchNewRedditAccessTokenAsync()
{
Dictionary<string, object> postData = new()
{
["grant_type"] = "client_credentials",
["device_id"] = Guid.NewGuid().ToString("N"),
};
var response = await "https://www.reddit.com/api/v1/access_token".PostToUrlAsync(postData, requestFilter: req =>
{
req.AddBasicAuth(
appConfig.RedditClient ?? throw new ArgumentNullException(nameof(appConfig.RedditClient)),
appConfig.RedditSecret ?? throw new ArgumentNullException(nameof(appConfig.RedditSecret)));
req.AddHeader("User-Agent", "pvq.app");
});
var obj = (Dictionary<string,object>)JSON.parse(response);
return (string)obj["access_token"];
}

public static AskQuestion? CreateFromStackOverflowInlineEdit(string html)
{
var span = html.AsSpan();
Expand Down
3 changes: 3 additions & 0 deletions MyApp.ServiceInterface/Data/AppConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ public class AppConfig

public string AiServerBaseUrl { get; set; }
public string AiServerApiKey { get; set; }
public string? RedditClient { get; set; }
public string? RedditSecret { get; set; }
public string? RedditAccessToken { get; set; }
public JsonApiClient CreateAiServerClient() => new(AiServerBaseUrl) { BearerToken = AiServerApiKey };

public string CacheDir { get; set; }
Expand Down
46 changes: 38 additions & 8 deletions MyApp.Tests/ImportTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using MyApp.Data;
using MyApp.ServiceInterface.App;
using MyApp.ServiceModel;
Expand All @@ -17,7 +18,11 @@ private static AppConfig CreateAppConfig()
var hostDir = TestUtils.GetHostDir();
var allTagsFile = new FileInfo(Path.GetFullPath(Path.Combine(hostDir, "wwwroot", "data", "tags.txt")));

var to = new AppConfig();
var to = new AppConfig
{
RedditClient = Environment.GetEnvironmentVariable("REDDIT_CLIENT"),
RedditSecret = Environment.GetEnvironmentVariable("REDDIT_SECRET")
};
to.LoadTags(allTagsFile);
return to;
}
Expand All @@ -27,13 +32,7 @@ public ImportTests()
appConfig = CreateAppConfig();
}

class MockLogger<T> : ILogger<T>
{
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => false;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) {}
}
private ImportQuestionCommand CreateCommand() => new (new MockLogger<ImportQuestionCommand>(), appConfig);
private ImportQuestionCommand CreateCommand() => new(new NullLogger<ImportQuestionCommand>(), appConfig);

[Test]
public async Task Can_import_from_discourse_url()
Expand Down Expand Up @@ -174,6 +173,7 @@ record RedditTest(string Url, string Title, string BodyPrefix, string[] Tags, st
)
];

[Explicit("Requires REDDIT_CLIENT and REDDIT_SECRET env vars")]
[Test]
public async Task Can_import_from_reddit()
{
Expand All @@ -195,6 +195,36 @@ await command.ExecuteAsync(new ImportQuestion
}
}

[Explicit("Requires REDDIT_CLIENT and REDDIT_SECRET env vars")]
[Test]
public async Task Can_request_using_OAuth()
{
var redditOAuthUrl = "https://www.reddit.com/api/v1/access_token";
var uuid = Guid.NewGuid().ToString("N");
Dictionary<string, object> postData = new()
{
["grant_type"] = "client_credentials",
["device_id"] = uuid,
};
var response = await redditOAuthUrl.PostToUrlAsync(postData, requestFilter: req =>
{
req.AddBasicAuth(Environment.GetEnvironmentVariable("REDDIT_CLIENT")!, Environment.GetEnvironmentVariable("REDDIT_SECRET")!);
req.AddHeader("User-Agent", "pvq.app");
});

var obj = (Dictionary<string,object>)JSON.parse(response);
obj.PrintDump();
var accessToken = (string)obj["access_token"];
var json = await "https://oauth.reddit.com/r/dotnet/comments/1byolum/all_the_net_tech_i_use_what_else_is_out_there.json"
.GetJsonFromUrlAsync(requestFilter: req =>
{
req.AddBearerToken(accessToken);
req.AddHeader("User-Agent", "pvq.app");
});

json.Print();
}

[Explicit("Requires curl")]
[Test]
public async Task Can_call_curl_to_get_url()
Expand Down
3 changes: 3 additions & 0 deletions MyApp/Configure.AppHost.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ public void Configure(IWebHostBuilder builder) => builder
services.AddSingleton<ImageCreator>();
AppConfig.Instance.RedditClient ??= Environment.GetEnvironmentVariable("REDDIT_CLIENT");
AppConfig.Instance.RedditSecret ??= Environment.GetEnvironmentVariable("REDDIT_SECRET");
var r2Bucket = context.Configuration.GetValue("R2Bucket", "pvq-dev");
var r2AccountId = context.Configuration.GetValue("R2AccountId", Environment.GetEnvironmentVariable("R2_ACCOUNT_ID"));
var r2AccessId = context.Configuration.GetValue("R2AccessKeyId", Environment.GetEnvironmentVariable("R2_ACCESS_KEY_ID"));
Expand Down

0 comments on commit a4f0fbc

Please sign in to comment.