extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2024-11-21 23:23:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Check robots.txt and use a user agent (#17)

Browse Source
This commit is contained in:
Gervasio Marchand 2022-12-20 19:15:13 -08:00 committed by GitHub
parent d535512a6b
commit 23be734a5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/GetMoarFediverse.csproj
View File

@ -11,6 +11,7 @@
<ItemGroup>
<PackageReference Include="Npgsql" Version="7.0.1" />
<PackageReference Include="TurnerSoftware.RobotsExclusionTools" Version="0.9.1" />
</ItemGroup>
</Project>

39
src/Program.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Concurrent;
using System.Text.Json;
using GetMoarFediverse;
using TurnerSoftware.RobotsExclusionTools;
var configPath = Environment.GetEnvironmentVariable("CONFIG_PATH");
if (args.Length == 1){
@ -20,6 +21,8 @@ if (Config.Instance == null)
}
var client = new HttpClient();
client.DefaultRequestHeaders.Add("User-Agent", "GetMoarFediverse");
var authClient = new HttpClient
{
BaseAddress = new Uri(Config.Instance.FakeRelayUrl)
@ -32,9 +35,17 @@ if (!File.Exists(importedPath))
File.WriteAllText(importedPath, "");
}
var importedList = File.ReadAllLines(importedPath).ToList();
var imported = importedList.ToHashSet();
var statusesToLoadBag = new ConcurrentBag<string>();
ParallelOptions parallelOptions = new()
{
MaxDegreeOfParallelism = 8
};
var robotsFileParser = new RobotsFileParser();
var sitesRobotFile = new ConcurrentDictionary<string, RobotsFile>();
await Parallel.ForEachAsync(Config.Instance.Sites, parallelOptions, async (site, _) =>
{
sitesRobotFile[site.Host] = await robotsFileParser.FromUriAsync(new Uri($"http://{site.Host}/robots.txt"));
});
List<(string host, string tag)> sitesTags;
if (Config.Instance.MastodonPostgresConnectionString.HasValue())
@ -53,19 +64,29 @@ else
.ToList();
}
ParallelOptions parallelOptions = new()
{
MaxDegreeOfParallelism = 8
};
var importedList = File.ReadAllLines(importedPath).ToList();
var imported = importedList.ToHashSet();
var statusesToLoadBag = new ConcurrentBag<string>();
await Parallel.ForEachAsync(sitesTags, parallelOptions, async (st, _) =>
{
var (site, tag) = st;
Console.WriteLine($"Fetching tag #{tag} from {site}");
var url = $"https://{site}/tags/{tag}.json";
if (sitesRobotFile.TryGetValue(site, out var robotsFile))
{
var allowed = robotsFile.IsAllowedAccess(new Uri(url), "GetMoarFediverse");
if (!allowed)
{
Console.WriteLine($"Scraping {url} is not allowed based on their robots.txt file");
return;
}
}
HttpResponseMessage? response = null;
try
{
response = await client.GetAsync($"https://{site}/tags/{tag}.json");
response = await client.GetAsync(url);
response.EnsureSuccessStatusCode();
}
catch (Exception e)