extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2024-11-21 15:13:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

If we can't pull the robots information, don't break completely (#37)

Browse Source
This commit is contained in:
Gervasio Marchand 2023-02-04 18:29:44 -03:00 committed by GitHub
parent 32280c334b
commit 91ac0e2385
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/Program.cs
View File

@ -41,7 +41,14 @@ await Parallel.ForEachAsync(Config.Instance.Sites,
new ParallelOptions { MaxDegreeOfParallelism = Config.Instance.Sites.Length },
async (site, _) =>
{
sitesRobotFile[site.Host] = await robotsFileParser.FromUriAsync(new Uri($"http://{site.Host}/robots.txt"));
try
{
sitesRobotFile[site.Host] = await robotsFileParser.FromUriAsync(new Uri($"http://{site.Host}/robots.txt"));
}
catch
{
Console.WriteLine($"Ignoring {site.Host} because had issues fetching its robots data (is the site down?)");
}
}
);
@ -88,6 +95,11 @@ await Parallel.ForEachAsync(sitesTags, new ParallelOptions{MaxDegreeOfParallelis
return;
}
}
else
{
Console.WriteLine($"Not scraping {url} because I couldn't fetch robots data.");
return;
}
HttpResponseMessage? response = null;
try