extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2025-01-10 07:08:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

make it more parallel

Browse Source
This commit is contained in:
Gervasio Marchand 2022-12-05 08:56:11 -03:00
parent 54828b9469
commit 28fe45a075
No known key found for this signature in database
GPG Key ID: B7736CB188DD0A38
1 changed files with 34 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/Program.cs
View File

@ -17,43 +17,46 @@ if (!File.Exists(importedPath))
File.WriteAllText(importedPath, ""); File.WriteAllText(importedPath, "");
} }
var imported = File.ReadAllLines(importedPath).ToHashSet(); var importedList = File.ReadAllLines(importedPath).ToList();
var imported = importedList.ToHashSet();
var statusesToLoadBag = new ConcurrentBag<string>(); var statusesToLoadBag = new ConcurrentBag<string>();
var sitesTags = Config.Instance.Sites
.SelectMany(s => Config.Instance.Tags.Select(tag => (s.Host, tag)))
.Concat(Config.Instance.Sites.SelectMany(s => s.SiteSpecificTags.Select(tag => (s.Host, tag))))
.OrderBy(t => t.tag)
.ToList();
ParallelOptions parallelOptions = new() ParallelOptions parallelOptions = new()
{ {
MaxDegreeOfParallelism = 8 MaxDegreeOfParallelism = 8
}; };
await Parallel.ForEachAsync(Config.Instance.Sites, parallelOptions, async (site, _) => await Parallel.ForEachAsync(sitesTags, parallelOptions, async (st, _) =>
{ {
var tags = site.SiteSpecificTags.Concat(Config.Instance.Tags).ToList(); var (site, tag) = st;
foreach (var tag in tags) Console.WriteLine($"Fetching tag #{tag} from {site}");
var response = await client.GetAsync($"https://{site}/tags/{tag}.json");
try
{ {
Console.WriteLine($"Fetching tag #{tag} from {site.Host}"); response.EnsureSuccessStatusCode();
var response = await client.GetAsync($"https://{site.Host}/tags/{tag}.json"); }
try catch (Exception e)
{ {
response.EnsureSuccessStatusCode(); Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}");
} return;
catch (Exception e) }
{
Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}");
continue;
}
var json = await response.Content.ReadAsStringAsync(); var json = await response.Content.ReadAsStringAsync();
var data = JSON.Deserialize<TagResponse>(json, Options.CamelCase); var data = JSON.Deserialize<TagResponse>(json, Options.CamelCase);
foreach (var statusLink in data.OrderedItems.Where(i=>!imported.Contains(i))) foreach (var statusLink in data.OrderedItems.Where(i=>!imported.Contains(i)))
{ {
statusesToLoadBag.Add(statusLink); statusesToLoadBag.Add(statusLink);
}
} }
}); });
var statusesToLoad = statusesToLoadBag.ToHashSet(); var statusesToLoad = statusesToLoadBag.ToHashSet();
var importedOnThisRun = new List<string>();
foreach (var statusLink in statusesToLoad) foreach (var statusLink in statusesToLoad)
{ {
Console.WriteLine($"Bringing in {statusLink}"); Console.WriteLine($"Bringing in {statusLink}");
@ -64,7 +67,7 @@ foreach (var statusLink in statusesToLoad)
var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content)); var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
res.EnsureSuccessStatusCode(); res.EnsureSuccessStatusCode();
importedOnThisRun.Add(statusLink); importedList.Add(statusLink);
} }
catch (Exception e) catch (Exception e)
{ {
@ -72,7 +75,14 @@ foreach (var statusLink in statusesToLoad)
} }
} }
File.AppendAllLines(importedPath, importedOnThisRun); if (importedList.Count > 1000)
{
importedList = importedList
.Skip(importedList.Count - 1000)
.ToList();
}
File.WriteAllLines(importedPath, importedList);
public class TagResponse public class TagResponse
{ {