extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2024-11-21 23:23:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Handle 429s gracefully (#48)

Browse Source
This commit is contained in:
Gervasio Marchand 2023-12-15 19:19:39 -03:00 committed by GitHub
parent 3577a6072b
commit cbc49531ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/Program.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Net;
using System.Text.Json; using System.Text.Json;
using GetMoarFediverse; using GetMoarFediverse;
using GetMoarFediverse.Configuration; using GetMoarFediverse.Configuration;
@ -153,6 +154,7 @@ await Parallel.ForEachAsync(sitesTags, new ParallelOptions{MaxDegreeOfParallelis
var statusesToLoad = statusesToLoadBag.ToHashSet(); var statusesToLoad = statusesToLoadBag.ToHashSet();
Console.WriteLine($"Originally retrieved {statusesToLoadBag.Count} statuses. After removing duplicates, I got {statusesToLoad.Count} really unique ones"); Console.WriteLine($"Originally retrieved {statusesToLoadBag.Count} statuses. After removing duplicates, I got {statusesToLoad.Count} really unique ones");
var rateLimited = false;
foreach (var statusLink in statusesToLoad) foreach (var statusLink in statusesToLoad)
{ {
Console.WriteLine($"Bringing in {statusLink}"); Console.WriteLine($"Bringing in {statusLink}");
@ -164,8 +166,21 @@ foreach (var statusLink in statusesToLoad)
}; };
var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content)); var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
if (res.StatusCode == HttpStatusCode.TooManyRequests)
{
// fakerelay.gervas.io has a token bucket rate limit of 30 tokens per minute, allowing bursts of up to
// 60 requests per minute. Once we hit a 429, we should do a request every 2 seconds.
Console.WriteLine("Got a 429 from FakeRelay, waiting 2 second and retrying");
rateLimited = true;
await Task.Delay(TimeSpan.FromSeconds(2));
res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
}
res.EnsureSuccessStatusCode(); res.EnsureSuccessStatusCode();
importedList.Add(statusLink); importedList.Add(statusLink);
if (rateLimited)
{
await Task.Delay(TimeSpan.FromSeconds(2));
}
} }
catch (Exception e) catch (Exception e)
{ {