From cbc49531ad200daab5d06ec2fbb0e21f89802963 Mon Sep 17 00:00:00 2001
From: Gervasio Marchand <github@gervas.io>
Date: Fri, 15 Dec 2023 19:19:39 -0300
Subject: [PATCH] Handle 429s gracefully (#48)

---
 src/Program.cs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/Program.cs b/src/Program.cs
index d6a7562..e4a9bab 100644
--- a/src/Program.cs
+++ b/src/Program.cs
@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Net;
 using System.Text.Json;
 using GetMoarFediverse;
 using GetMoarFediverse.Configuration;
@@ -153,6 +154,7 @@ await Parallel.ForEachAsync(sitesTags, new ParallelOptions{MaxDegreeOfParallelis
 
 var statusesToLoad = statusesToLoadBag.ToHashSet();
 Console.WriteLine($"Originally retrieved {statusesToLoadBag.Count} statuses. After removing duplicates, I got {statusesToLoad.Count} really unique ones");
+var rateLimited = false;
 foreach (var statusLink in statusesToLoad)
 {
     Console.WriteLine($"Bringing in {statusLink}");
@@ -164,8 +166,21 @@ foreach (var statusLink in statusesToLoad)
         };
 
         var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
+        if (res.StatusCode == HttpStatusCode.TooManyRequests)
+        {
+            // fakerelay.gervas.io has a token bucket rate limit of 30 tokens per minute, allowing bursts of up to
+            // 60 requests per minute. Once we hit a 429, we should do a request every 2 seconds. 
+            Console.WriteLine("Got a 429 from FakeRelay, waiting 2 second and retrying");
+            rateLimited = true;
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
+        }
         res.EnsureSuccessStatusCode();
         importedList.Add(statusLink);
+        if (rateLimited)
+        {
+            await Task.Delay(TimeSpan.FromSeconds(2));
+        }
     }
     catch (Exception e)
     {