From 460341491dcee0e94b995707ed82d05d45aafbaa Mon Sep 17 00:00:00 2001
From: Marcel Hellkamp <marc@gsites.de>
Date: Fri, 27 Dec 2024 10:17:44 +0100
Subject: [PATCH] Smarter delay between requests
Wait a randomized and slowly growing amount of time between requests to the same domain.
---
src/App.vue | 2 +-
src/main.ts | 2 +-
src/sources.ts | 8 ++++----
src/utils.ts | 4 ++++
4 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/src/App.vue b/src/App.vue
index 21fad69..6931a09 100644
--- a/src/App.vue
+++ b/src/App.vue
@@ -135,7 +135,7 @@ async function updateWall() {
if (progress.errors.length) {
setStatus(progress.errors.slice(-1)[0].message, true)
} else if (progress.finished < progress.total) {
- setStatus(`Loading [${progress.finished}/${progress.total}] sources ...`)
+ setStatus(`Loading ${progress.finished}/${progress.total} sources ...`)
} else {
setStatus(false)
}
diff --git a/src/main.ts b/src/main.ts
index e4c0621..b42a9d9 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -1,5 +1,5 @@
/**
- Copyright (C) 2023 Marcel Hellkamp
+ Copyright (C) 2024 Marcel Hellkamp
Copyright (C) 2023 Gesellschaft für wissenschaftliche Datenverarbeitung mbH Göttingen
This program is free software: you can redistribute it and/or modify
diff --git a/src/sources.ts b/src/sources.ts
index 4a0c958..50d8d55 100644
--- a/src/sources.ts
+++ b/src/sources.ts
@@ -102,11 +102,13 @@ export async function fetchPosts(cfg: Config, onProgress: (progress: Progress) =
}
// Be nice and not overwhelm servers with parallel requests.
- // Run tasks for the same domain in sequence instead.
+ // Run tasks for the same domain in sequence instead, and wait between
+ // requests for a small random amount of time.
const groupedTasks = Object.entries(domainTasks)
.map(([domain, tasks]) => {
return async () => {
- for (const task of tasks) {
+ for (const [taskIndex, task] of tasks.entries()) {
+ await sleep(Math.min(Math.random() * 100 * taskIndex, 500))
progress.started += 1;
try {
(await task())
@@ -168,8 +170,6 @@ async function fetchJson(domain: string, path: string, query?: Record<string, an
let rs: Response;
let json: any;
- await new Promise(resolve => setTimeout(resolve, 1000));
-
try {
rs = await fetch(url)
diff --git a/src/utils.ts b/src/utils.ts
index d0277f4..bdb8424 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -27,6 +27,10 @@ export function deepClone(obj: any) {
return JSON.parse(JSON.stringify(obj))
}
+export function sleep(ms:number) {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
/**
* Find all text nodes and replace each occurrences of a pattern with either