extern/fediwall
1
0
Fork 1
mirror of https://github.com/defnull/fediwall.git synced 2025-02-02 18:29:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Smarter delay between requests

Browse Source 
Wait a randomized and slowly growing amount of time between requests to the same domain.
This commit is contained in:
Marcel Hellkamp 2024-12-27 10:17:44 +01:00
parent ceea0b4aff
commit 460341491d
4 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/App.vue
View File

@ -135,7 +135,7 @@ async function updateWall() {
if (progress.errors.length) { if (progress.errors.length) {
setStatus(progress.errors.slice(-1)[0].message, true) setStatus(progress.errors.slice(-1)[0].message, true)
} else if (progress.finished < progress.total) { } else if (progress.finished < progress.total) {
setStatus(`Loading [${progress.finished}/${progress.total}] sources ...`) setStatus(`Loading ${progress.finished}/${progress.total} sources ...`)
} else { } else {
setStatus(false) setStatus(false)
} }

2
src/main.ts
View File

@ -1,5 +1,5 @@
/** /**
Copyright (C) 2023 Marcel Hellkamp Copyright (C) 2024 Marcel Hellkamp
Copyright (C) 2023 Gesellschaft für wissenschaftliche Datenverarbeitung mbH Göttingen Copyright (C) 2023 Gesellschaft für wissenschaftliche Datenverarbeitung mbH Göttingen
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify

8
src/sources.ts
View File

@ -102,11 +102,13 @@ export async function fetchPosts(cfg: Config, onProgress: (progress: Progress) =
} }
// Be nice and not overwhelm servers with parallel requests. // Be nice and not overwhelm servers with parallel requests.
// Run tasks for the same domain in sequence instead. // Run tasks for the same domain in sequence instead, and wait between
// requests for a small random amount of time.
const groupedTasks = Object.entries(domainTasks) const groupedTasks = Object.entries(domainTasks)
.map(([domain, tasks]) => { .map(([domain, tasks]) => {
return async () => { return async () => {
for (const task of tasks) { for (const [taskIndex, task] of tasks.entries()) {
await sleep(Math.min(Math.random() * 100 * taskIndex, 500))
progress.started += 1; progress.started += 1;
try { try {
(await task()) (await task())
@ -168,8 +170,6 @@ async function fetchJson(domain: string, path: string, query?: Record<string, an
let rs: Response; let rs: Response;
let json: any; let json: any;
await new Promise(resolve => setTimeout(resolve, 1000));
try { try {
rs = await fetch(url) rs = await fetch(url)

4
src/utils.ts
View File

@ -27,6 +27,10 @@ export function deepClone(obj: any) {
return JSON.parse(JSON.stringify(obj)) return JSON.parse(JSON.stringify(obj))
} }
export function sleep(ms:number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/** /**
* Find all text nodes and replace each occurrences of a pattern with either * Find all text nodes and replace each occurrences of a pattern with either