From 73c21242b4a70f0314bbfc62dd2841a38afc4bcb Mon Sep 17 00:00:00 2001 From: jfrazx Date: Mon, 22 Jan 2024 20:36:20 -0800 Subject: [PATCH] feat: utilize p-throttle instad of limiter --- package-lock.json | 26 +++++++++---------- package.json | 2 +- server/providers/Audnexus.js | 48 +++++++++++------------------------- 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/package-lock.json b/package-lock.json index 65316f65..30b40e28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,11 +15,11 @@ "express-session": "^1.17.3", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", - "limiter": "^2.1.0", "lru-cache": "^10.0.3", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "openid-client": "^5.6.1", + "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "sequelize": "^6.35.2", @@ -2841,11 +2841,6 @@ "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, - "node_modules/just-performance": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz", - "integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==" - }, "node_modules/jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", @@ -2865,14 +2860,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/limiter": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz", - "integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==", - "dependencies": { - "just-performance": "4.3.0" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -3977,6 +3964,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-4.1.1.tgz", + "integrity": "sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", diff --git a/package.json b/package.json index 65dbdd1e..46624752 100644 --- a/package.json +++ b/package.json @@ -41,11 +41,11 @@ "express-session": "^1.17.3", "graceful-fs": "^4.2.10", "htmlparser2": "^8.0.1", - "limiter": "^2.1.0", "lru-cache": "^10.0.3", "node-tone": "^1.0.1", "nodemailer": "^6.9.2", "openid-client": "^5.6.1", + "p-throttle": "^4.1.1", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "sequelize": "^6.35.2", diff --git a/server/providers/Audnexus.js b/server/providers/Audnexus.js index d9223374..14eab4a1 100644 --- a/server/providers/Audnexus.js +++ b/server/providers/Audnexus.js @@ -1,7 +1,7 @@ const axios = require('axios') const { levenshteinDistance } = require('../utils/index') const Logger = require('../Logger') -const { RateLimiter } = require('limiter') +const pThrottle = require('p-throttle') class Audnexus { static _instance = null @@ -14,11 +14,15 @@ class Audnexus { this.baseUrl = 'https://api.audnex.us' + // Rate limit is 100 requests per minute. // @see https://github.com/laxamentumtech/audnexus#-deployment- - this.limiter = new RateLimiter({ - tokensPerInterval: 100, - fireImmediately: true, - interval: 'minute', + this.limiter = pThrottle({ + // Setting the limit to 1 allows for a short pause between requests that is almost imperceptible to + // the end user. A larger limit will grab blocks faster and then wait for the alloted time(interval) before + // fetching another batch. + limit: 1, + strict: true, + interval: 300 }) Audnexus._instance = this @@ -31,8 +35,8 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return this._processRequest(() => axios.get(authorRequestUrl)) - .then((res) => res.data || []) + const throttle = this.limiter(() => axios.get(authorRequestUrl)) + return throttle().then((res) => res.data || []) .catch((error) => { Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error) return [] @@ -46,36 +50,14 @@ class Audnexus { Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`) - return this._processRequest(() => axios.get(authorRequestUrl)) - .then((res) => res.data) + const throttle = this.limiter(() => axios.get(authorRequestUrl)) + return throttle().then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Author request failed for ${asin}`, error) return null }) } - /** - * @description Process a request with a rate limiter - * - * @param {*} request - * @returns - */ - async _processRequest(request) { - const remainingTokens = await this.limiter.removeTokens(1) - Logger.info(`[Audnexus] Attempting request with ${remainingTokens} remaining tokens and ${this.limiter.tokensThisInterval} this interval`) - - if (remainingTokens >= 1) { - return request() - } - - // 100 tokens(requests) per minute give a refresh of ~1.67 per second, - // so a 10 second wait will yield ~16.7 additional tokens - Logger.info('[Audnexus] Sleeping for 10 seconds') - await new Promise(resolve => setTimeout(resolve, 10000)) - - return this._processRequest(request) - } - async findAuthorByASIN(asin, region) { const author = await this.authorRequest(asin, region) @@ -111,8 +93,8 @@ class Audnexus { getChaptersByASIN(asin, region) { Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`) - return this._processRequest(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) - .then((res) => res.data) + const throttle = this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`)) + return throttle().then((res) => res.data) .catch((error) => { Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error) return null