mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-11-07 08:34:10 +01:00
feat: utilize p-throttle instad of limiter
This commit is contained in:
parent
06391b9b37
commit
73c21242b4
26
package-lock.json
generated
26
package-lock.json
generated
@ -15,11 +15,11 @@
|
|||||||
"express-session": "^1.17.3",
|
"express-session": "^1.17.3",
|
||||||
"graceful-fs": "^4.2.10",
|
"graceful-fs": "^4.2.10",
|
||||||
"htmlparser2": "^8.0.1",
|
"htmlparser2": "^8.0.1",
|
||||||
"limiter": "^2.1.0",
|
|
||||||
"lru-cache": "^10.0.3",
|
"lru-cache": "^10.0.3",
|
||||||
"node-tone": "^1.0.1",
|
"node-tone": "^1.0.1",
|
||||||
"nodemailer": "^6.9.2",
|
"nodemailer": "^6.9.2",
|
||||||
"openid-client": "^5.6.1",
|
"openid-client": "^5.6.1",
|
||||||
|
"p-throttle": "^4.1.1",
|
||||||
"passport": "^0.6.0",
|
"passport": "^0.6.0",
|
||||||
"passport-jwt": "^4.0.1",
|
"passport-jwt": "^4.0.1",
|
||||||
"sequelize": "^6.35.2",
|
"sequelize": "^6.35.2",
|
||||||
@ -2841,11 +2841,6 @@
|
|||||||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/just-performance": {
|
|
||||||
"version": "4.3.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.3.0.tgz",
|
|
||||||
"integrity": "sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q=="
|
|
||||||
},
|
|
||||||
"node_modules/jwa": {
|
"node_modules/jwa": {
|
||||||
"version": "1.4.1",
|
"version": "1.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
|
"resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
|
||||||
@ -2865,14 +2860,6 @@
|
|||||||
"safe-buffer": "^5.0.1"
|
"safe-buffer": "^5.0.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/limiter": {
|
|
||||||
"version": "2.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/limiter/-/limiter-2.1.0.tgz",
|
|
||||||
"integrity": "sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==",
|
|
||||||
"dependencies": {
|
|
||||||
"just-performance": "4.3.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/locate-path": {
|
"node_modules/locate-path": {
|
||||||
"version": "5.0.0",
|
"version": "5.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||||
@ -3977,6 +3964,17 @@
|
|||||||
"url": "https://github.com/sponsors/sindresorhus"
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/p-throttle": {
|
||||||
|
"version": "4.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-4.1.1.tgz",
|
||||||
|
"integrity": "sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/p-try": {
|
"node_modules/p-try": {
|
||||||
"version": "2.2.0",
|
"version": "2.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
|
||||||
|
@ -41,11 +41,11 @@
|
|||||||
"express-session": "^1.17.3",
|
"express-session": "^1.17.3",
|
||||||
"graceful-fs": "^4.2.10",
|
"graceful-fs": "^4.2.10",
|
||||||
"htmlparser2": "^8.0.1",
|
"htmlparser2": "^8.0.1",
|
||||||
"limiter": "^2.1.0",
|
|
||||||
"lru-cache": "^10.0.3",
|
"lru-cache": "^10.0.3",
|
||||||
"node-tone": "^1.0.1",
|
"node-tone": "^1.0.1",
|
||||||
"nodemailer": "^6.9.2",
|
"nodemailer": "^6.9.2",
|
||||||
"openid-client": "^5.6.1",
|
"openid-client": "^5.6.1",
|
||||||
|
"p-throttle": "^4.1.1",
|
||||||
"passport": "^0.6.0",
|
"passport": "^0.6.0",
|
||||||
"passport-jwt": "^4.0.1",
|
"passport-jwt": "^4.0.1",
|
||||||
"sequelize": "^6.35.2",
|
"sequelize": "^6.35.2",
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
const axios = require('axios')
|
const axios = require('axios')
|
||||||
const { levenshteinDistance } = require('../utils/index')
|
const { levenshteinDistance } = require('../utils/index')
|
||||||
const Logger = require('../Logger')
|
const Logger = require('../Logger')
|
||||||
const { RateLimiter } = require('limiter')
|
const pThrottle = require('p-throttle')
|
||||||
|
|
||||||
class Audnexus {
|
class Audnexus {
|
||||||
static _instance = null
|
static _instance = null
|
||||||
@ -14,11 +14,15 @@ class Audnexus {
|
|||||||
|
|
||||||
this.baseUrl = 'https://api.audnex.us'
|
this.baseUrl = 'https://api.audnex.us'
|
||||||
|
|
||||||
|
// Rate limit is 100 requests per minute.
|
||||||
// @see https://github.com/laxamentumtech/audnexus#-deployment-
|
// @see https://github.com/laxamentumtech/audnexus#-deployment-
|
||||||
this.limiter = new RateLimiter({
|
this.limiter = pThrottle({
|
||||||
tokensPerInterval: 100,
|
// Setting the limit to 1 allows for a short pause between requests that is almost imperceptible to
|
||||||
fireImmediately: true,
|
// the end user. A larger limit will grab blocks faster and then wait for the alloted time(interval) before
|
||||||
interval: 'minute',
|
// fetching another batch.
|
||||||
|
limit: 1,
|
||||||
|
strict: true,
|
||||||
|
interval: 300
|
||||||
})
|
})
|
||||||
|
|
||||||
Audnexus._instance = this
|
Audnexus._instance = this
|
||||||
@ -31,8 +35,8 @@ class Audnexus {
|
|||||||
|
|
||||||
Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
|
Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
|
||||||
|
|
||||||
return this._processRequest(() => axios.get(authorRequestUrl))
|
const throttle = this.limiter(() => axios.get(authorRequestUrl))
|
||||||
.then((res) => res.data || [])
|
return throttle().then((res) => res.data || [])
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error)
|
Logger.error(`[Audnexus] Author ASIN request failed for ${name}`, error)
|
||||||
return []
|
return []
|
||||||
@ -46,36 +50,14 @@ class Audnexus {
|
|||||||
|
|
||||||
Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
|
Logger.info(`[Audnexus] Searching for author "${authorRequestUrl}"`)
|
||||||
|
|
||||||
return this._processRequest(() => axios.get(authorRequestUrl))
|
const throttle = this.limiter(() => axios.get(authorRequestUrl))
|
||||||
.then((res) => res.data)
|
return throttle().then((res) => res.data)
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
Logger.error(`[Audnexus] Author request failed for ${asin}`, error)
|
Logger.error(`[Audnexus] Author request failed for ${asin}`, error)
|
||||||
return null
|
return null
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @description Process a request with a rate limiter
|
|
||||||
*
|
|
||||||
* @param {*} request
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
async _processRequest(request) {
|
|
||||||
const remainingTokens = await this.limiter.removeTokens(1)
|
|
||||||
Logger.info(`[Audnexus] Attempting request with ${remainingTokens} remaining tokens and ${this.limiter.tokensThisInterval} this interval`)
|
|
||||||
|
|
||||||
if (remainingTokens >= 1) {
|
|
||||||
return request()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 100 tokens(requests) per minute give a refresh of ~1.67 per second,
|
|
||||||
// so a 10 second wait will yield ~16.7 additional tokens
|
|
||||||
Logger.info('[Audnexus] Sleeping for 10 seconds')
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 10000))
|
|
||||||
|
|
||||||
return this._processRequest(request)
|
|
||||||
}
|
|
||||||
|
|
||||||
async findAuthorByASIN(asin, region) {
|
async findAuthorByASIN(asin, region) {
|
||||||
const author = await this.authorRequest(asin, region)
|
const author = await this.authorRequest(asin, region)
|
||||||
|
|
||||||
@ -111,8 +93,8 @@ class Audnexus {
|
|||||||
getChaptersByASIN(asin, region) {
|
getChaptersByASIN(asin, region) {
|
||||||
Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`)
|
Logger.debug(`[Audnexus] Get chapters for ASIN ${asin}/${region}`)
|
||||||
|
|
||||||
return this._processRequest(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`))
|
const throttle = this.limiter(() => axios.get(`${this.baseUrl}/books/${asin}/chapters?region=${region}`))
|
||||||
.then((res) => res.data)
|
return throttle().then((res) => res.data)
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error)
|
Logger.error(`[Audnexus] Chapter ASIN request failed for ${asin}/${region}`, error)
|
||||||
return null
|
return null
|
||||||
|
Loading…
Reference in New Issue
Block a user