extern/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-03-14 15:18:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update:Watcher pending delay to 10s. Increase file mtime check interval to 3s and timeout to 600s. Remove file from pending scan if it times out.

Browse Source
This commit is contained in:
advplyr 2024-06-14 16:50:09 -05:00
parent d6438590d7
commit b89bbd2187
1 changed files with 70 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
server/Watcher.js
View File

@ -23,7 +23,7 @@ class FolderWatcher extends EventEmitter {
this.libraryWatchers = []
/** @type {PendingFileUpdate[]} */
this.pendingFileUpdates = []
this.pendingDelay = 4000
this.pendingDelay = 10000
/** @type {NodeJS.Timeout} */
this.pendingTimeout = null
/** @type {Task} */
@ -42,11 +42,11 @@ class FolderWatcher extends EventEmitter {
}
get pendingFilePaths() {
return this.pendingFileUpdates.map(f => f.path)
return this.pendingFileUpdates.map((f) => f.path)
}
buildLibraryWatcher(library) {
if (this.libraryWatchers.find(w => w.id === library.id)) {
if (this.libraryWatchers.find((w) => w.id === library.id)) {
Logger.warn('[Watcher] Already watching library', library.name)
return
}
@ -67,17 +67,23 @@ class FolderWatcher extends EventEmitter {
watcher
.on('add', (path) => {
this.onFileAdded(library.id, filePathToPOSIX(path))
}).on('change', (path) => {
})
.on('change', (path) => {
// This is triggered from metadata changes, not what we want
}).on('unlink', path => {
})
.on('unlink', (path) => {
this.onFileRemoved(library.id, filePathToPOSIX(path))
}).on('rename', (path, pathNext) => {
})
.on('rename', (path, pathNext) => {
this.onFileRename(library.id, filePathToPOSIX(path), filePathToPOSIX(pathNext))
}).on('error', (error) => {
})
.on('error', (error) => {
Logger.error(`[Watcher] ${error}`)
}).on('ready', () => {
})
.on('ready', () => {
Logger.info(`[Watcher] "${library.name}" Ready`)
}).on('close', () => {
})
.on('close', () => {
Logger.debug(`[Watcher] "${library.name}" Closed`)
})
@ -104,32 +110,32 @@ class FolderWatcher extends EventEmitter {
}
/**
*
* @param {import('./objects/Library')} library
*
* @param {import('./objects/Library')} library
*/
updateLibrary(library) {
if (this.disabled) return
const libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
const libwatcher = this.libraryWatchers.find((lib) => lib.id === library.id)
if (libwatcher) {
// Library watcher was disabled
if (library.settings.disableWatcher) {
Logger.info(`[Watcher] updateLibrary: Library "${library.name}" watcher disabled`)
libwatcher.watcher.close()
this.libraryWatchers = this.libraryWatchers.filter(lw => lw.id !== libwatcher.id)
this.libraryWatchers = this.libraryWatchers.filter((lw) => lw.id !== libwatcher.id)
return
}
libwatcher.name = library.name
// If any folder paths were added or removed then re-init watcher
const pathsToAdd = library.folderPaths.filter(path => !libwatcher.paths.includes(path))
const pathsRemoved = libwatcher.paths.filter(path => !library.folderPaths.includes(path))
const pathsToAdd = library.folderPaths.filter((path) => !libwatcher.paths.includes(path))
const pathsRemoved = libwatcher.paths.filter((path) => !library.folderPaths.includes(path))
if (pathsToAdd.length || pathsRemoved.length) {
Logger.info(`[Watcher] Re-Initializing watcher for "${library.name}".`)
libwatcher.watcher.close()
this.libraryWatchers = this.libraryWatchers.filter(lw => lw.id !== libwatcher.id)
this.libraryWatchers = this.libraryWatchers.filter((lw) => lw.id !== libwatcher.id)
this.buildLibraryWatcher(library)
}
} else if (!library.settings.disableWatcher) {
@ -141,25 +147,25 @@ class FolderWatcher extends EventEmitter {
removeLibrary(library) {
if (this.disabled) return
var libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
var libwatcher = this.libraryWatchers.find((lib) => lib.id === library.id)
if (libwatcher) {
Logger.info(`[Watcher] Removed watcher for "${library.name}"`)
libwatcher.watcher.close()
this.libraryWatchers = this.libraryWatchers.filter(lib => lib.id !== library.id)
this.libraryWatchers = this.libraryWatchers.filter((lib) => lib.id !== library.id)
} else {
Logger.error(`[Watcher] Library watcher not found for "${library.name}"`)
}
}
close() {
return this.libraryWatchers.map(lib => lib.watcher.close())
return this.libraryWatchers.map((lib) => lib.watcher.close())
}
/**
* Watcher detected file added
*
* @param {string} libraryId
* @param {string} path
*
* @param {string} libraryId
* @param {string} path
*/
onFileAdded(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
@ -176,9 +182,9 @@ class FolderWatcher extends EventEmitter {
/**
* Watcher detected file removed
*
* @param {string} libraryId
* @param {string} path
*
* @param {string} libraryId
* @param {string} path
*/
onFileRemoved(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
@ -190,9 +196,9 @@ class FolderWatcher extends EventEmitter {
/**
* Watcher detected file renamed
*
* @param {string} libraryId
* @param {string} path
*
* @param {string} libraryId
* @param {string} path
*/
onFileRename(libraryId, pathFrom, pathTo) {
if (this.checkShouldIgnorePath(pathTo)) {
@ -203,17 +209,18 @@ class FolderWatcher extends EventEmitter {
}
/**
* Get mtimeMs from an added file every second until it is no longer changing
* Times out after 180s
*
* @param {string} path
* @param {number} [lastMTimeMs=0]
* @param {number} [loop=0]
* Get mtimeMs from an added file every 3 seconds until it is no longer changing
* Times out after 600s
*
* @param {string} path
* @param {number} [lastMTimeMs=0]
* @param {number} [loop=0]
*/
async waitForFileToAdd(path, lastMTimeMs = 0, loop = 0) {
// Safety to catch infinite loop (180s)
if (loop >= 180) {
Logger.warn(`[Watcher] Waiting to add file at "${path}" timeout (loop ${loop}) - proceeding`)
// Safety to catch infinite loop (600s)
if (loop >= 200) {
Logger.warn(`[Watcher] Waiting to add file at "${path}" timeout (loop ${loop}) - bailing`)
this.pendingFileUpdates = this.pendingFileUpdates.filter((pfu) => pfu.path !== path)
return this.filesBeingAdded.delete(path)
}
@ -222,33 +229,33 @@ class FolderWatcher extends EventEmitter {
if (lastMTimeMs) Logger.debug(`[Watcher] File finished adding at "${path}"`)
return this.filesBeingAdded.delete(path)
}
if (lastMTimeMs % 5 === 0) {
if (loop % 5 === 0) {
Logger.debug(`[Watcher] Waiting to add file at "${path}". mtimeMs=${mtimeMs} lastMTimeMs=${lastMTimeMs} (loop ${loop})`)
}
// Wait 1 second
await new Promise((resolve) => setTimeout(resolve, 1000))
// Wait 3 seconds
await new Promise((resolve) => setTimeout(resolve, 3000))
this.waitForFileToAdd(path, mtimeMs, ++loop)
}
/**
* Queue file update
*
* @param {string} libraryId
* @param {string} path
* @param {string} type
*
* @param {string} libraryId
* @param {string} path
* @param {string} type
*/
addFileUpdate(libraryId, path, type) {
if (this.pendingFilePaths.includes(path)) return
// Get file library
const libwatcher = this.libraryWatchers.find(lw => lw.id === libraryId)
const libwatcher = this.libraryWatchers.find((lw) => lw.id === libraryId)
if (!libwatcher) {
Logger.error(`[Watcher] Invalid library id from watcher ${libraryId}`)
return
}
// Get file folder
const folder = libwatcher.folders.find(fold => isSameOrSubPath(fold.fullPath, path))
const folder = libwatcher.folders.find((fold) => isSameOrSubPath(fold.fullPath, path))
if (!folder) {
Logger.error(`[Watcher] New file folder not found in library "${libwatcher.name}" with path "${path}"`)
return
@ -264,7 +271,7 @@ class FolderWatcher extends EventEmitter {
}
// Ignore files/folders starting with "."
const hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
const hasDotPath = relPath.split('/').find((p) => p.startsWith('.'))
if (hasDotPath) {
Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`)
return
@ -298,12 +305,17 @@ class FolderWatcher extends EventEmitter {
clearTimeout(this.pendingTimeout)
this.pendingTimeout = setTimeout(() => {
// Check that files are not still being added
if (this.pendingFileUpdates.some(pfu => this.filesBeingAdded.has(pfu.path))) {
if (this.pendingFileUpdates.some((pfu) => this.filesBeingAdded.has(pfu.path))) {
Logger.debug(`[Watcher] Still waiting for pending files "${[...this.filesBeingAdded].join(', ')}"`)
return this.handlePendingFileUpdatesTimeout()
}
LibraryScanner.scanFilesChanged(this.pendingFileUpdates, this.pendingTask)
if (this.pendingFileUpdates.length) {
LibraryScanner.scanFilesChanged(this.pendingFileUpdates, this.pendingTask)
} else {
this.pendingTask.setFinished('Scan abandoned. No files to scan.')
TaskManager.taskFinished(this.pendingTask)
}
this.pendingTask = null
this.pendingFileUpdates = []
this.filesBeingAdded.clear()
@ -311,14 +323,14 @@ class FolderWatcher extends EventEmitter {
}
checkShouldIgnorePath(path) {
return !!this.ignoreDirs.find(dirpath => {
return !!this.ignoreDirs.find((dirpath) => {
return isSameOrSubPath(dirpath, path)
})
}
/**
* Convert to POSIX and remove trailing slash
* @param {string} path
* @param {string} path
* @returns {string}
*/
cleanDirPath(path) {
@ -329,11 +341,11 @@ class FolderWatcher extends EventEmitter {
/**
* Ignore this directory if files are picked up by watcher
* @param {string} path
* @param {string} path
*/
addIgnoreDir(path) {
path = this.cleanDirPath(path)
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter((p) => p !== path)
if (this.ignoreDirs.includes(path)) {
// Already ignoring dir
return
@ -346,8 +358,8 @@ class FolderWatcher extends EventEmitter {
* When downloading a podcast episode we dont want the scanner triggering for that podcast
* when the episode finishes the watcher may have a delayed response so a timeout is added
* to prevent the watcher from picking up the episode
*
* @param {string} path
*
* @param {string} path
*/
removeIgnoreDir(path) {
path = this.cleanDirPath(path)
@ -364,11 +376,11 @@ class FolderWatcher extends EventEmitter {
clearTimeout(this.removeFromIgnoreTimer)
this.removeFromIgnoreTimer = setTimeout(() => {
if (this.pendingDirsToRemoveFromIgnore.includes(path)) {
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter((p) => p !== path)
Logger.debug(`[Watcher] removeIgnoreDir: No longer ignoring directory "${path}"`)
this.ignoreDirs = this.ignoreDirs.filter(p => p !== path)
this.ignoreDirs = this.ignoreDirs.filter((p) => p !== path)
}
}, 5000)
}
}
module.exports = FolderWatcher
module.exports = FolderWatcher