audiobookshelf/server/Watcher.js

357 lines
11 KiB
JavaScript

const Path = require('path')
const EventEmitter = require('events')
const Watcher = require('./libs/watcher/watcher')
const Logger = require('./Logger')
const LibraryScanner = require('./scanner/LibraryScanner')
const Task = require('./objects/Task')
const TaskManager = require('./managers/TaskManager')
const { filePathToPOSIX, isSameOrSubPath, getFileMTimeMs } = require('./utils/fileUtils')
/**
* @typedef PendingFileUpdate
* @property {string} path
* @property {string} relPath
* @property {string} folderId
* @property {string} type
*/
class FolderWatcher extends EventEmitter {
constructor() {
super()
/** @type {{id:string, name:string, folders:import('./objects/Folder')[], paths:string[], watcher:Watcher[]}[]} */
this.libraryWatchers = []
/** @type {PendingFileUpdate[]} */
this.pendingFileUpdates = []
this.pendingDelay = 4000
/** @type {NodeJS.Timeout} */
this.pendingTimeout = null
/** @type {Task} */
this.pendingTask = null
this.filesBeingAdded = new Set()
/** @type {string[]} */
this.ignoreDirs = []
/** @type {string[]} */
this.pendingDirsToRemoveFromIgnore = []
/** @type {NodeJS.Timeout} */
this.removeFromIgnoreTimer = null
this.disabled = false
}
get pendingFilePaths() {
return this.pendingFileUpdates.map(f => f.path)
}
buildLibraryWatcher(library) {
if (this.libraryWatchers.find(w => w.id === library.id)) {
Logger.warn('[Watcher] Already watching library', library.name)
return
}
Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
const folderPaths = library.folderPaths
folderPaths.forEach((fp) => {
Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
})
const watcher = new Watcher(folderPaths, {
ignored: /(^|[\/\\])\../, // ignore dotfiles
renameDetection: true,
renameTimeout: 2000,
recursive: true,
ignoreInitial: true,
persistent: true
})
watcher
.on('add', (path) => {
this.onFileAdded(library.id, filePathToPOSIX(path))
}).on('change', (path) => {
// This is triggered from metadata changes, not what we want
}).on('unlink', path => {
this.onFileRemoved(library.id, filePathToPOSIX(path))
}).on('rename', (path, pathNext) => {
this.onFileRename(library.id, filePathToPOSIX(path), filePathToPOSIX(pathNext))
}).on('error', (error) => {
Logger.error(`[Watcher] ${error}`)
}).on('ready', () => {
Logger.info(`[Watcher] "${library.name}" Ready`)
}).on('close', () => {
Logger.debug(`[Watcher] "${library.name}" Closed`)
})
this.libraryWatchers.push({
id: library.id,
name: library.name,
folders: library.folders,
paths: library.folderPaths,
watcher
})
}
initWatcher(libraries) {
libraries.forEach((lib) => {
if (!lib.settings.disableWatcher) {
this.buildLibraryWatcher(lib)
}
})
}
addLibrary(library) {
if (this.disabled || library.settings.disableWatcher) return
this.buildLibraryWatcher(library)
}
updateLibrary(library) {
if (this.disabled || library.settings.disableWatcher) return
var libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
if (libwatcher) {
libwatcher.name = library.name
// If any folder paths were added or removed then re-init watcher
var pathsToAdd = library.folderPaths.filter(path => !libwatcher.paths.includes(path))
var pathsRemoved = libwatcher.paths.filter(path => !library.folderPaths.includes(path))
if (pathsToAdd.length || pathsRemoved.length) {
Logger.info(`[Watcher] Re-Initializing watcher for "${library.name}".`)
libwatcher.watcher.close()
this.libraryWatchers = this.libraryWatchers.filter(lw => lw.id !== libwatcher.id)
this.buildLibraryWatcher(library)
}
}
}
removeLibrary(library) {
if (this.disabled) return
var libwatcher = this.libraryWatchers.find(lib => lib.id === library.id)
if (libwatcher) {
Logger.info(`[Watcher] Removed watcher for "${library.name}"`)
libwatcher.watcher.close()
this.libraryWatchers = this.libraryWatchers.filter(lib => lib.id !== library.id)
} else {
Logger.error(`[Watcher] Library watcher not found for "${library.name}"`)
}
}
close() {
return this.libraryWatchers.map(lib => lib.watcher.close())
}
/**
* Watcher detected file added
*
* @param {string} libraryId
* @param {string} path
*/
onFileAdded(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
return
}
Logger.debug('[Watcher] File Added', path)
this.addFileUpdate(libraryId, path, 'added')
if (!this.filesBeingAdded.has(path)) {
this.filesBeingAdded.add(path)
this.waitForFileToAdd(path)
}
}
/**
* Watcher detected file removed
*
* @param {string} libraryId
* @param {string} path
*/
onFileRemoved(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
return
}
Logger.debug('[Watcher] File Removed', path)
this.addFileUpdate(libraryId, path, 'deleted')
}
/**
* Watcher detected file renamed
*
* @param {string} libraryId
* @param {string} path
*/
onFileRename(libraryId, pathFrom, pathTo) {
if (this.checkShouldIgnorePath(pathTo)) {
return
}
Logger.debug(`[Watcher] Rename ${pathFrom} => ${pathTo}`)
this.addFileUpdate(libraryId, pathTo, 'renamed')
}
/**
* Get mtimeMs from an added file every second until it is no longer changing
* Times out after 180s
*
* @param {string} path
* @param {number} [lastMTimeMs=0]
* @param {number} [loop=0]
*/
async waitForFileToAdd(path, lastMTimeMs = 0, loop = 0) {
// Safety to catch infinite loop (180s)
if (loop >= 180) {
Logger.warn(`[Watcher] Waiting to add file at "${path}" timeout (loop ${loop}) - proceeding`)
return this.filesBeingAdded.delete(path)
}
const mtimeMs = await getFileMTimeMs(path)
if (mtimeMs === lastMTimeMs) {
if (lastMTimeMs) Logger.debug(`[Watcher] File finished adding at "${path}"`)
return this.filesBeingAdded.delete(path)
}
if (lastMTimeMs % 5 === 0) {
Logger.debug(`[Watcher] Waiting to add file at "${path}". mtimeMs=${mtimeMs} lastMTimeMs=${lastMTimeMs} (loop ${loop})`)
}
// Wait 1 second
await new Promise((resolve) => setTimeout(resolve, 1000))
this.waitForFileToAdd(path, mtimeMs, ++loop)
}
/**
* Queue file update
*
* @param {string} libraryId
* @param {string} path
* @param {string} type
*/
addFileUpdate(libraryId, path, type) {
if (this.pendingFilePaths.includes(path)) return
// Get file library
const libwatcher = this.libraryWatchers.find(lw => lw.id === libraryId)
if (!libwatcher) {
Logger.error(`[Watcher] Invalid library id from watcher ${libraryId}`)
return
}
// Get file folder
const folder = libwatcher.folders.find(fold => isSameOrSubPath(fold.fullPath, path))
if (!folder) {
Logger.error(`[Watcher] New file folder not found in library "${libwatcher.name}" with path "${path}"`)
return
}
const folderFullPath = filePathToPOSIX(folder.fullPath)
const relPath = path.replace(folderFullPath, '')
if (Path.extname(relPath).toLowerCase() === '.part') {
Logger.debug(`[Watcher] Ignoring .part file "${relPath}"`)
return
}
// Ignore files/folders starting with "."
const hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
if (hasDotPath) {
Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`)
return
}
Logger.debug(`[Watcher] Modified file in library "${libwatcher.name}" and folder "${folder.id}" with relPath "${relPath}"`)
if (!this.pendingTask) {
const taskData = {
libraryId,
libraryName: libwatcher.name
}
this.pendingTask = TaskManager.createAndAddTask('watcher-scan', `Scanning file changes in "${libwatcher.name}"`, null, true, taskData)
}
this.pendingFileUpdates.push({
path,
relPath,
folderId: folder.id,
libraryId,
type
})
this.handlePendingFileUpdatesTimeout()
}
/**
* Wait X seconds before notifying scanner that files changed
* reset timer if files are still copying
*/
handlePendingFileUpdatesTimeout() {
clearTimeout(this.pendingTimeout)
this.pendingTimeout = setTimeout(() => {
// Check that files are not still being added
if (this.pendingFileUpdates.some(pfu => this.filesBeingAdded.has(pfu.path))) {
Logger.debug(`[Watcher] Still waiting for pending files "${[...this.filesBeingAdded].join(', ')}"`)
return this.handlePendingFileUpdatesTimeout()
}
LibraryScanner.scanFilesChanged(this.pendingFileUpdates, this.pendingTask)
this.pendingTask = null
this.pendingFileUpdates = []
this.filesBeingAdded.clear()
}, this.pendingDelay)
}
checkShouldIgnorePath(path) {
return !!this.ignoreDirs.find(dirpath => {
return isSameOrSubPath(dirpath, path)
})
}
/**
* Convert to POSIX and remove trailing slash
* @param {string} path
* @returns {string}
*/
cleanDirPath(path) {
path = filePathToPOSIX(path)
if (path.endsWith('/')) path = path.slice(0, -1)
return path
}
/**
* Ignore this directory if files are picked up by watcher
* @param {string} path
*/
addIgnoreDir(path) {
path = this.cleanDirPath(path)
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
if (this.ignoreDirs.includes(path)) {
// Already ignoring dir
return
}
Logger.debug(`[Watcher] addIgnoreDir: Ignoring directory "${path}"`)
this.ignoreDirs.push(path)
}
/**
* When downloading a podcast episode we dont want the scanner triggering for that podcast
* when the episode finishes the watcher may have a delayed response so a timeout is added
* to prevent the watcher from picking up the episode
*
* @param {string} path
*/
removeIgnoreDir(path) {
path = this.cleanDirPath(path)
if (!this.ignoreDirs.includes(path)) {
Logger.debug(`[Watcher] removeIgnoreDir: Path is not being ignored "${path}"`)
return
}
// Add a 5 second delay before removing the ignore from this dir
if (!this.pendingDirsToRemoveFromIgnore.includes(path)) {
this.pendingDirsToRemoveFromIgnore.push(path)
}
clearTimeout(this.removeFromIgnoreTimer)
this.removeFromIgnoreTimer = setTimeout(() => {
if (this.pendingDirsToRemoveFromIgnore.includes(path)) {
this.pendingDirsToRemoveFromIgnore = this.pendingDirsToRemoveFromIgnore.filter(p => p !== path)
Logger.debug(`[Watcher] removeIgnoreDir: No longer ignoring directory "${path}"`)
this.ignoreDirs = this.ignoreDirs.filter(p => p !== path)
}
}, 5000)
}
}
module.exports = FolderWatcher