diff --git a/server/Server.js b/server/Server.js index ec6eb66c..9bb22b39 100644 --- a/server/Server.js +++ b/server/Server.js @@ -35,6 +35,7 @@ const AudioMetadataMangaer = require('./managers/AudioMetadataManager') const RssFeedManager = require('./managers/RssFeedManager') const CronManager = require('./managers/CronManager') const TaskManager = require('./managers/TaskManager') +const LibraryScanner = require('./scanner/LibraryScanner') class Server { constructor(SOURCE, PORT, HOST, UID, GID, CONFIG_PATH, METADATA_PATH, ROUTER_BASE_PATH) { @@ -76,7 +77,7 @@ class Server { this.rssFeedManager = new RssFeedManager() this.scanner = new Scanner(this.coverManager, this.taskManager) - this.cronManager = new CronManager(this.scanner, this.podcastManager) + this.cronManager = new CronManager(this.podcastManager) // Routers this.apiRouter = new ApiRouter(this) @@ -92,6 +93,10 @@ class Server { this.auth.authMiddleware(req, res, next) } + cancelLibraryScan(libraryId) { + LibraryScanner.setCancelLibraryScan(libraryId) + } + /** * Initialize database, backups, logs, rss feeds, cron jobs & watcher * Cleanup stale/invalid data @@ -122,7 +127,6 @@ class Server { this.watcher.disabled = true } else { this.watcher.initWatcher(libraries) - this.watcher.on('files', this.filesChanged.bind(this)) } } @@ -241,11 +245,6 @@ class Server { res.sendStatus(200) } - async filesChanged(fileUpdates) { - Logger.info('[Server]', fileUpdates.length, 'Files Changed') - await this.scanner.scanFilesChanged(fileUpdates) - } - /** * Remove user media progress for items that no longer exist & remove seriesHideFrom that no longer exist */ diff --git a/server/SocketAuthority.js b/server/SocketAuthority.js index 14a1ddaa..f7ab98e1 100644 --- a/server/SocketAuthority.js +++ b/server/SocketAuthority.js @@ -95,7 +95,7 @@ class SocketAuthority { socket.on('auth', (token) => this.authenticateSocket(socket, token)) // Scanning - socket.on('cancel_scan', this.cancelScan.bind(this)) + socket.on('cancel_scan', (libraryId) => this.cancelScan(libraryId)) // Logs socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level)) @@ -209,7 +209,7 @@ class SocketAuthority { cancelScan(id) { Logger.debug('[SocketAuthority] Cancel scan', id) - this.Server.scanner.setCancelLibraryScan(id) + this.Server.cancelLibraryScan(id) } } module.exports = new SocketAuthority() \ No newline at end of file diff --git a/server/Watcher.js b/server/Watcher.js index 7d62296b..6c17b90e 100644 --- a/server/Watcher.js +++ b/server/Watcher.js @@ -1,20 +1,29 @@ const EventEmitter = require('events') const Watcher = require('./libs/watcher/watcher') const Logger = require('./Logger') +const LibraryScanner = require('./scanner/LibraryScanner') const { filePathToPOSIX } = require('./utils/fileUtils') +/** + * @typedef PendingFileUpdate + * @property {string} path + * @property {string} relPath + * @property {string} folderId + * @property {string} type + */ class FolderWatcher extends EventEmitter { constructor() { super() - this.paths = [] // Not used - this.pendingFiles = [] // Not used + /** @type {{id:string, name:string, folders:import('./objects/Folder')[], paths:string[], watcher:Watcher[]}[]} */ this.libraryWatchers = [] + /** @type {PendingFileUpdate[]} */ this.pendingFileUpdates = [] this.pendingDelay = 4000 this.pendingTimeout = null + /** @type {string[]} */ this.ignoreDirs = [] this.disabled = false } @@ -29,11 +38,12 @@ class FolderWatcher extends EventEmitter { return } Logger.info(`[Watcher] Initializing watcher for "${library.name}".`) - var folderPaths = library.folderPaths + + const folderPaths = library.folderPaths folderPaths.forEach((fp) => { Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`) }) - var watcher = new Watcher(folderPaths, { + const watcher = new Watcher(folderPaths, { ignored: /(^|[\/\\])\../, // ignore dotfiles renameDetection: true, renameTimeout: 2000, @@ -144,6 +154,12 @@ class FolderWatcher extends EventEmitter { this.addFileUpdate(libraryId, pathTo, 'renamed') } + /** + * File update detected from watcher + * @param {string} libraryId + * @param {string} path + * @param {string} type + */ addFileUpdate(libraryId, path, type) { path = filePathToPOSIX(path) if (this.pendingFilePaths.includes(path)) return @@ -163,9 +179,10 @@ class FolderWatcher extends EventEmitter { } const folderFullPath = filePathToPOSIX(folder.fullPath) - var relPath = path.replace(folderFullPath, '') + const relPath = path.replace(folderFullPath, '') - var hasDotPath = relPath.split('/').find(p => p.startsWith('.')) + // Ignore files/folders starting with "." + const hasDotPath = relPath.split('/').find(p => p.startsWith('.')) if (hasDotPath) { Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`) return @@ -184,7 +201,8 @@ class FolderWatcher extends EventEmitter { // Notify server of update after "pendingDelay" clearTimeout(this.pendingTimeout) this.pendingTimeout = setTimeout(() => { - this.emit('files', this.pendingFileUpdates) + // this.emit('files', this.pendingFileUpdates) + LibraryScanner.scanFilesChanged(this.pendingFileUpdates) this.pendingFileUpdates = [] }, this.pendingDelay) } diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index bbd5536d..7a0d1710 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -978,12 +978,8 @@ class LibraryController { forceRescan: req.query.force == 1 } res.sendStatus(200) - if (req.library.mediaType === 'podcast') { - // TODO: New library scanner for podcasts - await this.scanner.scan(req.library, options) - } else { - await LibraryScanner.scan(req.library, options) - } + + await LibraryScanner.scan(req.library, options) await Database.resetLibraryIssuesFilterData(req.library.id) Logger.info('[LibraryController] Scan complete') diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js index 761fc6ed..1520bf03 100644 --- a/server/controllers/LibraryItemController.js +++ b/server/controllers/LibraryItemController.js @@ -445,7 +445,12 @@ class LibraryItemController { return res.sendStatus(400) } - const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li) + const libraryItems = await Database.libraryItemModel.findAll({ + where: { + id: req.body.libraryItemIds + }, + attributes: ['id', 'libraryId', 'isFile'] + }) if (!libraryItems?.length) { return res.sendStatus(400) } @@ -457,7 +462,7 @@ class LibraryItemController { if (libraryItem.isFile) { Logger.warn(`[LibraryItemController] Re-scanning file library items not yet supported`) } else { - await this.scanner.scanLibraryItemByRequest(libraryItem) + await LibraryItemScanner.scanLibraryItem(libraryItem.id) } } @@ -476,14 +481,7 @@ class LibraryItemController { return res.sendStatus(500) } - let result = 0 - if (req.libraryItem.isPodcast) { - // TODO: New library item scanner for podcast - result = await this.scanner.scanLibraryItemByRequest(req.libraryItem) - } else { - result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id) - } - + const result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id) await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId) res.json({ result: Object.keys(ScanResult).find(key => ScanResult[key] == result) diff --git a/server/managers/CronManager.js b/server/managers/CronManager.js index 8b2b1849..c44ad70d 100644 --- a/server/managers/CronManager.js +++ b/server/managers/CronManager.js @@ -2,10 +2,10 @@ const Sequelize = require('sequelize') const cron = require('../libs/nodeCron') const Logger = require('../Logger') const Database = require('../Database') +const LibraryScanner = require('../scanner/LibraryScanner') class CronManager { - constructor(scanner, podcastManager) { - this.scanner = scanner + constructor(podcastManager) { this.podcastManager = podcastManager this.libraryScanCrons = [] @@ -39,7 +39,7 @@ class CronManager { Logger.debug(`[CronManager] Init library scan cron for ${library.name} on schedule ${library.settings.autoScanCronExpression}`) const libScanCron = cron.schedule(library.settings.autoScanCronExpression, () => { Logger.debug(`[CronManager] Library scan cron executing for ${library.name}`) - this.scanner.scan(library) + LibraryScanner.scan(library) }) this.libraryScanCrons.push({ libraryId: library.id, diff --git a/server/models/PodcastEpisode.js b/server/models/PodcastEpisode.js index 31fda60f..6416627a 100644 --- a/server/models/PodcastEpisode.js +++ b/server/models/PodcastEpisode.js @@ -39,7 +39,7 @@ class PodcastEpisode extends Model { this.enclosureType /** @type {Date} */ this.publishedAt - /** @type {Object} */ + /** @type {import('./Book').AudioFileObject} */ this.audioFile /** @type {ChapterObject[]} */ this.chapters diff --git a/server/objects/files/AudioFile.js b/server/objects/files/AudioFile.js index 0de03606..7ffff19c 100644 --- a/server/objects/files/AudioFile.js +++ b/server/objects/files/AudioFile.js @@ -6,6 +6,7 @@ class AudioFile { constructor(data) { this.index = null this.ino = null + /** @type {FileMetadata} */ this.metadata = null this.addedAt = null this.updatedAt = null @@ -27,6 +28,7 @@ class AudioFile { this.embeddedCoverArt = null // Tags scraped from the audio file + /** @type {AudioMetaTags} */ this.metaTags = null this.manuallyVerified = false diff --git a/server/objects/mediaTypes/Podcast.js b/server/objects/mediaTypes/Podcast.js index 808bfe32..ae33cd81 100644 --- a/server/objects/mediaTypes/Podcast.js +++ b/server/objects/mediaTypes/Podcast.js @@ -4,10 +4,6 @@ const PodcastMetadata = require('../metadata/PodcastMetadata') const { areEquivalent, copyValue, cleanStringForSearch } = require('../../utils/index') const abmetadataGenerator = require('../../utils/generators/abmetadataGenerator') const { readTextFile, filePathToPOSIX } = require('../../utils/fileUtils') -const { createNewSortInstance } = require('../../libs/fastSort') -const naturalSort = createNewSortInstance({ - comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare -}) class Podcast { constructor(podcast) { diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index b23a92be..b16c799c 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -360,7 +360,7 @@ class BookScanner { * @param {import('./LibraryItemScanData')} libraryItemData * @param {import('../models/Library').LibrarySettingsObject} librarySettings * @param {import('./LibraryScan')} libraryScan - * @returns {import('../models/LibraryItem')} + * @returns {Promise} */ async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) { // Scan audio files found @@ -439,6 +439,7 @@ class BookScanner { libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image libraryItemObj.isMissing = false libraryItemObj.isInvalid = false + libraryItemObj.extraData = {} // Set isSupplementary flag on ebook library files for (const libraryFile of libraryItemObj.libraryFiles) { @@ -739,6 +740,8 @@ class BookScanner { bookMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path } + bookMetadata.titleIgnorePrefix = getTitleIgnorePrefix(bookMetadata.title) + return bookMetadata } diff --git a/server/scanner/LibraryItemScanner.js b/server/scanner/LibraryItemScanner.js index f71e49a6..271ad256 100644 --- a/server/scanner/LibraryItemScanner.js +++ b/server/scanner/LibraryItemScanner.js @@ -8,6 +8,7 @@ const Database = require('../Database') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') const BookScanner = require('./BookScanner') +const PodcastScanner = require('./PodcastScanner') const ScanLogger = require('./ScanLogger') const LibraryItem = require('../models/LibraryItem') const LibraryFile = require('../objects/files/LibraryFile') @@ -158,13 +159,13 @@ class LibraryItemScanner { * @returns {Promise} */ async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) { + let newLibraryItem = null if (existingLibraryItem.mediaType === 'book') { - const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) - return libraryItem + newLibraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) } else { - // TODO: Scan updated podcast - return null + newLibraryItem = await PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) } + return newLibraryItem } /** @@ -175,16 +176,34 @@ class LibraryItemScanner { * @returns {Promise} */ async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) { - if (libraryScan.libraryMediaType === 'book') { - const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) - if (newLibraryItem) { - libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`) - } - return newLibraryItem + let newLibraryItem = null + if (libraryItemData.mediaType === 'book') { + newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) } else { - // TODO: Scan new podcast - return null + newLibraryItem = await PodcastScanner.scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan) } + if (newLibraryItem) { + libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`) + } + return newLibraryItem + } + + /** + * Scan library item folder coming from Watcher + * @param {string} libraryItemPath + * @param {import('../models/Library')} library + * @param {import('../models/LibraryFolder')} folder + * @param {boolean} isSingleMediaItem + * @returns {Promise} ScanResult + */ + async scanPotentialNewLibraryItem(libraryItemPath, library, folder, isSingleMediaItem) { + const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem) + + const scanLogger = new ScanLogger() + scanLogger.verbose = true + scanLogger.setData('libraryItem', libraryItemScanData.relPath) + + return this.scanNewLibraryItem(libraryItemScanData, library.settings, scanLogger) } } module.exports = new LibraryItemScanner() \ No newline at end of file diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index 1412e211..bfbec4a9 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -7,7 +7,7 @@ const Database = require('../Database') const fs = require('../libs/fsExtra') const fileUtils = require('../utils/fileUtils') const scanUtils = require('../utils/scandir') -const { LogLevel } = require('../utils/constants') +const { LogLevel, ScanResult } = require('../utils/constants') const libraryFilters = require('../utils/queries/libraryFilters') const LibraryItemScanner = require('./LibraryItemScanner') const ScanOptions = require('./ScanOptions') @@ -18,6 +18,10 @@ class LibraryScanner { constructor() { this.cancelLibraryScan = {} this.librariesScanning = [] + + this.scanningFilesChanged = false + /** @type {import('../Watcher').PendingFileUpdate[][]} */ + this.pendingFileUpdatesToScan = [] } /** @@ -28,6 +32,16 @@ class LibraryScanner { return this.librariesScanning.some(ls => ls.id === libraryId) } + /** + * + * @param {string} libraryId + */ + setCancelLibraryScan(libraryId) { + const libraryScanning = this.librariesScanning.find(ls => ls.id === libraryId) + if (!libraryScanning) return + this.cancelLibraryScan[libraryId] = true + } + /** * * @param {import('../objects/Library')} library @@ -290,5 +304,229 @@ class LibraryScanner { } return items } + + /** + * Scan files changed from Watcher + * @param {import('../Watcher').PendingFileUpdate[]} fileUpdates + */ + async scanFilesChanged(fileUpdates) { + if (!fileUpdates?.length) return + + // If already scanning files from watcher then add these updates to queue + if (this.scanningFilesChanged) { + this.pendingFileUpdatesToScan.push(fileUpdates) + Logger.debug(`[LibraryScanner] Already scanning files from watcher - file updates pushed to queue (size ${this.pendingFileUpdatesToScan.length})`) + return + } + this.scanningFilesChanged = true + + // files grouped by folder + const folderGroups = this.getFileUpdatesGrouped(fileUpdates) + + for (const folderId in folderGroups) { + const libraryId = folderGroups[folderId].libraryId + // const library = await Database.libraryModel.getOldById(libraryId) + const library = await Database.libraryModel.findByPk(libraryId, { + include: { + model: Database.libraryFolderModel, + where: { + id: folderId + } + } + }) + if (!library) { + Logger.error(`[LibraryScanner] Library "${libraryId}" not found in files changed ${libraryId}`) + continue + } + const folder = library.libraryFolders[0] + + const relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath) + const fileUpdateGroup = scanUtils.groupFilesIntoLibraryItemPaths(library.mediaType, relFilePaths) + + if (!Object.keys(fileUpdateGroup).length) { + Logger.info(`[LibraryScanner] No important changes to scan for in folder "${folderId}"`) + continue + } + const folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateGroup) + Logger.debug(`[LibraryScanner] Folder scan results`, folderScanResults) + + // If something was updated then reset numIssues filter data for library + if (Object.values(folderScanResults).some(scanResult => scanResult !== ScanResult.NOTHING && scanResult !== ScanResult.UPTODATE)) { + await Database.resetLibraryIssuesFilterData(libraryId) + } + } + + this.scanningFilesChanged = false + + if (this.pendingFileUpdatesToScan.length) { + Logger.debug(`[LibraryScanner] File updates finished scanning with more updates in queue (${this.pendingFileUpdatesToScan.length})`) + this.scanFilesChanged(this.pendingFileUpdatesToScan.shift()) + } + } + + /** + * Group array of PendingFileUpdate from Watcher by folder + * @param {import('../Watcher').PendingFileUpdate[]} fileUpdates + * @returns {Record} + */ + getFileUpdatesGrouped(fileUpdates) { + const folderGroups = {} + fileUpdates.forEach((file) => { + if (folderGroups[file.folderId]) { + folderGroups[file.folderId].fileUpdates.push(file) + } else { + folderGroups[file.folderId] = { + libraryId: file.libraryId, + folderId: file.folderId, + fileUpdates: [file] + } + } + }) + return folderGroups + } + + /** + * Scan grouped paths for library folder coming from Watcher + * @param {import('../models/Library')} library + * @param {import('../models/LibraryFolder')} folder + * @param {Record} fileUpdateGroup + * @returns {Promise>} + */ + async scanFolderUpdates(library, folder, fileUpdateGroup) { + // Make sure library filter data is set + // this is used to check for existing authors & series + await libraryFilters.getFilterData(library.mediaType, library.id) + Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`) + Logger.debug(`[Scanner] scanFolderUpdates fileUpdateGroup`, fileUpdateGroup) + + // First pass - Remove files in parent dirs of items and remap the fileupdate group + // Test Case: Moving audio files from library item folder to author folder should trigger a re-scan of the item + const updateGroup = { ...fileUpdateGroup } + for (const itemDir in updateGroup) { + if (itemDir == fileUpdateGroup[itemDir]) continue // Media in root path + + const itemDirNestedFiles = fileUpdateGroup[itemDir].filter(b => b.includes('/')) + if (!itemDirNestedFiles.length) continue + + const firstNest = itemDirNestedFiles[0].split('/').shift() + const altDir = `${itemDir}/${firstNest}` + + const fullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir) + const childLibraryItem = await Database.libraryItemModel.findOne({ + attributes: ['id', 'path'], + where: { + path: { + [sequelize.Op.not]: fullPath + }, + path: { + [sequelize.Op.startsWith]: fullPath + } + } + }) + if (!childLibraryItem) { + continue + } + + const altFullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), altDir) + const altChildLibraryItem = await Database.libraryItemModel.findOne({ + attributes: ['id', 'path'], + where: { + path: { + [sequelize.Op.not]: altFullPath + }, + path: { + [sequelize.Op.startsWith]: altFullPath + } + } + }) + if (altChildLibraryItem) { + continue + } + + delete fileUpdateGroup[itemDir] + fileUpdateGroup[altDir] = itemDirNestedFiles.map((f) => f.split('/').slice(1).join('/')) + Logger.warn(`[LibraryScanner] Some files were modified in a parent directory of a library item "${childLibraryItem.path}" - ignoring`) + } + + // Second pass: Check for new/updated/removed items + const itemGroupingResults = {} + for (const itemDir in fileUpdateGroup) { + const fullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir) + const dirIno = await fileUtils.getIno(fullPath) + + const itemDirParts = itemDir.split('/').slice(0, -1) + const potentialChildDirs = [] + for (let i = 0; i < itemDirParts.length; i++) { + potentialChildDirs.push(Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir.split('/').slice(0, -1 - i).join('/'))) + } + + // Check if book dir group is already an item + let existingLibraryItem = await Database.libraryItemModel.findOneOld({ + path: potentialChildDirs + }) + + if (!existingLibraryItem) { + existingLibraryItem = await Database.libraryItemModel.findOneOld({ + ino: dirIno + }) + if (existingLibraryItem) { + Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`) + // Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData + existingLibraryItem.path = fullPath + existingLibraryItem.relPath = itemDir + } + } + if (existingLibraryItem) { + // Is the item exactly - check if was deleted + if (existingLibraryItem.path === fullPath) { + const exists = await fs.pathExists(fullPath) + if (!exists) { + Logger.info(`[LibraryScanner] Scanning file update group and library item was deleted "${existingLibraryItem.media.metadata.title}" - marking as missing`) + existingLibraryItem.setMissing() + await Database.updateLibraryItem(existingLibraryItem) + SocketAuthority.emitter('item_updated', existingLibraryItem.toJSONExpanded()) + + itemGroupingResults[itemDir] = ScanResult.REMOVED + continue + } + } + + // Scan library item for updates + Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`) + // itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem) + itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id) + continue + } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(scanUtils.checkFilepathIsAudioFile)) { + Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" has no audio files`) + continue + } + + // Check if a library item is a subdirectory of this dir + const childItem = await Database.libraryItemModel.findOne({ + attributes: ['id', 'path'], + where: { + path: { + [sequelize.Op.startsWith]: fullPath + '/' + } + } + }) + if (childItem) { + Logger.warn(`[LibraryScanner] Files were modified in a parent directory of a library item "${childItem.path}" - ignoring`) + itemGroupingResults[itemDir] = ScanResult.NOTHING + continue + } + + Logger.debug(`[LibraryScanner] Folder update group must be a new item "${itemDir}" in library "${library.name}"`) + const isSingleMediaItem = itemDir === fileUpdateGroup[itemDir] + const newLibraryItem = await LibraryItemScanner.scanPotentialNewLibraryItem(fullPath, library, folder, isSingleMediaItem) + if (newLibraryItem) { + const oldNewLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem) + SocketAuthority.emitter('item_added', oldNewLibraryItem.toJSONExpanded()) + } + itemGroupingResults[itemDir] = newLibraryItem ? ScanResult.ADDED : ScanResult.NOTHING + } + + return itemGroupingResults + } } module.exports = new LibraryScanner() \ No newline at end of file diff --git a/server/scanner/PodcastScanner.js b/server/scanner/PodcastScanner.js new file mode 100644 index 00000000..15236263 --- /dev/null +++ b/server/scanner/PodcastScanner.js @@ -0,0 +1,626 @@ +const uuidv4 = require("uuid").v4 +const Path = require('path') +const { LogLevel } = require('../utils/constants') +const { getTitleIgnorePrefix } = require('../utils/index') +const abmetadataGenerator = require('../utils/generators/abmetadataGenerator') +const AudioFileScanner = require('./AudioFileScanner') +const Database = require('../Database') +const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils') +const AudioFile = require('../objects/files/AudioFile') +const CoverManager = require('../managers/CoverManager') +const LibraryFile = require('../objects/files/LibraryFile') +const fsExtra = require("../libs/fsExtra") +const PodcastEpisode = require("../models/PodcastEpisode") + +/** + * Metadata for podcasts pulled from files + * @typedef PodcastMetadataObject + * @property {string} title + * @property {string} titleIgnorePrefix + * @property {string} author + * @property {string} releaseDate + * @property {string} feedURL + * @property {string} imageURL + * @property {string} description + * @property {string} itunesPageURL + * @property {string} itunesId + * @property {string} language + * @property {string} podcastType + * @property {string[]} genres + * @property {string[]} tags + * @property {boolean} explicit + */ + +class PodcastScanner { + constructor() { } + + /** + * @param {import('../models/LibraryItem')} existingLibraryItem + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) { + /** @type {import('../models/Podcast')} */ + const media = await existingLibraryItem.getMedia({ + include: [ + { + model: Database.podcastEpisodeModel + } + ] + }) + + /** @type {import('../models/PodcastEpisode')[]} */ + let existingPodcastEpisodes = media.podcastEpisodes + + /** @type {AudioFile[]} */ + let newAudioFiles = [] + + if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== existingPodcastEpisodes.length) { + // Filter out and destroy episodes that were removed + existingPodcastEpisodes = await Promise.all(existingPodcastEpisodes.filter(async ep => { + if (libraryItemData.checkAudioFileRemoved(ep.audioFile)) { + libraryScan.addLog(LogLevel.INFO, `Podcast episode "${ep.title}" audio file was removed`) + // TODO: Should clean up other data linked to this episode + await ep.destroy() + return false + } + return true + })) + + // Update audio files that were modified + if (libraryItemData.audioLibraryFilesModified.length) { + let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified) + + for (const podcastEpisode of existingPodcastEpisodes) { + let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === podcastEpisode.audioFile.metadata.path) + if (!matchedScannedAudioFile) { + matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === podcastEpisode.audioFile.ino) + } + + if (matchedScannedAudioFile) { + scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile) + const audioFile = new AudioFile(podcastEpisode.audioFile) + audioFile.updateFromScan(matchedScannedAudioFile) + podcastEpisode.audioFile = audioFile.toJSON() + podcastEpisode.changed('audioFile', true) + + // Set metadata and save episode + this.setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, libraryScan) + libraryScan.addLog(LogLevel.INFO, `Podcast episode "${podcastEpisode.title}" keys changed [${podcastEpisode.changed()?.join(', ')}]`) + await podcastEpisode.save() + } + } + + // Modified audio files that were not found as a podcast episode + if (scannedAudioFiles.length) { + newAudioFiles.push(...scannedAudioFiles) + } + } + + // Add new audio files scanned in + if (libraryItemData.audioLibraryFilesAdded.length) { + const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded) + newAudioFiles.push(...scannedAudioFiles) + } + + // Create new podcast episodes from new found audio files + for (const newAudioFile of newAudioFiles) { + const newEpisode = { + title: newAudioFile.metaTags.tagTitle || newAudioFile.metadata.filenameNoExt, + subtitle: null, + season: null, + episode: null, + episodeType: null, + pubDate: null, + publishedAt: null, + description: null, + audioFile: newAudioFile.toJSON(), + chapters: newAudioFile.chapters || [], + podcastId: media.id + } + const newPodcastEpisode = Database.podcastEpisodeModel.build(newEpisode) + // Set metadata and save new episode + this.setPodcastEpisodeMetadataFromAudioFile(newPodcastEpisode, libraryScan) + libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newPodcastEpisode.title}" added`) + await newPodcastEpisode.save() + existingPodcastEpisodes.push(newPodcastEpisode) + } + } + + let hasMediaChanges = false + + // Check if cover was removed + if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) { + media.coverPath = null + hasMediaChanges = true + } + + // Check if cover is not set and image files were found + if (!media.coverPath && libraryItemData.imageLibraryFiles.length) { + // Prefer using a cover image with the name "cover" otherwise use the first image + const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) + media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path + hasMediaChanges = true + } + + // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this + const podcastMetadata = await this.getPodcastMetadataFromScanData(existingPodcastEpisodes, libraryItemData, libraryScan) + + for (const key in podcastMetadata) { + // Ignore unset metadata and empty arrays + if (podcastMetadata[key] === undefined || (Array.isArray(podcastMetadata[key]) && !podcastMetadata[key].length)) continue + + if (key === 'genres') { + const existingGenres = media.genres || [] + if (podcastMetadata.genres.some(g => !existingGenres.includes(g)) || existingGenres.some(g => !podcastMetadata.genres.includes(g))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating podcast genres "${existingGenres.join(',')}" => "${podcastMetadata.genres.join(',')}" for podcast "${podcastMetadata.title}"`) + media.genres = podcastMetadata.genres + media.changed('genres', true) + hasMediaChanges = true + } + } else if (key === 'tags') { + const existingTags = media.tags || [] + if (podcastMetadata.tags.some(t => !existingTags.includes(t)) || existingTags.some(t => !podcastMetadata.tags.includes(t))) { + libraryScan.addLog(LogLevel.DEBUG, `Updating podcast tags "${existingTags.join(',')}" => "${podcastMetadata.tags.join(',')}" for podcast "${podcastMetadata.title}"`) + media.tags = podcastMetadata.tags + media.changed('tags', true) + hasMediaChanges = true + } + } else if (podcastMetadata[key] !== media[key]) { + libraryScan.addLog(LogLevel.DEBUG, `Updating podcast ${key} "${media[key]}" => "${podcastMetadata[key]}" for podcast "${podcastMetadata.title}"`) + media[key] = podcastMetadata[key] + hasMediaChanges = true + } + } + + // If no cover then extract cover from audio file if available + if (!media.coverPath && existingPodcastEpisodes.length) { + const audioFiles = existingPodcastEpisodes.map(ep => ep.audioFile) + const extractedCoverPath = await CoverManager.saveEmbeddedCoverArtNew(audioFiles, existingLibraryItem.id, existingLibraryItem.path) + if (extractedCoverPath) { + libraryScan.addLog(LogLevel.DEBUG, `Updating podcast "${podcastMetadata.title}" extracted embedded cover art from audio file to path "${extractedCoverPath}"`) + media.coverPath = extractedCoverPath + hasMediaChanges = true + } + } + + existingLibraryItem.media = media + + let libraryItemUpdated = false + + // Save Podcast changes to db + if (hasMediaChanges) { + await media.save() + await this.saveMetadataFile(existingLibraryItem, libraryScan) + libraryItemUpdated = global.ServerSettings.storeMetadataWithItem + } + + if (libraryItemUpdated) { + existingLibraryItem.changed('libraryFiles', true) + await existingLibraryItem.save() + } + + return existingLibraryItem + } + + /** + * + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan) { + // Scan audio files found + let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryItemData.mediaType, libraryItemData, libraryItemData.audioLibraryFiles) + + // Do not add library items that have no valid audio files + if (!scannedAudioFiles.length) { + libraryScan.addLog(LogLevel.WARN, `Library item at path "${libraryItemData.relPath}" has no audio files - ignoring`) + return null + } + + const newPodcastEpisodes = [] + + // Create podcast episodes from audio files + for (const audioFile of scannedAudioFiles) { + const newEpisode = { + title: audioFile.metaTags.tagTitle || audioFile.metadata.filenameNoExt, + subtitle: null, + season: null, + episode: null, + episodeType: null, + pubDate: null, + publishedAt: null, + description: null, + audioFile: audioFile.toJSON(), + chapters: audioFile.chapters || [] + } + + // Set metadata and save new episode + this.setPodcastEpisodeMetadataFromAudioFile(newEpisode, libraryScan) + libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newEpisode.title}" found`) + newPodcastEpisodes.push(newEpisode) + } + + const podcastMetadata = await this.getPodcastMetadataFromScanData(newPodcastEpisodes, libraryItemData, libraryScan) + podcastMetadata.explicit = !!podcastMetadata.explicit // Ensure boolean + + // Set cover image from library file + if (libraryItemData.imageLibraryFiles.length) { + // Prefer using a cover image with the name "cover" otherwise use the first image + const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path)) + podcastMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path + } + + // Set default podcastType to episodic + if (!podcastMetadata.podcastType) { + podcastMetadata.podcastType = 'episodic' + } + + const podcastObject = { + ...podcastMetadata, + autoDownloadEpisodes: false, + autoDownloadSchedule: '0 * * * *', + lastEpisodeCheck: 0, + maxEpisodesToKeep: 0, + maxNewEpisodesToDownload: 3, + podcastEpisodes: newPodcastEpisodes + } + + const libraryItemObj = libraryItemData.libraryItemObject + libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image + libraryItemObj.isMissing = false + libraryItemObj.isInvalid = false + libraryItemObj.extraData = {} + + // If cover was not found in folder then check embedded covers in audio files + if (!podcastObject.coverPath && scannedAudioFiles.length) { + // Extract and save embedded cover art + podcastObject.coverPath = await CoverManager.saveEmbeddedCoverArtNew(scannedAudioFiles, libraryItemObj.id, libraryItemObj.path) + } + + libraryItemObj.podcast = podcastObject + const libraryItem = await Database.libraryItemModel.create(libraryItemObj, { + include: { + model: Database.podcastModel, + include: Database.podcastEpisodeModel + } + }) + + Database.addGenresToFilterData(libraryItemData.libraryId, libraryItem.podcast.genres) + Database.addTagsToFilterData(libraryItemData.libraryId, libraryItem.podcast.tags) + + // Load for emitting to client + libraryItem.media = await libraryItem.getMedia({ + include: Database.podcastEpisodeModel + }) + + await this.saveMetadataFile(libraryItem, libraryScan) + if (global.ServerSettings.storeMetadataWithItem) { + libraryItem.changed('libraryFiles', true) + await libraryItem.save() + } + + return libraryItem + } + + /** + * + * @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts + * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan) { + const podcastMetadata = { + title: libraryItemData.mediaMetadata.title, + titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title), + author: undefined, + releaseDate: undefined, + feedURL: undefined, + imageURL: undefined, + description: undefined, + itunesPageURL: undefined, + itunesId: undefined, + itunesArtistId: undefined, + language: undefined, + podcastType: undefined, + explicit: undefined, + tags: [], + genres: [] + } + + if (podcastEpisodes.length) { + const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags + const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata + + const MetadataMapArray = [ + { + tag: 'tagAlbum', + altTag: 'tagSeries', + key: 'title' + }, + { + tag: 'tagArtist', + key: 'author' + }, + { + tag: 'tagGenre', + key: 'genres' + }, + { + tag: 'tagLanguage', + key: 'language' + }, + { + tag: 'tagItunesId', + key: 'itunesId' + }, + { + tag: 'tagPodcastType', + key: 'podcastType', + } + ] + + MetadataMapArray.forEach((mapping) => { + let value = audioFileMetaTags[mapping.tag] + let tagToUse = mapping.tag + if (!value && mapping.altTag) { + value = audioFileMetaTags[mapping.altTag] + tagToUse = mapping.altTag + } + + if (value && typeof value === 'string') { + value = value.trim() // Trim whitespace + + if (mapping.key === 'genres' && (!podcastMetadata.genres.length || overrideExistingDetails)) { + podcastMetadata.genres = this.parseGenresString(value) + libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`) + } else if (!podcastMetadata[mapping.key] || overrideExistingDetails) { + podcastMetadata[mapping.key] = value + libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`) + } + } + }) + } + + // If metadata.json or metadata.abs use this for metadata + const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile + const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null + if (metadataText) { + libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`) + let abMetadata = null + if (!!libraryItemData.metadataJsonLibraryFile) { + abMetadata = abmetadataGenerator.parseJson(metadataText) + } else { + abMetadata = abmetadataGenerator.parse(metadataText, 'podcast') + } + + if (abMetadata) { + if (abMetadata.tags?.length) { + podcastMetadata.tags = abMetadata.tags + } + for (const key in abMetadata.metadata) { + if (abMetadata.metadata[key] === undefined) continue + + // TODO: New podcast model changed some keys, need to update the abmetadataGenerator + let newModelKey = key + if (key === 'feedUrl') newModelKey = 'feedURL' + else if (key === 'imageUrl') newModelKey = 'imageURL' + else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL' + else if (key === 'type') newModelKey = 'podcastType' + + podcastMetadata[newModelKey] = abMetadata.metadata[key] + } + } + } + + podcastMetadata.titleIgnorePrefix = getTitleIgnorePrefix(podcastMetadata.title) + + return podcastMetadata + } + + /** + * Parse a genre string into multiple genres + * @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"] + * @param {string} genreTag + * @returns {string[]} + */ + parseGenresString(genreTag) { + if (!genreTag?.length) return [] + const separators = ['/', '//', ';'] + for (let i = 0; i < separators.length; i++) { + if (genreTag.includes(separators[i])) { + return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g) + } + } + return [genreTag] + } + + /** + * + * @param {import('../models/LibraryItem')} libraryItem + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async saveMetadataFile(libraryItem, libraryScan) { + let metadataPath = Path.join(global.MetadataPath, 'items', libraryItem.id) + let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem + if (storeMetadataWithItem) { + metadataPath = libraryItem.path + } else { + // Make sure metadata book dir exists + storeMetadataWithItem = false + await fsExtra.ensureDir(metadataPath) + } + + const metadataFileFormat = global.ServerSettings.metadataFileFormat + const metadataFilePath = Path.join(metadataPath, `metadata.${metadataFileFormat}`) + if (metadataFileFormat === 'json') { + // Remove metadata.abs if it exists + if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.abs`))) { + libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.abs for item "${libraryItem.media.title}"`) + await fsExtra.remove(Path.join(metadataPath, `metadata.abs`)) + libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.abs`))) + } + + // TODO: Update to not use `metadata` so it fits the updated model + const jsonObject = { + tags: libraryItem.media.tags || [], + metadata: { + title: libraryItem.media.title, + author: libraryItem.media.author, + description: libraryItem.media.description, + releaseDate: libraryItem.media.releaseDate, + genres: libraryItem.media.genres || [], + feedUrl: libraryItem.media.feedURL, + imageUrl: libraryItem.media.imageURL, + itunesPageUrl: libraryItem.media.itunesPageURL, + itunesId: libraryItem.media.itunesId, + itunesArtistId: libraryItem.media.itunesArtistId, + asin: libraryItem.media.asin, + language: libraryItem.media.language, + explicit: !!libraryItem.media.explicit, + type: libraryItem.media.podcastType + } + } + return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => { + // Add metadata.json to libraryFiles array if it is new + let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) + + return metadataLibraryFile + }).catch((error) => { + libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error) + return null + }) + } else { + // Remove metadata.json if it exists + if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.json`))) { + libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.json for item "${libraryItem.media.title}"`) + await fsExtra.remove(Path.join(metadataPath, `metadata.json`)) + libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.json`))) + } + + return abmetadataGenerator.generateFromNewModel(libraryItem, metadataFilePath).then(async (success) => { + if (!success) { + libraryScan.addLog(LogLevel.ERROR, `Failed saving abmetadata to "${metadataFilePath}"`) + return null + } + // Add metadata.abs to libraryFiles array if it is new + let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) + return metadataLibraryFile + }) + } + } + + /** + * + * @param {PodcastEpisode} podcastEpisode Not the model when creating new podcast + * @param {import('./ScanLogger')} scanLogger + */ + setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, scanLogger) { + const MetadataMapArray = [ + { + tag: 'tagComment', + altTag: 'tagSubtitle', + key: 'description' + }, + { + tag: 'tagSubtitle', + key: 'subtitle' + }, + { + tag: 'tagDate', + key: 'pubDate' + }, + { + tag: 'tagDisc', + key: 'season', + }, + { + tag: 'tagTrack', + altTag: 'tagSeriesPart', + key: 'episode' + }, + { + tag: 'tagTitle', + key: 'title' + }, + { + tag: 'tagEpisodeType', + key: 'episodeType' + } + ] + + const audioFileMetaTags = podcastEpisode.audioFile.metaTags + const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata + MetadataMapArray.forEach((mapping) => { + let value = audioFileMetaTags[mapping.tag] + let tagToUse = mapping.tag + if (!value && mapping.altTag) { + tagToUse = mapping.altTag + value = audioFileMetaTags[mapping.altTag] + } + + if (value && typeof value === 'string') { + value = value.trim() // Trim whitespace + + if (mapping.key === 'pubDate' && (!podcastEpisode.pubDate || overrideExistingDetails)) { + const pubJsDate = new Date(value) + if (pubJsDate && !isNaN(pubJsDate)) { + podcastEpisode.publishedAt = pubJsDate.valueOf() + podcastEpisode.pubDate = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } else { + scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`) + } + } else if (mapping.key === 'episodeType' && (!podcastEpisode.episodeType || overrideExistingDetails)) { + if (['full', 'trailer', 'bonus'].includes(value)) { + podcastEpisode.episodeType = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } else { + scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`) + } + } else if (!podcastEpisode[mapping.key] || overrideExistingDetails) { + podcastEpisode[mapping.key] = value + scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`) + } + } + }) + } +} +module.exports = new PodcastScanner() \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index d7f264bb..091c4e48 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -1,27 +1,17 @@ -const Sequelize = require('sequelize') -const fs = require('../libs/fsExtra') -const Path = require('path') const Logger = require('../Logger') const SocketAuthority = require('../SocketAuthority') const Database = require('../Database') // Utils -const { groupFilesIntoLibraryItemPaths, getLibraryItemFileData, scanFolder, checkFilepathIsAudioFile } = require('../utils/scandir') -const { comparePaths } = require('../utils/index') -const { getIno, filePathToPOSIX } = require('../utils/fileUtils') -const { ScanResult, LogLevel } = require('../utils/constants') +const { LogLevel } = require('../utils/constants') const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils') const MediaFileScanner = require('./MediaFileScanner') const BookFinder = require('../finders/BookFinder') const PodcastFinder = require('../finders/PodcastFinder') -const LibraryItem = require('../objects/LibraryItem') const LibraryScan = require('./LibraryScan') -const ScanOptions = require('./ScanOptions') - const Author = require('../objects/entities/Author') const Series = require('../objects/entities/Series') -const Task = require('../objects/Task') class Scanner { constructor(coverManager, taskManager) { @@ -31,710 +21,10 @@ class Scanner { this.cancelLibraryScan = {} this.librariesScanning = [] - // Watcher file update scan vars - this.pendingFileUpdatesToScan = [] - this.scanningFilesChanged = false - this.bookFinder = new BookFinder() this.podcastFinder = new PodcastFinder() } - isLibraryScanning(libraryId) { - return this.librariesScanning.find(ls => ls.id === libraryId) - } - - setCancelLibraryScan(libraryId) { - var libraryScanning = this.librariesScanning.find(ls => ls.id === libraryId) - if (!libraryScanning) return - this.cancelLibraryScan[libraryId] = true - } - - getScanResultDescription(result) { - switch (result) { - case ScanResult.ADDED: - return 'Added to library' - case ScanResult.NOTHING: - return 'No updates necessary' - case ScanResult.REMOVED: - return 'Removed from library' - case ScanResult.UPDATED: - return 'Item was updated' - case ScanResult.UPTODATE: - return 'No updates necessary' - default: - return '' - } - } - - async scanLibraryItemByRequest(libraryItem) { - const library = await Database.libraryModel.getOldById(libraryItem.libraryId) - if (!library) { - Logger.error(`[Scanner] Scan libraryItem by id library not found "${libraryItem.libraryId}"`) - return ScanResult.NOTHING - } - const folder = library.folders.find(f => f.id === libraryItem.folderId) - if (!folder) { - Logger.error(`[Scanner] Scan libraryItem by id folder not found "${libraryItem.folderId}" in library "${library.name}"`) - return ScanResult.NOTHING - } - Logger.info(`[Scanner] Scanning Library Item "${libraryItem.media.metadata.title}"`) - - const task = new Task() - task.setData('scan-item', `Scan ${libraryItem.media.metadata.title}`, '', true, { - libraryItemId: libraryItem.id, - libraryId: library.id, - mediaType: library.mediaType - }) - this.taskManager.addTask(task) - - const result = await this.scanLibraryItem(library, folder, libraryItem) - - task.setFinished(this.getScanResultDescription(result)) - this.taskManager.taskFinished(task) - - return result - } - - async scanLibraryItem(library, folder, libraryItem) { - const libraryMediaType = library.mediaType - - // TODO: Support for single media item - const libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, libraryItem.path, false) - if (!libraryItemData) { - return ScanResult.NOTHING - } - let hasUpdated = false - - const checkRes = libraryItem.checkScanData(libraryItemData) - if (checkRes.updated) hasUpdated = true - - // Sync other files first so that local images are used as cover art - if (await libraryItem.syncFiles(Database.serverSettings.scannerPreferOpfMetadata, library.settings)) { - hasUpdated = true - } - - // Scan all audio files - if (libraryItem.hasAudioFiles) { - const libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio') - if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItem)) { - hasUpdated = true - } - - // Extract embedded cover art if cover is not already in directory - if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) { - const coverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem) - if (coverPath) { - Logger.debug(`[Scanner] Saved embedded cover art "${coverPath}"`) - hasUpdated = true - } - } - } - - await this.createNewAuthorsAndSeries(libraryItem) - - // Library Item is invalid - (a book has no audio files or ebook files) - if (!libraryItem.hasMediaEntities && libraryItem.mediaType !== 'podcast') { - libraryItem.setInvalid() - hasUpdated = true - } else if (libraryItem.isInvalid) { - libraryItem.isInvalid = false - hasUpdated = true - } - - if (hasUpdated) { - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - return ScanResult.UPDATED - } - return ScanResult.UPTODATE - } - - async scan(library, options = {}) { - if (this.isLibraryScanning(library.id)) { - Logger.error(`[Scanner] Already scanning ${library.id}`) - return - } - - if (!library.folders.length) { - Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`) - return - } - - const scanOptions = new ScanOptions() - scanOptions.setData(options, Database.serverSettings) - - const libraryScan = new LibraryScan() - libraryScan.setData(library, scanOptions) - libraryScan.verbose = false - this.librariesScanning.push(libraryScan.getScanEmitData) - - SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData) - - Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`) - - const canceled = await this.scanLibrary(libraryScan) - - if (canceled) { - Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`) - delete this.cancelLibraryScan[libraryScan.libraryId] - } - - libraryScan.setComplete() - - Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`) - this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) - - if (canceled && !libraryScan.totalResults) { - const emitData = libraryScan.getScanEmitData - emitData.results = null - SocketAuthority.emitter('scan_complete', emitData) - return - } - - SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData) - - if (libraryScan.totalResults) { - libraryScan.saveLog() - } - } - - async scanLibrary(libraryScan) { - let libraryItemDataFound = [] - - // Scan each library - for (let i = 0; i < libraryScan.folders.length; i++) { - const folder = libraryScan.folders[i] - const itemDataFoundInFolder = await scanFolder(libraryScan.library, folder) - libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`) - libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder) - } - - if (this.cancelLibraryScan[libraryScan.libraryId]) return true - - // Remove items with no inode - libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino) - const libraryItemsInLibrary = Database.libraryItems.filter(li => li.libraryId === libraryScan.libraryId) - - const MaxSizePerChunk = 2.5e9 - const itemDataToRescanChunks = [] - const newItemDataToScanChunks = [] - let itemsToUpdate = [] - let itemDataToRescan = [] - let itemDataToRescanSize = 0 - let newItemDataToScan = [] - let newItemDataToScanSize = 0 - const itemsToFindCovers = [] - - // Check for existing & removed library items - for (let i = 0; i < libraryItemsInLibrary.length; i++) { - const libraryItem = libraryItemsInLibrary[i] - // Find library item folder with matching inode or matching path - const dataFound = libraryItemDataFound.find(lid => lid.ino === libraryItem.ino || comparePaths(lid.relPath, libraryItem.relPath)) - if (!dataFound) { - // Podcast folder can have no episodes and still be valid - if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(libraryItem.path)) { - Logger.info(`[Scanner] Library item "${libraryItem.media.metadata.title}" folder exists but has no episodes`) - if (libraryItem.isMissing) { - libraryScan.resultsUpdated++ - libraryItem.isMissing = false - libraryItem.setLastScan() - itemsToUpdate.push(libraryItem) - } - } else { - libraryScan.addLog(LogLevel.WARN, `Library Item "${libraryItem.media.metadata.title}" is missing`) - Logger.warn(`[Scanner] Library item "${libraryItem.media.metadata.title}" is missing (inode "${libraryItem.ino}")`) - libraryScan.resultsMissing++ - libraryItem.setMissing() - itemsToUpdate.push(libraryItem) - } - } else { - const checkRes = libraryItem.checkScanData(dataFound) - if (checkRes.newLibraryFiles.length || libraryScan.scanOptions.forceRescan) { // Item has new files - checkRes.libraryItem = libraryItem - checkRes.scanData = dataFound - - // If this item will go over max size then push current chunk - if (libraryItem.audioFileTotalSize + itemDataToRescanSize > MaxSizePerChunk && itemDataToRescan.length > 0) { - itemDataToRescanChunks.push(itemDataToRescan) - itemDataToRescanSize = 0 - itemDataToRescan = [] - } - - itemDataToRescan.push(checkRes) - itemDataToRescanSize += libraryItem.audioFileTotalSize - if (itemDataToRescanSize >= MaxSizePerChunk) { - itemDataToRescanChunks.push(itemDataToRescan) - itemDataToRescanSize = 0 - itemDataToRescan = [] - } - - } else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) { // Search cover - libraryScan.resultsUpdated++ - itemsToFindCovers.push(libraryItem) - itemsToUpdate.push(libraryItem) - } else if (checkRes.updated) { // Updated but no scan required - libraryScan.resultsUpdated++ - itemsToUpdate.push(libraryItem) - } - libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino !== dataFound.ino) - } - } - if (itemDataToRescan.length) itemDataToRescanChunks.push(itemDataToRescan) - - // Potential NEW Library Items - for (let i = 0; i < libraryItemDataFound.length; i++) { - const dataFound = libraryItemDataFound[i] - - const hasMediaFile = dataFound.libraryFiles.some(lf => lf.isMediaFile) - if (!hasMediaFile) { - libraryScan.addLog(LogLevel.WARN, `Item found "${libraryItemDataFound.path}" has no media files`) - } else { - // If this item will go over max size then push current chunk - let mediaFileSize = 0 - dataFound.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video').forEach(lf => mediaFileSize += lf.metadata.size) - if (mediaFileSize + newItemDataToScanSize > MaxSizePerChunk && newItemDataToScan.length > 0) { - newItemDataToScanChunks.push(newItemDataToScan) - newItemDataToScanSize = 0 - newItemDataToScan = [] - } - - newItemDataToScan.push(dataFound) - newItemDataToScanSize += mediaFileSize - - if (newItemDataToScanSize >= MaxSizePerChunk) { - newItemDataToScanChunks.push(newItemDataToScan) - newItemDataToScanSize = 0 - newItemDataToScan = [] - } - } - } - if (newItemDataToScan.length) newItemDataToScanChunks.push(newItemDataToScan) - - // Library Items not requiring a scan but require a search for cover - for (let i = 0; i < itemsToFindCovers.length; i++) { - const libraryItem = itemsToFindCovers[i] - const updatedCover = await this.searchForCover(libraryItem, libraryScan) - libraryItem.media.updateLastCoverSearch(updatedCover) - } - - if (itemsToUpdate.length) { - await this.updateLibraryItemChunk(itemsToUpdate) - if (this.cancelLibraryScan[libraryScan.libraryId]) return true - } - - // Chunking will be removed when legacy single threaded scanner is removed - for (let i = 0; i < itemDataToRescanChunks.length; i++) { - await this.rescanLibraryItemDataChunk(itemDataToRescanChunks[i], libraryScan) - if (this.cancelLibraryScan[libraryScan.libraryId]) return true - } - for (let i = 0; i < newItemDataToScanChunks.length; i++) { - await this.scanNewLibraryItemDataChunk(newItemDataToScanChunks[i], libraryScan) - if (this.cancelLibraryScan[libraryScan.libraryId]) return true - } - } - - async updateLibraryItemChunk(itemsToUpdate) { - await Database.updateBulkLibraryItems(itemsToUpdate) - SocketAuthority.emitter('items_updated', itemsToUpdate.map(li => li.toJSONExpanded())) - } - - async rescanLibraryItemDataChunk(itemDataToRescan, libraryScan) { - var itemsUpdated = await Promise.all(itemDataToRescan.map((lid) => { - return this.rescanLibraryItem(lid, libraryScan) - })) - - itemsUpdated = itemsUpdated.filter(li => li) // Filter out nulls - - for (const libraryItem of itemsUpdated) { - // Temp authors & series are inserted - create them if found - await this.createNewAuthorsAndSeries(libraryItem) - } - - if (itemsUpdated.length) { - libraryScan.resultsUpdated += itemsUpdated.length - await Database.updateBulkLibraryItems(itemsUpdated) - SocketAuthority.emitter('items_updated', itemsUpdated.map(li => li.toJSONExpanded())) - } - } - - async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) { - let newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => { - return this.scanNewLibraryItem(lid, libraryScan.library, libraryScan) - })) - newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls - - for (const libraryItem of newLibraryItems) { - // Temp authors & series are inserted - create them if found - await this.createNewAuthorsAndSeries(libraryItem) - } - - libraryScan.resultsAdded += newLibraryItems.length - await Database.createBulkLibraryItems(newLibraryItems) - SocketAuthority.emitter('items_added', newLibraryItems.map(li => li.toJSONExpanded())) - } - - async rescanLibraryItem(libraryItemCheckData, libraryScan) { - const { newLibraryFiles, filesRemoved, existingLibraryFiles, libraryItem, scanData, updated } = libraryItemCheckData - libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${libraryItem.path}"`) - let hasUpdated = updated - - // Sync other files first to use local images as cover before extracting audio file cover - if (await libraryItem.syncFiles(libraryScan.preferOpfMetadata, libraryScan.library.settings)) { - hasUpdated = true - } - - // forceRescan all existing audio files - will probe and update ID3 tag metadata - const existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio') - if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) { - if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, libraryItem, libraryScan)) { - hasUpdated = true - } - } - // Scan new audio files - const newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio') - const removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio') - if (newAudioFiles.length || removedAudioFiles.length) { - if (await MediaFileScanner.scanMediaFiles(newAudioFiles, libraryItem, libraryScan)) { - hasUpdated = true - } - } - // If an audio file has embedded cover art and no cover is set yet, extract & use it - if (newAudioFiles.length || libraryScan.scanOptions.forceRescan) { - if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) { - const savedCoverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem) - if (savedCoverPath) { - hasUpdated = true - libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${savedCoverPath}"`) - } - } - } - - // Library Item is invalid - (a book has no audio files or ebook files) - if (!libraryItem.hasMediaEntities && libraryItem.mediaType !== 'podcast') { - libraryItem.setInvalid() - hasUpdated = true - } else if (libraryItem.isInvalid) { - libraryItem.isInvalid = false - hasUpdated = true - } - - // Scan for cover if enabled and has no cover (and author or title has changed OR has been 7 days since last lookup) - if (libraryScan.findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) { - const updatedCover = await this.searchForCover(libraryItem, libraryScan) - libraryItem.media.updateLastCoverSearch(updatedCover) - hasUpdated = true - } - - return hasUpdated ? libraryItem : null - } - - async scanNewLibraryItem(libraryItemData, library, libraryScan = null) { - if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`) - else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`) - - const preferOpfMetadata = libraryScan ? !!libraryScan.preferOpfMetadata : !!global.ServerSettings.scannerPreferOpfMetadata - const findCovers = libraryScan ? !!libraryScan.findCovers : !!global.ServerSettings.scannerFindCovers - - const libraryItem = new LibraryItem() - libraryItem.setData(library.mediaType, libraryItemData) - libraryItem.setLastScan() - - const mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video') - if (mediaFiles.length) { - await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItem, libraryScan) - } - - await libraryItem.syncFiles(preferOpfMetadata, library.settings) - - if (!libraryItem.hasMediaEntities) { - Logger.warn(`[Scanner] Library item has no media files "${libraryItemData.path}"`) - return null - } - - // Extract embedded cover art if cover is not already in directory - if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) { - const coverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem) - if (coverPath) { - if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${coverPath}"`) - else Logger.debug(`[Scanner] Saved embedded cover art "${coverPath}"`) - } - } - - // Scan for cover if enabled and has no cover - if (library.isBook) { - if (libraryItem && findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) { - const updatedCover = await this.searchForCover(libraryItem, libraryScan) - libraryItem.media.updateLastCoverSearch(updatedCover) - } - } - - return libraryItem - } - - // Any series or author object on library item with an id starting with "new" - // will create a new author/series OR find a matching author/series - async createNewAuthorsAndSeries(libraryItem) { - if (libraryItem.mediaType !== 'book') return - - // Create or match all new authors and series - if (libraryItem.media.metadata.authors.some(au => au.id.startsWith('new'))) { - const newAuthors = [] - libraryItem.media.metadata.authors = Promise.all(libraryItem.media.metadata.authors.map(async (tempMinAuthor) => { - let _author = await Database.authorModel.getOldByNameAndLibrary(tempMinAuthor.name, libraryItem.libraryId) - if (!_author) _author = newAuthors.find(au => au.libraryId === libraryItem.libraryId && au.checkNameEquals(tempMinAuthor.name)) // Check new unsaved authors - if (!_author) { // Must create new author - _author = new Author() - _author.setData(tempMinAuthor, libraryItem.libraryId) - newAuthors.push(_author) - // Update filter data - Database.addAuthorToFilterData(libraryItem.libraryId, _author.name, _author.id) - } - - return { - id: _author.id, - name: _author.name - } - })) - if (newAuthors.length) { - await Database.createBulkAuthors(newAuthors) - SocketAuthority.emitter('authors_added', newAuthors.map(au => au.toJSON())) - } - } - if (libraryItem.media.metadata.series.some(se => se.id.startsWith('new'))) { - const newSeries = [] - libraryItem.media.metadata.series = await Promise.all(libraryItem.media.metadata.series.map(async (tempMinSeries) => { - let _series = await Database.seriesModel.getOldByNameAndLibrary(tempMinSeries.name, libraryItem.libraryId) - if (!_series) { - // Check new unsaved series - _series = newSeries.find(se => se.libraryId === libraryItem.libraryId && se.checkNameEquals(tempMinSeries.name)) - } - - if (!_series) { // Must create new series - _series = new Series() - _series.setData(tempMinSeries, libraryItem.libraryId) - newSeries.push(_series) - // Update filter data - Database.addSeriesToFilterData(libraryItem.libraryId, _series.name, _series.id) - } - return { - id: _series.id, - name: _series.name, - sequence: tempMinSeries.sequence - } - })) - if (newSeries.length) { - await Database.createBulkSeries(newSeries) - SocketAuthority.emitter('multiple_series_added', newSeries.map(se => se.toJSON())) - } - } - } - - getFileUpdatesGrouped(fileUpdates) { - var folderGroups = {} - fileUpdates.forEach((file) => { - if (folderGroups[file.folderId]) { - folderGroups[file.folderId].fileUpdates.push(file) - } else { - folderGroups[file.folderId] = { - libraryId: file.libraryId, - folderId: file.folderId, - fileUpdates: [file] - } - } - }) - return folderGroups - } - - async scanFilesChanged(fileUpdates) { - if (!fileUpdates?.length) return - - // If already scanning files from watcher then add these updates to queue - if (this.scanningFilesChanged) { - this.pendingFileUpdatesToScan.push(fileUpdates) - Logger.debug(`[Scanner] Already scanning files from watcher - file updates pushed to queue (size ${this.pendingFileUpdatesToScan.length})`) - return - } - this.scanningFilesChanged = true - - // files grouped by folder - const folderGroups = this.getFileUpdatesGrouped(fileUpdates) - - for (const folderId in folderGroups) { - const libraryId = folderGroups[folderId].libraryId - const library = await Database.libraryModel.getOldById(libraryId) - if (!library) { - Logger.error(`[Scanner] Library not found in files changed ${libraryId}`) - continue - } - const folder = library.getFolderById(folderId) - if (!folder) { - Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`) - continue - } - const relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath) - const fileUpdateGroup = groupFilesIntoLibraryItemPaths(library.mediaType, relFilePaths, false) - - if (!Object.keys(fileUpdateGroup).length) { - Logger.info(`[Scanner] No important changes to scan for in folder "${folderId}"`) - continue - } - const folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateGroup) - Logger.debug(`[Scanner] Folder scan results`, folderScanResults) - - // If something was updated then reset numIssues filter data for library - if (Object.values(folderScanResults).some(scanResult => scanResult !== ScanResult.NOTHING && scanResult !== ScanResult.UPTODATE)) { - await Database.resetLibraryIssuesFilterData(libraryId) - } - } - - this.scanningFilesChanged = false - - if (this.pendingFileUpdatesToScan.length) { - Logger.debug(`[Scanner] File updates finished scanning with more updates in queue (${this.pendingFileUpdatesToScan.length})`) - this.scanFilesChanged(this.pendingFileUpdatesToScan.shift()) - } - } - - async scanFolderUpdates(library, folder, fileUpdateGroup) { - Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`) - Logger.debug(`[Scanner] scanFolderUpdates fileUpdateGroup`, fileUpdateGroup) - - // First pass - Remove files in parent dirs of items and remap the fileupdate group - // Test Case: Moving audio files from library item folder to author folder should trigger a re-scan of the item - const updateGroup = { ...fileUpdateGroup } - for (const itemDir in updateGroup) { - if (itemDir == fileUpdateGroup[itemDir]) continue // Media in root path - - const itemDirNestedFiles = fileUpdateGroup[itemDir].filter(b => b.includes('/')) - if (!itemDirNestedFiles.length) continue - - const firstNest = itemDirNestedFiles[0].split('/').shift() - const altDir = `${itemDir}/${firstNest}` - - const fullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir) - const childLibraryItem = await Database.libraryItemModel.findOne({ - attributes: ['id', 'path'], - where: { - path: { - [Sequelize.Op.not]: fullPath - }, - path: { - [Sequelize.Op.startsWith]: fullPath - } - } - }) - if (!childLibraryItem) { - continue - } - - const altFullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), altDir) - const altChildLibraryItem = await Database.libraryItemModel.findOne({ - attributes: ['id', 'path'], - where: { - path: { - [Sequelize.Op.not]: altFullPath - }, - path: { - [Sequelize.Op.startsWith]: altFullPath - } - } - }) - if (altChildLibraryItem) { - continue - } - - delete fileUpdateGroup[itemDir] - fileUpdateGroup[altDir] = itemDirNestedFiles.map((f) => f.split('/').slice(1).join('/')) - Logger.warn(`[Scanner] Some files were modified in a parent directory of a library item "${childLibraryItem.path}" - ignoring`) - } - - // Second pass: Check for new/updated/removed items - const itemGroupingResults = {} - for (const itemDir in fileUpdateGroup) { - const fullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir) - const dirIno = await getIno(fullPath) - - const itemDirParts = itemDir.split('/').slice(0, -1) - const potentialChildDirs = [] - for (let i = 0; i < itemDirParts.length; i++) { - potentialChildDirs.push(Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir.split('/').slice(0, -1 - i).join('/'))) - } - - // Check if book dir group is already an item - let existingLibraryItem = await Database.libraryItemModel.findOneOld({ - path: potentialChildDirs - }) - - if (!existingLibraryItem) { - existingLibraryItem = await Database.libraryItemModel.findOneOld({ - ino: dirIno - }) - if (existingLibraryItem) { - Logger.debug(`[Scanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`) - // Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData - existingLibraryItem.path = fullPath - existingLibraryItem.relPath = itemDir - } - } - if (existingLibraryItem) { - // Is the item exactly - check if was deleted - if (existingLibraryItem.path === fullPath) { - const exists = await fs.pathExists(fullPath) - if (!exists) { - Logger.info(`[Scanner] Scanning file update group and library item was deleted "${existingLibraryItem.media.metadata.title}" - marking as missing`) - existingLibraryItem.setMissing() - await Database.updateLibraryItem(existingLibraryItem) - SocketAuthority.emitter('item_updated', existingLibraryItem.toJSONExpanded()) - - itemGroupingResults[itemDir] = ScanResult.REMOVED - continue - } - } - - // Scan library item for updates - Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`) - itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem) - continue - } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(checkFilepathIsAudioFile)) { - Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" has no audio files`) - continue - } - - // Check if a library item is a subdirectory of this dir - const childItem = await Database.libraryItemModel.findOne({ - attributes: ['id', 'path'], - where: { - path: { - [Sequelize.Op.startsWith]: fullPath + '/' - } - } - }) - if (childItem) { - Logger.warn(`[Scanner] Files were modified in a parent directory of a library item "${childItem.path}" - ignoring`) - itemGroupingResults[itemDir] = ScanResult.NOTHING - continue - } - - Logger.debug(`[Scanner] Folder update group must be a new item "${itemDir}" in library "${library.name}"`) - var isSingleMediaItem = itemDir === fileUpdateGroup[itemDir] - var newLibraryItem = await this.scanPotentialNewLibraryItem(library, folder, fullPath, isSingleMediaItem) - if (newLibraryItem) { - await this.createNewAuthorsAndSeries(newLibraryItem) - await Database.createLibraryItem(newLibraryItem) - SocketAuthority.emitter('item_added', newLibraryItem.toJSONExpanded()) - } - itemGroupingResults[itemDir] = newLibraryItem ? ScanResult.ADDED : ScanResult.NOTHING - } - - return itemGroupingResults - } - - async scanPotentialNewLibraryItem(library, folder, fullPath, isSingleMediaItem = false) { - const libraryItemData = await getLibraryItemFileData(library.mediaType, folder, fullPath, isSingleMediaItem) - if (!libraryItemData) return null - return this.scanNewLibraryItem(libraryItemData, library) - } - async searchForCover(libraryItem, libraryScan = null) { const options = { titleDistance: 2, @@ -1032,11 +322,6 @@ class Scanner { return } - if (this.isLibraryScanning(library.id)) { - Logger.error(`[Scanner] matchLibraryItems: Already scanning ${library.id}`) - return - } - const itemsInLibrary = Database.libraryItems.filter(li => li.libraryId === library.id) if (!itemsInLibrary.length) { Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`) diff --git a/server/utils/queries/libraryItemsPodcastFilters.js b/server/utils/queries/libraryItemsPodcastFilters.js index 0176cdb9..35d9ecf9 100644 --- a/server/utils/queries/libraryItemsPodcastFilters.js +++ b/server/utils/queries/libraryItemsPodcastFilters.js @@ -303,7 +303,7 @@ module.exports = { * @returns {{podcast:object[], tags:object[]}} */ async search(oldUser, oldLibrary, query, limit, offset) { - const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user) + const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser) // Search title, author, itunesId, itunesArtistId const podcasts = await Database.podcastModel.findAll({ where: [ diff --git a/server/utils/scandir.js b/server/utils/scandir.js index 08dc1fe4..acf7ed60 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -1,7 +1,5 @@ const Path = require('path') -const fs = require('../libs/fsExtra') -const Logger = require('../Logger') -const { recurseFiles, getFileTimestampsWithIno, filePathToPOSIX } = require('./fileUtils') +const { filePathToPOSIX } = require('./fileUtils') const globals = require('./globals') const LibraryFile = require('../objects/files/LibraryFile') @@ -198,73 +196,6 @@ function buildLibraryFile(libraryItemPath, files) { } module.exports.buildLibraryFile = buildLibraryFile -// Scan folder -async function scanFolder(library, folder) { - const folderPath = filePathToPOSIX(folder.fullPath) - - const pathExists = await fs.pathExists(folderPath) - if (!pathExists) { - Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`) - return [] - } - - const fileItems = await recurseFiles(folderPath) - const libraryItemGrouping = groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly) - - if (!Object.keys(libraryItemGrouping).length) { - Logger.error(`Root path has no media folders: ${folderPath}`) - return [] - } - - const items = [] - for (const libraryItemPath in libraryItemGrouping) { - let isFile = false // item is not in a folder - let libraryItemData = null - let fileObjs = [] - if (library.mediaType === 'music') { - libraryItemData = { - path: Path.posix.join(folderPath, libraryItemPath), - relPath: libraryItemPath - } - fileObjs = await buildLibraryFile(folderPath, [libraryItemPath]) - isFile = true - } else if (libraryItemPath === libraryItemGrouping[libraryItemPath]) { - // Media file in root only get title - libraryItemData = { - mediaMetadata: { - title: Path.basename(libraryItemPath, Path.extname(libraryItemPath)) - }, - path: Path.posix.join(folderPath, libraryItemPath), - relPath: libraryItemPath - } - fileObjs = await buildLibraryFile(folderPath, [libraryItemPath]) - isFile = true - } else { - libraryItemData = getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath) - fileObjs = await buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath]) - } - - const libraryItemFolderStats = await getFileTimestampsWithIno(libraryItemData.path) - items.push({ - folderId: folder.id, - libraryId: folder.libraryId, - ino: libraryItemFolderStats.ino, - mtimeMs: libraryItemFolderStats.mtimeMs || 0, - ctimeMs: libraryItemFolderStats.ctimeMs || 0, - birthtimeMs: libraryItemFolderStats.birthtimeMs || 0, - path: libraryItemData.path, - relPath: libraryItemData.relPath, - isFile, - media: { - metadata: libraryItemData.mediaMetadata || null - }, - libraryFiles: fileObjs - }) - } - return items -} -module.exports.scanFolder = scanFolder - // Input relative filepath, output all details that can be parsed function getBookDataFromDir(folderPath, relPath, parseSubtitle = false) { relPath = filePathToPOSIX(relPath) @@ -381,60 +312,3 @@ function getDataFromMediaDir(libraryMediaType, folderPath, relPath) { } } module.exports.getDataFromMediaDir = getDataFromMediaDir - -// Called from Scanner.js -async function getLibraryItemFileData(libraryMediaType, folder, libraryItemPath, isSingleMediaItem) { - libraryItemPath = filePathToPOSIX(libraryItemPath) - const folderFullPath = filePathToPOSIX(folder.fullPath) - - const libraryItemDir = libraryItemPath.replace(folderFullPath, '').slice(1) - let libraryItemData = {} - - let fileItems = [] - - if (isSingleMediaItem) { // Single media item in root of folder - fileItems = [ - { - fullpath: libraryItemPath, - path: libraryItemDir // actually the relPath (only filename here) - } - ] - libraryItemData = { - path: libraryItemPath, // full path - relPath: libraryItemDir, // only filename - mediaMetadata: { - title: Path.basename(libraryItemDir, Path.extname(libraryItemDir)) - } - } - } else { - fileItems = await recurseFiles(libraryItemPath) - libraryItemData = getDataFromMediaDir(libraryMediaType, folderFullPath, libraryItemDir) - } - - const libraryItemDirStats = await getFileTimestampsWithIno(libraryItemData.path) - const libraryItem = { - ino: libraryItemDirStats.ino, - mtimeMs: libraryItemDirStats.mtimeMs || 0, - ctimeMs: libraryItemDirStats.ctimeMs || 0, - birthtimeMs: libraryItemDirStats.birthtimeMs || 0, - folderId: folder.id, - libraryId: folder.libraryId, - path: libraryItemData.path, - relPath: libraryItemData.relPath, - isFile: isSingleMediaItem, - media: { - metadata: libraryItemData.mediaMetadata || null - }, - libraryFiles: [] - } - - for (let i = 0; i < fileItems.length; i++) { - const fileItem = fileItems[i] - const newLibraryFile = new LibraryFile() - // fileItem.path is the relative path - await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path) - libraryItem.libraryFiles.push(newLibraryFile) - } - return libraryItem -} -module.exports.getLibraryItemFileData = getLibraryItemFileData