diff --git a/server/SocketAuthority.js b/server/SocketAuthority.js index 6ef4523b..14a1ddaa 100644 --- a/server/SocketAuthority.js +++ b/server/SocketAuthority.js @@ -34,9 +34,12 @@ class SocketAuthority { return Object.values(this.clients).filter(c => c.user && c.user.id === userId) } - // Emits event to all authorized clients - // optional filter function to only send event to specific users - // TODO: validate that filter is actually a function + /** + * Emits event to all authorized clients + * @param {string} evt + * @param {any} data + * @param {Function} [filter] optional filter function to only send event to specific users + */ emitter(evt, data, filter = null) { for (const socketId in this.clients) { if (this.clients[socketId].user) { diff --git a/server/scanner/AudioFileScanner.js b/server/scanner/AudioFileScanner.js new file mode 100644 index 00000000..cd596f86 --- /dev/null +++ b/server/scanner/AudioFileScanner.js @@ -0,0 +1,6 @@ +class AudioFileScanner { + constructor() { } + + +} +module.exports = new AudioFileScanner() \ No newline at end of file diff --git a/server/scanner/LibraryItemScanData.js b/server/scanner/LibraryItemScanData.js new file mode 100644 index 00000000..8e8cb10e --- /dev/null +++ b/server/scanner/LibraryItemScanData.js @@ -0,0 +1,147 @@ +const packageJson = require('../../package.json') +const { LogLevel } = require('../utils/constants') +const LibraryItem = require('../models/LibraryItem') + +class LibraryItemScanData { + constructor(data) { + /** @type {string} */ + this.libraryFolderId = data.libraryFolderId + /** @type {string} */ + this.libraryId = data.libraryId + /** @type {string} */ + this.ino = data.ino + /** @type {number} */ + this.mtimeMs = data.mtimeMs + /** @type {number} */ + this.ctimeMs = data.ctimeMs + /** @type {number} */ + this.birthtimeMs = data.birthtimeMs + /** @type {string} */ + this.path = data.path + /** @type {string} */ + this.relPath = data.relPath + /** @type {boolean} */ + this.isFile = data.isFile + /** @type {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */ + this.mediaMetadata = data.mediaMetadata + /** @type {import('../objects/files/LibraryFile')[]} */ + this.libraryFiles = data.libraryFiles + + // Set after check + /** @type {boolean} */ + this.hasChanges + /** @type {boolean} */ + this.hasPathChange + /** @type {LibraryItem.LibraryFileObject[]} */ + this.libraryFilesRemoved + /** @type {LibraryItem.LibraryFileObject[]} */ + this.libraryFilesAdded + /** @type {LibraryItem.LibraryFileObject[]} */ + this.libraryFilesModified + } + + /** + * + * @param {LibraryItem} existingLibraryItem + * @param {import('./LibraryScan')} libraryScan + */ + async checkLibraryItemData(existingLibraryItem, libraryScan) { + const keysToCompare = ['libraryFolderId', 'ino', 'mtimeMs', 'ctimeMs', 'birthtimeMs', 'path', 'relPath', 'isFile'] + this.hasChanges = false + this.hasPathChange = false + for (const key of keysToCompare) { + if (existingLibraryItem[key] !== this[key]) { + libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" key "${key}" changed from "${existingLibraryItem[key]}" to "${this[key]}"`) + existingLibraryItem[key] = this[key] + this.hasChanges = true + + if (key === 'relPath') { + this.hasPathChange = true + } + } + } + + this.libraryFilesRemoved = [] + this.libraryFilesModified = [] + let libraryFilesAdded = this.libraryFiles.map(lf => lf) + + for (const existingLibraryFile of existingLibraryItem.libraryFiles) { + // Find matching library file using path first and fallback to using inode value + let matchingLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === existingLibraryFile.metadata.path) + if (!matchingLibraryFile) { + matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === existingLibraryFile.ino) + if (matchingLibraryFile) { + libraryScan.addLog(LogLevel.INFO, `Library file with path "${existingLibraryFile.metadata.path}" not found, but found file with matching inode value "${existingLibraryFile.ino}" at path "${matchingLibraryFile.metadata.path}"`) + } + } + + if (!matchingLibraryFile) { // Library file removed + libraryScan.addLog(LogLevel.INFO, `Library file "${existingLibraryFile.metadata.path}" was removed from library item "${existingLibraryItem.path}"`) + this.libraryFilesRemoved.push(existingLibraryFile) + existingLibraryItem.libraryFiles = existingLibraryItem.libraryFiles.filter(lf => lf !== existingLibraryFile) + this.hasChanges = true + } else { + libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile) + if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) { + this.libraryFilesModified.push(existingLibraryFile) + this.hasChanges = true + } + } + } + + // Log new library files found + if (libraryFilesAdded.length) { + this.hasChanges = true + for (const libraryFile of libraryFilesAdded) { + libraryScan.addLog(LogLevel.INFO, `New library file found with path "${libraryFile.metadata.path}" for library item "${existingLibraryItem.path}"`) + existingLibraryItem.libraryFiles.push(libraryFile.toJSON()) + } + } + + if (this.hasChanges) { + existingLibraryItem.lastScan = Date.now() + existingLibraryItem.lastScanVersion = packageJson.version + await existingLibraryItem.save() + } else { + libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.path}" is up-to-date`) + } + + this.libraryFilesAdded = libraryFilesAdded + } + + /** + * Update existing library file with scanned in library file data + * @param {string} libraryItemPath + * @param {LibraryItem.LibraryFileObject} existingLibraryFile + * @param {import('../objects/files/LibraryFile')} scannedLibraryFile + * @param {import('./LibraryScan')} libraryScan + * @returns {boolean} false if no changes + */ + compareUpdateLibraryFile(libraryItemPath, existingLibraryFile, scannedLibraryFile, libraryScan) { + let hasChanges = false + + if (existingLibraryFile.ino !== scannedLibraryFile.ino) { + existingLibraryFile.ino = scannedLibraryFile.ino + hasChanges = true + } + + for (const key in existingLibraryFile.metadata) { + if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) { + if (key !== 'path' && key !== 'relPath') { + libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) + } else { + libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) + } + existingLibraryFile.metadata[key] = scannedLibraryFile.metadata[key] + hasChanges = true + } + } + + if (hasChanges) { + existingLibraryFile.updatedAt = Date.now() + } + + return hasChanges + } +} +module.exports = LibraryItemScanData \ No newline at end of file diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js new file mode 100644 index 00000000..4ca08e75 --- /dev/null +++ b/server/scanner/LibraryScanner.js @@ -0,0 +1,230 @@ +const Path = require('path') +const packageJson = require('../../package.json') +const Logger = require('../Logger') +const SocketAuthority = require('../SocketAuthority') +const Database = require('../Database') +const fs = require('../libs/fsExtra') +const fileUtils = require('../utils/fileUtils') +const scanUtils = require('../utils/scandir') +const { ScanResult, LogLevel } = require('../utils/constants') +const LibraryItemScanData = require('./LibraryItemScanData') + +class LibraryScanner { + constructor(coverManager, taskManager) { + this.coverManager = coverManager + this.taskManager = taskManager + + this.cancelLibraryScan = {} + this.librariesScanning = [] + } + + /** + * @param {string} libraryId + * @returns {boolean} + */ + isLibraryScanning(libraryId) { + return this.librariesScanning.some(ls => ls.id === libraryId) + } + + /** + * + * @param {import('../objects/Library')} library + * @param {*} options + */ + async scan(library, options = {}) { + if (this.isLibraryScanning(library.id)) { + Logger.error(`[Scanner] Already scanning ${library.id}`) + return + } + + if (!library.folders.length) { + Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`) + return + } + + const scanOptions = new ScanOptions() + scanOptions.setData(options, Database.serverSettings) + + const libraryScan = new LibraryScan() + libraryScan.setData(library, scanOptions) + libraryScan.verbose = true + this.librariesScanning.push(libraryScan.getScanEmitData) + + SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData) + + Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`) + + const canceled = await this.scanLibrary(libraryScan) + + if (canceled) { + Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`) + delete this.cancelLibraryScan[libraryScan.libraryId] + } + + libraryScan.setComplete() + + Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`) + this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id) + + if (canceled && !libraryScan.totalResults) { + const emitData = libraryScan.getScanEmitData + emitData.results = null + SocketAuthority.emitter('scan_complete', emitData) + return + } + + SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData) + + if (libraryScan.totalResults) { + libraryScan.saveLog() + } + } + + /** + * + * @param {import('./LibraryScan')} libraryScan + */ + async scanLibrary(libraryScan) { + /** @type {LibraryItemScanData[]} */ + let libraryItemDataFound = [] + + // Scan each library folder + for (let i = 0; i < libraryScan.folders.length; i++) { + const folder = libraryScan.folders[i] + const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder) + libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`) + libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder) + } + + if (this.cancelLibraryScan[libraryScan.libraryId]) return true + + const existingLibraryItems = await Database.libraryItemModel.findAll({ + where: { + libraryId: libraryScan.libraryId + }, + attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId'] + }) + + const libraryItemIdsMissing = [] + for (const existingLibraryItem of existingLibraryItems) { + // First try to find matching library item with exact file path + let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path) + if (!libraryItemData) { + // Fallback to finding matching library item with matching inode value + libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino) + if (libraryItemData) { + libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`) + } + } + + if (!libraryItemData) { + // Podcast folder can have no episodes and still be valid + if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) { + libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`) + } else { + libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`) + if (!existingLibraryItem.isMissing) { + libraryItemIdsMissing.push(existingLibraryItem.id) + } + } + } else { + await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan) + if (libraryItemData.hasChanges) { + await this.rescanLibraryItem(existingLibraryItem, libraryItemData) + } + } + } + + // Update missing library items + if (libraryItemIdsMissing.length) { + libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`) + await Database.libraryItemModel.update({ + isMissing: true, + lastScan: Date.now(), + lastScanVersion: packageJson.version + }, { + where: { + id: libraryItemIdsMissing + } + }) + } + } + + /** + * Get scan data for library folder + * @param {import('../objects/Library')} library + * @param {import('../objects/Folder')} folder + * @returns {LibraryItemScanData[]} + */ + async scanFolder(library, folder) { + const folderPath = fileUtils.filePathToPOSIX(folder.fullPath) + + const pathExists = await fs.pathExists(folderPath) + if (!pathExists) { + Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`) + return [] + } + + const fileItems = await fileUtils.recurseFiles(folderPath) + const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly) + + if (!Object.keys(libraryItemGrouping).length) { + Logger.error(`Root path has no media folders: ${folderPath}`) + return [] + } + + const items = [] + for (const libraryItemPath in libraryItemGrouping) { + let isFile = false // item is not in a folder + let libraryItemData = null + let fileObjs = [] + if (libraryItemPath === libraryItemGrouping[libraryItemPath]) { + // Media file in root only get title + libraryItemData = { + mediaMetadata: { + title: Path.basename(libraryItemPath, Path.extname(libraryItemPath)) + }, + path: Path.posix.join(folderPath, libraryItemPath), + relPath: libraryItemPath + } + fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath]) + isFile = true + } else { + libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath) + fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath]) + } + + const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path) + + if (!libraryItemFolderStats.ino) { + Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`) + continue + } + + items.push(new LibraryItemScanData({ + libraryFolderId: folder.id, + libraryId: folder.libraryId, + ino: libraryItemFolderStats.ino, + mtimeMs: libraryItemFolderStats.mtimeMs || 0, + ctimeMs: libraryItemFolderStats.ctimeMs || 0, + birthtimeMs: libraryItemFolderStats.birthtimeMs || 0, + path: libraryItemData.path, + relPath: libraryItemData.relPath, + isFile, + mediaMetadata: libraryItemData.mediaMetadata || null, + libraryFiles: fileObjs + })) + } + return items + } + + /** + * + * @param {import('../models/LibraryItem')} existingLibraryItem + * @param {LibraryItemScanData} libraryItemData + */ + async rescanLibraryItem(existingLibraryItem, libraryItemData) { + + } +} +module.exports = LibraryScanner \ No newline at end of file diff --git a/server/scanner/MediaFileScanner.js b/server/scanner/MediaFileScanner.js index a644a6e0..aa7255bf 100644 --- a/server/scanner/MediaFileScanner.js +++ b/server/scanner/MediaFileScanner.js @@ -4,13 +4,18 @@ const AudioFile = require('../objects/files/AudioFile') const VideoFile = require('../objects/files/VideoFile') const prober = require('../utils/prober') -const toneProber = require('../utils/toneProber') const Logger = require('../Logger') const { LogLevel } = require('../utils/constants') class MediaFileScanner { constructor() { } + /** + * Get track and disc number from audio filename + * @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan + * @param {import('../objects/files/LibraryFile')} audioLibraryFile + * @returns {{trackNumber:number, discNumber:number}} + */ getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) { const { title, author, series, publishedYear } = mediaMetadataFromScan const { filename, path } = audioLibraryFile.metadata @@ -102,7 +107,12 @@ class MediaFileScanner { } } - // Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects + /** + * Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects + * @param {import('../objects/LibraryItem')} libraryItem + * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles + * @returns {Promise} + */ async executeMediaFileScans(libraryItem, mediaLibraryFiles) { const mediaType = libraryItem.mediaType @@ -206,11 +216,10 @@ class MediaFileScanner { /** * Scans media files for a library item and adds them as audio tracks and sets library item metadata - * @async - * @param {Array} mediaLibraryFiles - Media files for this library item - * @param {LibraryItem} libraryItem - * @param {LibraryScan} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs - * @return {Promise} True if any updates were made + * @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles + * @param {import('../objects/LibraryItem')} libraryItem + * @param {import('./LibraryScan')} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs + * @return {Promise} True if any updates were made */ async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) { const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index 557798bd..7b6d0d12 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -695,7 +695,7 @@ class Scanner { Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`) itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem) continue - } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some(checkFilepathIsAudioFile)) { + } else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(checkFilepathIsAudioFile)) { Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" has no audio files`) continue } diff --git a/server/utils/fileUtils.js b/server/utils/fileUtils.js index 7988857f..c74909c0 100644 --- a/server/utils/fileUtils.js +++ b/server/utils/fileUtils.js @@ -92,6 +92,12 @@ function bytesPretty(bytes, decimals = 0) { } module.exports.bytesPretty = bytesPretty +/** + * Get array of files inside dir + * @param {string} path + * @param {string} [relPathToReplace] + * @returns {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]} + */ async function recurseFiles(path, relPathToReplace = null) { path = filePathToPOSIX(path) if (!path.endsWith('/')) path = path + '/' diff --git a/server/utils/scandir.js b/server/utils/scandir.js index 30a6406a..08dc1fe4 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -22,9 +22,12 @@ function checkFilepathIsAudioFile(filepath) { } module.exports.checkFilepathIsAudioFile = checkFilepathIsAudioFile -// TODO: Function needs to be re-done -// Input: array of relative file paths -// Output: map of files grouped into potential item dirs +/** + * TODO: Function needs to be re-done + * @param {string} mediaType + * @param {string[]} paths array of relative file paths + * @returns {Record} map of files grouped into potential libarary item dirs + */ function groupFilesIntoLibraryItemPaths(mediaType, paths) { // Step 1: Clean path, Remove leading "/", Filter out non-media files in root dir var nonMediaFilePaths = [] @@ -97,8 +100,12 @@ function groupFilesIntoLibraryItemPaths(mediaType, paths) { } module.exports.groupFilesIntoLibraryItemPaths = groupFilesIntoLibraryItemPaths -// Input: array of relative file items (see recurseFiles) -// Output: map of files grouped into potential libarary item dirs +/** + * @param {string} mediaType + * @param {{name:string, path:string, dirpath:string, reldirpath:string, fullpath:string, extension:string, deep:number}[]} fileItems (see recurseFiles) + * @param {boolean} [audiobooksOnly=false] + * @returns {Record} map of files grouped into potential libarary item dirs + */ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly = false) { // Handle music where every audio file is a library item if (mediaType === 'music') { @@ -173,8 +180,15 @@ function groupFileItemsIntoLibraryItemDirs(mediaType, fileItems, audiobooksOnly }) return libraryItemGroup } +module.exports.groupFileItemsIntoLibraryItemDirs = groupFileItemsIntoLibraryItemDirs -function cleanFileObjects(libraryItemPath, files) { +/** + * Get LibraryFile from filepath + * @param {string} libraryItemPath + * @param {string[]} files + * @returns {import('../objects/files/LibraryFile')} + */ +function buildLibraryFile(libraryItemPath, files) { return Promise.all(files.map(async (file) => { const filePath = Path.posix.join(libraryItemPath, file) const newLibraryFile = new LibraryFile() @@ -182,6 +196,7 @@ function cleanFileObjects(libraryItemPath, files) { return newLibraryFile })) } +module.exports.buildLibraryFile = buildLibraryFile // Scan folder async function scanFolder(library, folder) { @@ -211,7 +226,7 @@ async function scanFolder(library, folder) { path: Path.posix.join(folderPath, libraryItemPath), relPath: libraryItemPath } - fileObjs = await cleanFileObjects(folderPath, [libraryItemPath]) + fileObjs = await buildLibraryFile(folderPath, [libraryItemPath]) isFile = true } else if (libraryItemPath === libraryItemGrouping[libraryItemPath]) { // Media file in root only get title @@ -222,11 +237,11 @@ async function scanFolder(library, folder) { path: Path.posix.join(folderPath, libraryItemPath), relPath: libraryItemPath } - fileObjs = await cleanFileObjects(folderPath, [libraryItemPath]) + fileObjs = await buildLibraryFile(folderPath, [libraryItemPath]) isFile = true } else { libraryItemData = getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath) - fileObjs = await cleanFileObjects(libraryItemData.path, libraryItemGrouping[libraryItemPath]) + fileObjs = await buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath]) } const libraryItemFolderStats = await getFileTimestampsWithIno(libraryItemData.path) @@ -365,6 +380,7 @@ function getDataFromMediaDir(libraryMediaType, folderPath, relPath) { return getPodcastDataFromDir(folderPath, relPath) } } +module.exports.getDataFromMediaDir = getDataFromMediaDir // Called from Scanner.js async function getLibraryItemFileData(libraryMediaType, folder, libraryItemPath, isSingleMediaItem) {