From 42ff3d831415ad78a7bcee75f97d912516cac97d Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 3 Sep 2023 17:51:58 -0500 Subject: [PATCH] Add new library item scanner --- server/Database.js | 5 + server/Server.js | 2 - server/controllers/LibraryController.js | 20 ++- server/controllers/LibraryItemController.js | 10 +- server/models/Library.js | 12 +- server/routers/ApiRouter.js | 1 - server/scanner/BookScanner.js | 132 +++++++++++--- server/scanner/LibraryItemScanner.js | 190 ++++++++++++++++++++ server/scanner/LibraryScanner.js | 114 +----------- server/scanner/ScanLogger.js | 70 ++++++++ server/utils/queries/libraryFilters.js | 19 +- 11 files changed, 421 insertions(+), 154 deletions(-) create mode 100644 server/scanner/LibraryItemScanner.js create mode 100644 server/scanner/ScanLogger.js diff --git a/server/Database.js b/server/Database.js index 8557591e..e63ae988 100644 --- a/server/Database.js +++ b/server/Database.js @@ -45,6 +45,11 @@ class Database { return this.models.library } + /** @type {typeof import('./models/LibraryFolder')} */ + get libraryFolderModel() { + return this.models.libraryFolder + } + /** @type {typeof import('./models/Author')} */ get authorModel() { return this.models.author diff --git a/server/Server.js b/server/Server.js index 47cf6851..ec6eb66c 100644 --- a/server/Server.js +++ b/server/Server.js @@ -16,7 +16,6 @@ const Logger = require('./Logger') const Auth = require('./Auth') const Watcher = require('./Watcher') const Scanner = require('./scanner/Scanner') -const LibraryScanner = require('./scanner/LibraryScanner') const Database = require('./Database') const SocketAuthority = require('./SocketAuthority') @@ -77,7 +76,6 @@ class Server { this.rssFeedManager = new RssFeedManager() this.scanner = new Scanner(this.coverManager, this.taskManager) - this.libraryScanner = new LibraryScanner(this.coverManager, this.taskManager) this.cronManager = new CronManager(this.scanner, this.podcastManager) // Routers diff --git a/server/controllers/LibraryController.js b/server/controllers/LibraryController.js index 140eb84f..bbd5536d 100644 --- a/server/controllers/LibraryController.js +++ b/server/controllers/LibraryController.js @@ -9,11 +9,13 @@ const libraryHelpers = require('../utils/libraryHelpers') const libraryItemsBookFilters = require('../utils/queries/libraryItemsBookFilters') const libraryItemFilters = require('../utils/queries/libraryItemFilters') const seriesFilters = require('../utils/queries/seriesFilters') +const fileUtils = require('../utils/fileUtils') const { sort, createNewSortInstance } = require('../libs/fastSort') const naturalSort = createNewSortInstance({ comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare }) +const LibraryScanner = require('../scanner/LibraryScanner') const Database = require('../Database') const libraryFilters = require('../utils/queries/libraryFilters') const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters') @@ -33,7 +35,7 @@ class LibraryController { // Validate folder paths exist or can be created & resolve rel paths // returns 400 if a folder fails to access newLibraryPayload.folders = newLibraryPayload.folders.map(f => { - f.fullPath = Path.resolve(f.fullPath) + f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath)) return f }) for (const folder of newLibraryPayload.folders) { @@ -87,7 +89,7 @@ class LibraryController { async findOne(req, res) { const includeArray = (req.query.include || '').split(',') if (includeArray.includes('filterdata')) { - const filterdata = await libraryFilters.getFilterData(req.library) + const filterdata = await libraryFilters.getFilterData(req.library.mediaType, req.library.id) return res.json({ filterdata, @@ -119,7 +121,7 @@ class LibraryController { const newFolderPaths = [] req.body.folders = req.body.folders.map(f => { if (!f.id) { - f.fullPath = Path.resolve(f.fullPath) + f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath)) newFolderPaths.push(f.fullPath) } return f @@ -670,7 +672,7 @@ class LibraryController { * @param {import('express').Response} res */ async getLibraryFilterData(req, res) { - const filterData = await libraryFilters.getFilterData(req.library) + const filterData = await libraryFilters.getFilterData(req.library.mediaType, req.library.id) res.json(filterData) } @@ -976,9 +978,13 @@ class LibraryController { forceRescan: req.query.force == 1 } res.sendStatus(200) - await this.scanner.scan(req.library, options) - // TODO: New library scanner - // await this.libraryScanner.scan(req.library, options) + if (req.library.mediaType === 'podcast') { + // TODO: New library scanner for podcasts + await this.scanner.scan(req.library, options) + } else { + await LibraryScanner.scan(req.library, options) + } + await Database.resetLibraryIssuesFilterData(req.library.id) Logger.info('[LibraryController] Scan complete') } diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js index 657c3615..761fc6ed 100644 --- a/server/controllers/LibraryItemController.js +++ b/server/controllers/LibraryItemController.js @@ -8,6 +8,7 @@ const zipHelpers = require('../utils/zipHelpers') const { reqSupportsWebp } = require('../utils/index') const { ScanResult } = require('../utils/constants') const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils') +const LibraryItemScanner = require('../scanner/LibraryItemScanner') class LibraryItemController { constructor() { } @@ -475,7 +476,14 @@ class LibraryItemController { return res.sendStatus(500) } - const result = await this.scanner.scanLibraryItemByRequest(req.libraryItem) + let result = 0 + if (req.libraryItem.isPodcast) { + // TODO: New library item scanner for podcast + result = await this.scanner.scanLibraryItemByRequest(req.libraryItem) + } else { + result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id) + } + await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId) res.json({ result: Object.keys(ScanResult).find(key => ScanResult[key] == result) diff --git a/server/models/Library.js b/server/models/Library.js index 37ece3b5..ae4807e7 100644 --- a/server/models/Library.js +++ b/server/models/Library.js @@ -2,6 +2,16 @@ const { DataTypes, Model } = require('sequelize') const Logger = require('../Logger') const oldLibrary = require('../objects/Library') +/** + * @typedef LibrarySettingsObject + * @property {number} coverAspectRatio BookCoverAspectRatio + * @property {boolean} disableWatcher + * @property {boolean} skipMatchingMediaWithAsin + * @property {boolean} skipMatchingMediaWithIsbn + * @property {string} autoScanCronExpression + * @property {boolean} audiobooksOnly + * @property {boolean} hideSingleBookSeries Do not show series that only have 1 book + */ class Library extends Model { constructor(values, options) { @@ -23,7 +33,7 @@ class Library extends Model { this.lastScan /** @type {string} */ this.lastScanVersion - /** @type {Object} */ + /** @type {LibrarySettingsObject} */ this.settings /** @type {Object} */ this.extraData diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 14641e00..c94f3ea4 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -41,7 +41,6 @@ class ApiRouter { constructor(Server) { this.auth = Server.auth this.scanner = Server.scanner - this.libraryScanner = Server.libraryScanner this.playbackSessionManager = Server.playbackSessionManager this.abMergeManager = Server.abMergeManager this.backupManager = Server.backupManager diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index cc55b7e7..b23a92be 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -1,6 +1,6 @@ const uuidv4 = require("uuid").v4 const Path = require('path') -const { Sequelize } = require('sequelize') +const sequelize = require('sequelize') const { LogLevel } = require('../utils/constants') const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index') const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata') @@ -14,6 +14,7 @@ const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../ const AudioFile = require('../objects/files/AudioFile') const CoverManager = require('../managers/CoverManager') const LibraryFile = require('../objects/files/LibraryFile') +const SocketAuthority = require('../SocketAuthority') const fsExtra = require("../libs/fsExtra") /** @@ -45,10 +46,11 @@ class BookScanner { /** * @param {import('../models/LibraryItem')} existingLibraryItem * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings * @param {import('./LibraryScan')} libraryScan - * @returns {import('../models/LibraryItem')} + * @returns {Promise} */ - async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) { + async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) { /** @type {import('../models/Book')} */ const media = await existingLibraryItem.getMedia({ include: [ @@ -146,13 +148,13 @@ class BookScanner { } // Check if ebook was removed - if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) { + if (media.ebookFile && (librarySettings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) { media.ebookFile = null hasMediaChanges = true } // Check if ebook is not set and ebooks were found - if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) { + if (!media.ebookFile && !librarySettings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) { // Prefer to use an epub ebook then fallback to the first ebook found let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0] @@ -179,7 +181,7 @@ class BookScanner { // Check for authors added for (const authorName of bookMetadata.authors) { if (!media.authors.some(au => au.name === authorName)) { - const existingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName) + const existingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName) if (existingAuthor) { await Database.bookAuthorModel.create({ bookId: media.id, @@ -191,10 +193,10 @@ class BookScanner { const newAuthor = await Database.authorModel.create({ name: authorName, lastFirst: parseNameString.nameToLastFirst(authorName), - libraryId: libraryScan.libraryId + libraryId: libraryItemData.libraryId }) await media.addAuthor(newAuthor) - Database.addAuthorToFilterData(libraryScan.libraryId, newAuthor.name, newAuthor.id) + Database.addAuthorToFilterData(libraryItemData.libraryId, newAuthor.name, newAuthor.id) libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`) authorsUpdated = true } @@ -213,7 +215,7 @@ class BookScanner { // Check for series added for (const seriesObj of bookMetadata.series) { if (!media.series.some(se => se.name === seriesObj.name)) { - const existingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name) + const existingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name) if (existingSeries) { await Database.bookSeriesModel.create({ bookId: media.id, @@ -226,10 +228,10 @@ class BookScanner { const newSeries = await Database.seriesModel.create({ name: seriesObj.name, nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name), - libraryId: libraryScan.libraryId + libraryId: libraryItemData.libraryId }) - await media.addSeries(newSeries) - Database.addSeriesToFilterData(libraryScan.libraryId, newSeries.name, newSeries.id) + await media.addSeries(newSeries, { through: { sequence: seriesObj.sequence } }) + Database.addSeriesToFilterData(libraryItemData.libraryId, newSeries.name, newSeries.id) libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`) seriesUpdated = true } @@ -304,7 +306,7 @@ class BookScanner { media.authors = await media.getAuthors({ joinTableAttributes: ['createdAt'], order: [ - Sequelize.literal(`bookAuthor.createdAt ASC`) + sequelize.literal(`bookAuthor.createdAt ASC`) ] }) } @@ -312,7 +314,7 @@ class BookScanner { media.series = await media.getSeries({ joinTableAttributes: ['sequence', 'createdAt'], order: [ - Sequelize.literal(`bookSeries.createdAt ASC`) + sequelize.literal(`bookSeries.createdAt ASC`) ] }) } @@ -356,16 +358,17 @@ class BookScanner { /** * * @param {import('./LibraryItemScanData')} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings * @param {import('./LibraryScan')} libraryScan * @returns {import('../models/LibraryItem')} */ - async scanNewBookLibraryItem(libraryItemData, libraryScan) { + async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) { // Scan audio files found - let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryScan.libraryMediaType, libraryItemData, libraryItemData.audioLibraryFiles) + let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryItemData.mediaType, libraryItemData, libraryItemData.audioLibraryFiles) scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles) // Find ebook file (prefer epub) - let ebookLibraryFile = libraryScan.library.settings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0] + let ebookLibraryFile = librarySettings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0] // Do not add library items that have no valid audio files and no ebook file if (!ebookLibraryFile && !scannedAudioFiles.length) { @@ -393,7 +396,7 @@ class BookScanner { } if (bookMetadata.authors.length) { for (const authorName of bookMetadata.authors) { - const matchingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName) + const matchingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName) if (matchingAuthor) { bookObject.bookAuthors.push({ authorId: matchingAuthor.id @@ -402,7 +405,7 @@ class BookScanner { // New author bookObject.bookAuthors.push({ author: { - libraryId: libraryScan.libraryId, + libraryId: libraryItemData.libraryId, name: authorName, lastFirst: parseNameString.nameToLastFirst(authorName) } @@ -413,7 +416,7 @@ class BookScanner { if (bookMetadata.series.length) { for (const seriesObj of bookMetadata.series) { if (!seriesObj.name) continue - const matchingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name) + const matchingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name) if (matchingSeries) { bookObject.bookSeries.push({ seriesId: matchingSeries.id, @@ -425,7 +428,7 @@ class BookScanner { series: { name: seriesObj.name, nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name), - libraryId: libraryScan.libraryId + libraryId: libraryItemData.libraryId } }) } @@ -476,22 +479,22 @@ class BookScanner { if (libraryItem.book.bookSeries?.length) { for (const bs of libraryItem.book.bookSeries) { if (bs.series) { - Database.addSeriesToFilterData(libraryScan.libraryId, bs.series.name, bs.series.id) + Database.addSeriesToFilterData(libraryItemData.libraryId, bs.series.name, bs.series.id) } } } if (libraryItem.book.bookAuthors?.length) { for (const ba of libraryItem.book.bookAuthors) { if (ba.author) { - Database.addAuthorToFilterData(libraryScan.libraryId, ba.author.name, ba.author.id) + Database.addAuthorToFilterData(libraryItemData.libraryId, ba.author.name, ba.author.id) } } } - Database.addNarratorsToFilterData(libraryScan.libraryId, libraryItem.book.narrators) - Database.addGenresToFilterData(libraryScan.libraryId, libraryItem.book.genres) - Database.addTagsToFilterData(libraryScan.libraryId, libraryItem.book.tags) - Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher) - Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language) + Database.addNarratorsToFilterData(libraryItemData.libraryId, libraryItem.book.narrators) + Database.addGenresToFilterData(libraryItemData.libraryId, libraryItem.book.genres) + Database.addTagsToFilterData(libraryItemData.libraryId, libraryItem.book.tags) + Database.addPublisherToFilterData(libraryItemData.libraryId, libraryItem.book.publisher) + Database.addLanguageToFilterData(libraryItemData.libraryId, libraryItem.book.language) // Load for emitting to client libraryItem.media = await libraryItem.getMedia({ @@ -949,5 +952,78 @@ class BookScanner { }) } } + + /** + * Check authors that were removed from a book and remove them if they no longer have any books + * keep authors without books that have a asin, description or imagePath + * @param {string} libraryId + * @param {import('./ScanLogger')} scanLogger + * @returns {Promise} + */ + async checkAuthorsRemovedFromBooks(libraryId, scanLogger) { + const bookAuthorsToRemove = (await Database.authorModel.findAll({ + where: [ + { + id: scanLogger.authorsRemovedFromBooks, + asin: { + [sequelize.Op.or]: [null, ""] + }, + description: { + [sequelize.Op.or]: [null, ""] + }, + imagePath: { + [sequelize.Op.or]: [null, ""] + } + }, + sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0) + ], + attributes: ['id'], + raw: true + })).map(au => au.id) + if (bookAuthorsToRemove.length) { + await Database.authorModel.destroy({ + where: { + id: bookAuthorsToRemove + } + }) + bookAuthorsToRemove.forEach((authorId) => { + Database.removeAuthorFromFilterData(libraryId, authorId) + // TODO: Clients were expecting full author in payload but its unnecessary + SocketAuthority.emitter('author_removed', { id: authorId, libraryId }) + }) + scanLogger.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`) + } + } + + /** + * Check series that were removed from books and remove them if they no longer have any books + * @param {string} libraryId + * @param {import('./ScanLogger')} scanLogger + * @returns {Promise} + */ + async checkSeriesRemovedFromBooks(libraryId, scanLogger) { + const bookSeriesToRemove = (await Database.seriesModel.findAll({ + where: [ + { + id: scanLogger.seriesRemovedFromBooks + }, + sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0) + ], + attributes: ['id'], + raw: true + })).map(se => se.id) + if (bookSeriesToRemove.length) { + await Database.seriesModel.destroy({ + where: { + id: bookSeriesToRemove + } + }) + bookSeriesToRemove.forEach((seriesId) => { + Database.removeSeriesFromFilterData(libraryId, seriesId) + SocketAuthority.emitter('series_removed', { id: seriesId, libraryId }) + }) + scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`) + } + } } module.exports = new BookScanner() \ No newline at end of file diff --git a/server/scanner/LibraryItemScanner.js b/server/scanner/LibraryItemScanner.js new file mode 100644 index 00000000..f71e49a6 --- /dev/null +++ b/server/scanner/LibraryItemScanner.js @@ -0,0 +1,190 @@ +const Path = require('path') +const { LogLevel, ScanResult } = require('../utils/constants') + +const fileUtils = require('../utils/fileUtils') +const scanUtils = require('../utils/scandir') +const libraryFilters = require('../utils/queries/libraryFilters') +const Database = require('../Database') +const LibraryScan = require('./LibraryScan') +const LibraryItemScanData = require('./LibraryItemScanData') +const BookScanner = require('./BookScanner') +const ScanLogger = require('./ScanLogger') +const LibraryItem = require('../models/LibraryItem') +const LibraryFile = require('../objects/files/LibraryFile') +const SocketAuthority = require('../SocketAuthority') + +class LibraryItemScanner { + constructor() { } + + /** + * Scan single library item + * + * @param {string} libraryItemId + * @returns {number} ScanResult + */ + async scanLibraryItem(libraryItemId) { + // TODO: Add task manager + const libraryItem = await Database.libraryItemModel.findByPk(libraryItemId) + if (!libraryItem) { + Logger.error(`[LibraryItemScanner] Library item not found "${libraryItemId}"`) + return ScanResult.NOTHING + } + + const library = await Database.libraryModel.findByPk(libraryItem.libraryId, { + include: { + model: Database.libraryFolderModel, + where: { + id: libraryItem.libraryFolderId + } + } + }) + if (!library) { + Logger.error(`[LibraryItemScanner] Library "${libraryItem.libraryId}" not found for library item "${libraryItem.id}"`) + return ScanResult.NOTHING + } + + // Make sure library filter data is set + // this is used to check for existing authors & series + await libraryFilters.getFilterData(library.mediaType, library.id) + + const scanLogger = new ScanLogger() + scanLogger.verbose = true + scanLogger.setData('libraryItem', libraryItemId) + + const libraryItemPath = fileUtils.filePathToPOSIX(libraryItem.path) + const folder = library.libraryFolders[0] + const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false) + + if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) { + if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) { + const expandedLibraryItem = await this.rescanLibraryItem(libraryItem, libraryItemScanData, library.settings, scanLogger) + const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem) + SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded()) + + await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger) + } else { + // TODO: Temporary while using old model to socket emit + const oldLibraryItem = await Database.libraryItemModel.getOldById(libraryItem.id) + SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded()) + } + + return ScanResult.UPDATED + } + return ScanResult.UPTODATE + } + + /** + * Remove empty authors and series + * @param {string} libraryId + * @param {ScanLogger} scanLogger + * @returns {Promise} + */ + async checkAuthorsAndSeriesRemovedFromBooks(libraryId, scanLogger) { + if (scanLogger.authorsRemovedFromBooks.length) { + await BookScanner.checkAuthorsRemovedFromBooks(libraryId, scanLogger) + } + if (scanLogger.seriesRemovedFromBooks.length) { + await BookScanner.checkSeriesRemovedFromBooks(libraryId, scanLogger) + } + } + + /** + * + * @param {string} libraryItemPath + * @param {import('../models/Library')} library + * @param {import('../models/LibraryFolder')} folder + * @param {boolean} isSingleMediaItem + * @returns {Promise} + */ + async getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem) { + const libraryFolderPath = fileUtils.filePathToPOSIX(folder.path) + const libraryItemDir = libraryItemPath.replace(libraryFolderPath, '').slice(1) + + let libraryItemData = {} + + let fileItems = [] + + if (isSingleMediaItem) { // Single media item in root of folder + fileItems = [ + { + fullpath: libraryItemPath, + path: libraryItemDir // actually the relPath (only filename here) + } + ] + libraryItemData = { + path: libraryItemPath, // full path + relPath: libraryItemDir, // only filename + mediaMetadata: { + title: Path.basename(libraryItemDir, Path.extname(libraryItemDir)) + } + } + } else { + fileItems = await fileUtils.recurseFiles(libraryItemPath) + libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, libraryFolderPath, libraryItemDir) + } + + const libraryFiles = [] + for (let i = 0; i < fileItems.length; i++) { + const fileItem = fileItems[i] + const newLibraryFile = new LibraryFile() + // fileItem.path is the relative path + await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path) + libraryFiles.push(newLibraryFile) + } + + const libraryItemStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path) + return new LibraryItemScanData({ + libraryFolderId: folder.id, + libraryId: library.id, + mediaType: library.mediaType, + ino: libraryItemStats.ino, + mtimeMs: libraryItemStats.mtimeMs || 0, + ctimeMs: libraryItemStats.ctimeMs || 0, + birthtimeMs: libraryItemStats.birthtimeMs || 0, + path: libraryItemData.path, + relPath: libraryItemData.relPath, + isFile: isSingleMediaItem, + mediaMetadata: libraryItemData.mediaMetadata || null, + libraryFiles + }) + } + + /** + * + * @param {import('../models/LibraryItem')} existingLibraryItem + * @param {LibraryItemScanData} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings + * @param {LibraryScan} libraryScan + * @returns {Promise} + */ + async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) { + if (existingLibraryItem.mediaType === 'book') { + const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) + return libraryItem + } else { + // TODO: Scan updated podcast + return null + } + } + + /** + * + * @param {LibraryItemScanData} libraryItemData + * @param {import('../models/Library').LibrarySettingsObject} librarySettings + * @param {LibraryScan} libraryScan + * @returns {Promise} + */ + async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) { + if (libraryScan.libraryMediaType === 'book') { + const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) + if (newLibraryItem) { + libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`) + } + return newLibraryItem + } else { + // TODO: Scan new podcast + return null + } + } +} +module.exports = new LibraryItemScanner() \ No newline at end of file diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index a45d7410..1412e211 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -7,18 +7,15 @@ const Database = require('../Database') const fs = require('../libs/fsExtra') const fileUtils = require('../utils/fileUtils') const scanUtils = require('../utils/scandir') -const { ScanResult, LogLevel } = require('../utils/constants') +const { LogLevel } = require('../utils/constants') const libraryFilters = require('../utils/queries/libraryFilters') +const LibraryItemScanner = require('./LibraryItemScanner') const ScanOptions = require('./ScanOptions') const LibraryScan = require('./LibraryScan') const LibraryItemScanData = require('./LibraryItemScanData') -const BookScanner = require('./BookScanner') class LibraryScanner { - constructor(coverManager, taskManager) { - this.coverManager = coverManager - this.taskManager = taskManager - + constructor() { this.cancelLibraryScan = {} this.librariesScanning = [] } @@ -93,7 +90,7 @@ class LibraryScanner { async scanLibrary(libraryScan) { // Make sure library filter data is set // this is used to check for existing authors & series - await libraryFilters.getFilterData(libraryScan.library) + await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId) /** @type {LibraryItemScanData[]} */ let libraryItemDataFound = [] @@ -151,7 +148,7 @@ class LibraryScanner { if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) { libraryScan.resultsUpdated++ if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) { - const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) + const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan) const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem) oldLibraryItemsUpdated.push(oldLibraryItem) } else { @@ -177,68 +174,8 @@ class LibraryScanner { SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded())) } - // Check authors that were removed from a book and remove them if they no longer have any books - // keep authors without books that have a asin, description or imagePath - if (libraryScan.authorsRemovedFromBooks.length) { - const bookAuthorsToRemove = (await Database.authorModel.findAll({ - where: [ - { - id: libraryScan.authorsRemovedFromBooks, - asin: { - [sequelize.Op.or]: [null, ""] - }, - description: { - [sequelize.Op.or]: [null, ""] - }, - imagePath: { - [sequelize.Op.or]: [null, ""] - } - }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0) - ], - attributes: ['id'], - raw: true - })).map(au => au.id) - if (bookAuthorsToRemove.length) { - await Database.authorModel.destroy({ - where: { - id: bookAuthorsToRemove - } - }) - bookAuthorsToRemove.forEach((authorId) => { - Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId) - // TODO: Clients were expecting full author in payload but its unnecessary - SocketAuthority.emitter('author_removed', { id: authorId, libraryId: libraryScan.libraryId }) - }) - libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`) - } - } - - // Check series that were removed from books and remove them if they no longer have any books - if (libraryScan.seriesRemovedFromBooks.length) { - const bookSeriesToRemove = (await Database.seriesModel.findAll({ - where: [ - { - id: libraryScan.seriesRemovedFromBooks - }, - sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0) - ], - attributes: ['id'], - raw: true - })).map(se => se.id) - if (bookSeriesToRemove.length) { - await Database.seriesModel.destroy({ - where: { - id: bookSeriesToRemove - } - }) - bookSeriesToRemove.forEach((seriesId) => { - Database.removeSeriesFromFilterData(libraryScan.libraryId, seriesId) - SocketAuthority.emitter('series_removed', { id: seriesId, libraryId: libraryScan.libraryId }) - }) - libraryScan.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`) - } - } + // Authors and series that were removed from books should be removed if they are now empty + await LibraryItemScanner.checkAuthorsAndSeriesRemovedFromBooks(libraryScan.libraryId, libraryScan) // Update missing library items if (libraryItemIdsMissing.length) { @@ -260,7 +197,7 @@ class LibraryScanner { if (libraryItemDataFound.length) { let newOldLibraryItems = [] for (const libraryItemData of libraryItemDataFound) { - const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan) + const newLibraryItem = await LibraryItemScanner.scanNewLibraryItem(libraryItemData, libraryScan.library.settings, libraryScan) if (newLibraryItem) { const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem) newOldLibraryItems.push(oldLibraryItem) @@ -353,38 +290,5 @@ class LibraryScanner { } return items } - - /** - * - * @param {import('../models/LibraryItem')} existingLibraryItem - * @param {LibraryItemScanData} libraryItemData - * @param {LibraryScan} libraryScan - */ - async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) { - if (existingLibraryItem.mediaType === 'book') { - const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) - return libraryItem - } else { - // TODO: Scan updated podcast - } - } - - /** - * - * @param {LibraryItemScanData} libraryItemData - * @param {LibraryScan} libraryScan - */ - async scanNewLibraryItem(libraryItemData, libraryScan) { - if (libraryScan.libraryMediaType === 'book') { - const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan) - if (newLibraryItem) { - libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`) - } - return newLibraryItem - } else { - // TODO: Scan new podcast - return null - } - } } -module.exports = LibraryScanner \ No newline at end of file +module.exports = new LibraryScanner() \ No newline at end of file diff --git a/server/scanner/ScanLogger.js b/server/scanner/ScanLogger.js new file mode 100644 index 00000000..90f8a28b --- /dev/null +++ b/server/scanner/ScanLogger.js @@ -0,0 +1,70 @@ +const uuidv4 = require("uuid").v4 +const Logger = require('../Logger') +const { LogLevel } = require('../utils/constants') + +class ScanLogger { + constructor() { + this.id = null + this.type = null + this.name = null + this.verbose = false + + this.startedAt = null + this.finishedAt = null + this.elapsed = null + + /** @type {string[]} */ + this.authorsRemovedFromBooks = [] + /** @type {string[]} */ + this.seriesRemovedFromBooks = [] + + this.logs = [] + } + + toJSON() { + return { + id: this.id, + type: this.type, + name: this.name, + startedAt: this.startedAt, + finishedAt: this.finishedAt, + elapsed: this.elapsed + } + } + + setData(type, name) { + this.id = uuidv4() + this.type = type + this.name = name + this.startedAt = Date.now() + } + + setComplete() { + this.finishedAt = Date.now() + this.elapsed = this.finishedAt - this.startedAt + } + + getLogLevelString(level) { + for (const key in LogLevel) { + if (LogLevel[key] === level) { + return key + } + } + return 'UNKNOWN' + } + + addLog(level, ...args) { + const logObj = { + timestamp: (new Date()).toISOString(), + message: args.join(' '), + levelName: this.getLogLevelString(level), + level + } + + if (this.verbose) { + Logger.debug(`[Scan] "${this.name}":`, ...args) + } + this.logs.push(logObj) + } +} +module.exports = ScanLogger \ No newline at end of file diff --git a/server/utils/queries/libraryFilters.js b/server/utils/queries/libraryFilters.js index b8faf025..0ac5c219 100644 --- a/server/utils/queries/libraryFilters.js +++ b/server/utils/queries/libraryFilters.js @@ -390,11 +390,12 @@ module.exports = { /** * Get filter data used in filter menus - * @param {import('../../objects/Library')} oldLibrary + * @param {string} mediaType + * @param {string} libraryId * @returns {Promise} */ - async getFilterData(oldLibrary) { - const cachedFilterData = Database.libraryFilterData[oldLibrary.id] + async getFilterData(mediaType, libraryId) { + const cachedFilterData = Database.libraryFilterData[libraryId] if (cachedFilterData) { const cacheElapsed = Date.now() - cachedFilterData.loadedAt // Cache library filters for 30 mins @@ -416,13 +417,13 @@ module.exports = { numIssues: 0 } - if (oldLibrary.isPodcast) { + if (mediaType === 'podcast') { const podcasts = await Database.podcastModel.findAll({ include: { model: Database.libraryItemModel, attributes: [], where: { - libraryId: oldLibrary.id + libraryId: libraryId } }, attributes: ['tags', 'genres'] @@ -441,7 +442,7 @@ module.exports = { model: Database.libraryItemModel, attributes: ['isMissing', 'isInvalid'], where: { - libraryId: oldLibrary.id + libraryId: libraryId } }, attributes: ['tags', 'genres', 'publisher', 'narrators', 'language'] @@ -463,7 +464,7 @@ module.exports = { const series = await Database.seriesModel.findAll({ where: { - libraryId: oldLibrary.id + libraryId: libraryId }, attributes: ['id', 'name'] }) @@ -471,7 +472,7 @@ module.exports = { const authors = await Database.authorModel.findAll({ where: { - libraryId: oldLibrary.id + libraryId: libraryId }, attributes: ['id', 'name'] }) @@ -486,7 +487,7 @@ module.exports = { data.publishers = naturalSort([...data.publishers]).asc() data.languages = naturalSort([...data.languages]).asc() data.loadedAt = Date.now() - Database.libraryFilterData[oldLibrary.id] = data + Database.libraryFilterData[libraryId] = data Logger.debug(`Loaded filterdata in ${((Date.now() - start) / 1000).toFixed(2)}s`) return data