From e63aab95d889663cb8c5ce9fc828ba48516f8d6c Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 3 Sep 2023 15:14:58 -0500 Subject: [PATCH] Update new library scanner to handle metadata file changes --- server/Database.js | 12 +- server/objects/LibraryItem.js | 72 ++++--- server/scanner/BookScanner.js | 185 +++++++++++++++--- server/scanner/LibraryItemScanData.js | 2 +- server/scanner/LibraryScanner.js | 2 - .../utils/generators/abmetadataGenerator.js | 61 +++++- 6 files changed, 265 insertions(+), 69 deletions(-) diff --git a/server/Database.js b/server/Database.js index e131fb11..8557591e 100644 --- a/server/Database.js +++ b/server/Database.js @@ -325,9 +325,9 @@ class Database { return Promise.all(oldUsers.map(u => this.updateUser(u))) } - async removeUser(userId) { + removeUser(userId) { if (!this.sequelize) return false - await this.models.user.removeById(userId) + return this.models.user.removeById(userId) } upsertMediaProgress(oldMediaProgress) { @@ -345,9 +345,9 @@ class Database { return Promise.all(oldBooks.map(oldBook => this.models.book.saveFromOld(oldBook))) } - async createLibrary(oldLibrary) { + createLibrary(oldLibrary) { if (!this.sequelize) return false - await this.models.library.createFromOld(oldLibrary) + return this.models.library.createFromOld(oldLibrary) } updateLibrary(oldLibrary) { @@ -355,9 +355,9 @@ class Database { return this.models.library.updateFromOld(oldLibrary) } - async removeLibrary(libraryId) { + removeLibrary(libraryId) { if (!this.sequelize) return false - await this.models.library.removeById(libraryId) + return this.models.library.removeById(libraryId) } createBulkCollectionBooks(collectionBooks) { diff --git a/server/objects/LibraryItem.js b/server/objects/LibraryItem.js index d7b2acaa..1f8d6c47 100644 --- a/server/objects/LibraryItem.js +++ b/server/objects/LibraryItem.js @@ -10,7 +10,7 @@ const Podcast = require('./mediaTypes/Podcast') const Video = require('./mediaTypes/Video') const Music = require('./mediaTypes/Music') const { areEquivalent, copyValue, cleanStringForSearch } = require('../utils/index') -const { filePathToPOSIX } = require('../utils/fileUtils') +const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils') class LibraryItem { constructor(libraryItem = null) { @@ -40,6 +40,7 @@ class LibraryItem { this.mediaType = null this.media = null + /** @type {LibraryFile[]} */ this.libraryFiles = [] if (libraryItem) { @@ -525,19 +526,20 @@ class LibraryItem { /** * Save metadata.json/metadata.abs file - * @returns {boolean} true if saved + * @returns {Promise} null if not saved */ async saveMetadata() { - if (this.mediaType === 'video' || this.mediaType === 'music') return + if (this.isSavingMetadata) return null - if (this.isSavingMetadata) return this.isSavingMetadata = true let metadataPath = Path.join(global.MetadataPath, 'items', this.id) - if (global.ServerSettings.storeMetadataWithItem && !this.isFile) { + let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem + if (storeMetadataWithItem && !this.isFile) { metadataPath = this.path } else { // Make sure metadata book dir exists + storeMetadataWithItem = false await fs.ensureDir(metadataPath) } @@ -552,20 +554,29 @@ class LibraryItem { } return fs.writeFile(metadataFilePath, JSON.stringify(this.media.toJSONForMetadataFile(), null, 2)).then(async () => { - this.isSavingMetadata = false // Add metadata.json to libraryFiles array if it is new - if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) - this.libraryFiles.push(newLibraryFile) + let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + metadataLibraryFile = new LibraryFile() + await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + this.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } } Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`) - return true + return metadataLibraryFile }).catch((error) => { - this.isSavingMetadata = false Logger.error(`[LibraryItem] Failed to save json file at "${metadataFilePath}"`, error) - return false + return null + }).finally(() => { + this.isSavingMetadata = false }) } else { // Remove metadata.json if it exists @@ -576,19 +587,30 @@ class LibraryItem { } return abmetadataGenerator.generate(this, metadataFilePath).then(async (success) => { - this.isSavingMetadata = false - if (!success) Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`) - else { - // Add metadata.abs to libraryFiles array if it is new - if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) { - const newLibraryFile = new LibraryFile() - await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) - this.libraryFiles.push(newLibraryFile) - } - - Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`) + if (!success) { + Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`) + return null } - return success + // Add metadata.abs to libraryFiles array if it is new + let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + metadataLibraryFile = new LibraryFile() + await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + this.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + + Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`) + return metadataLibraryFile + }).finally(() => { + this.isSavingMetadata = false }) } } diff --git a/server/scanner/BookScanner.js b/server/scanner/BookScanner.js index be408a46..cc55b7e7 100644 --- a/server/scanner/BookScanner.js +++ b/server/scanner/BookScanner.js @@ -1,4 +1,5 @@ const uuidv4 = require("uuid").v4 +const Path = require('path') const { Sequelize } = require('sequelize') const { LogLevel } = require('../utils/constants') const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index') @@ -9,9 +10,10 @@ const parseNameString = require('../utils/parsers/parseNameString') const globals = require('../utils/globals') const AudioFileScanner = require('./AudioFileScanner') const Database = require('../Database') -const { readTextFile } = require('../utils/fileUtils') +const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils') const AudioFile = require('../objects/files/AudioFile') const CoverManager = require('../managers/CoverManager') +const LibraryFile = require('../objects/files/LibraryFile') const fsExtra = require("../libs/fsExtra") /** @@ -161,26 +163,6 @@ class BookScanner { hasMediaChanges = true } - // Check/update the isSupplementary flag on libraryFiles for the LibraryItem - let libraryItemUpdated = false - for (const libraryFile of existingLibraryItem.libraryFiles) { - if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) { - if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) { - if (libraryFile.isSupplementary !== false) { - libraryFile.isSupplementary = false - libraryItemUpdated = true - } - } else if (libraryFile.isSupplementary !== true) { - libraryFile.isSupplementary = true - libraryItemUpdated = true - } - } - } - if (libraryItemUpdated) { - existingLibraryItem.changed('libraryFiles', true) - await existingLibraryItem.save() - } - // TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this // TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan) @@ -317,11 +299,6 @@ class BookScanner { } } - // Save Book changes to db - if (hasMediaChanges) { - await media.save() - } - // Load authors/series again if updated (for sending back to client) if (authorsUpdated) { media.authors = await media.getAuthors({ @@ -340,10 +317,39 @@ class BookScanner { }) } + existingLibraryItem.media = media + + let libraryItemUpdated = false + + // Save Book changes to db + if (hasMediaChanges) { + await media.save() + await this.saveMetadataFile(existingLibraryItem, libraryScan) + libraryItemUpdated = global.ServerSettings.storeMetadataWithItem && !existingLibraryItem.isFile + } + + // Check/update the isSupplementary flag on libraryFiles for the LibraryItem + for (const libraryFile of existingLibraryItem.libraryFiles) { + if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) { + if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) { + if (libraryFile.isSupplementary !== false) { + libraryFile.isSupplementary = false + libraryItemUpdated = true + } + } else if (libraryFile.isSupplementary !== true) { + libraryFile.isSupplementary = true + libraryItemUpdated = true + } + } + } + if (libraryItemUpdated) { + existingLibraryItem.changed('libraryFiles', true) + await existingLibraryItem.save() + } + libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved) libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved) - existingLibraryItem.media = media return existingLibraryItem } @@ -509,6 +515,12 @@ class BookScanner { ] }) + await this.saveMetadataFile(libraryItem, libraryScan) + if (global.ServerSettings.storeMetadataWithItem && !libraryItem.isFile) { + libraryItem.changed('libraryFiles', true) + await libraryItem.save() + } + return libraryItem } @@ -691,7 +703,7 @@ class BookScanner { const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null if (metadataText) { - libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.relPath}" - preferring`) + libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`) let abMetadata = null if (!!libraryItemData.metadataJsonLibraryFile) { abMetadata = abmetadataGenerator.parseJson(metadataText) @@ -707,7 +719,7 @@ class BookScanner { bookMetadata.chapters = abMetadata.chapters } for (const key in abMetadata.metadata) { - if (bookMetadata[key] === undefined || abMetadata.metadata[key] === undefined) continue + if (abMetadata.metadata[key] === undefined) continue bookMetadata[key] = abMetadata.metadata[key] } } @@ -803,7 +815,7 @@ class BookScanner { // Build chapters from audio files let currChapterId = 0 let currStartTime = 0 - includedAudioFiles.forEach((file) => { + audioFiles.forEach((file) => { if (file.duration) { let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}` @@ -824,5 +836,118 @@ class BookScanner { } return chapters } + + /** + * + * @param {import('../models/LibraryItem')} libraryItem + * @param {import('./LibraryScan')} libraryScan + * @returns {Promise} + */ + async saveMetadataFile(libraryItem, libraryScan) { + let metadataPath = Path.join(global.MetadataPath, 'items', libraryItem.id) + let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem + if (storeMetadataWithItem && !libraryItem.isFile) { + metadataPath = libraryItem.path + } else { + // Make sure metadata book dir exists + storeMetadataWithItem = false + await fsExtra.ensureDir(metadataPath) + } + + const metadataFileFormat = global.ServerSettings.metadataFileFormat + const metadataFilePath = Path.join(metadataPath, `metadata.${metadataFileFormat}`) + if (metadataFileFormat === 'json') { + // Remove metadata.abs if it exists + if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.abs`))) { + libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.abs for item "${libraryItem.media.title}"`) + await fsExtra.remove(Path.join(metadataPath, `metadata.abs`)) + libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.abs`))) + } + + // TODO: Update to not use `metadata` so it fits the updated model + const jsonObject = { + tags: libraryItem.media.tags || [], + chapters: libraryItem.media.chapters?.map(c => ({ ...c })) || [], + metadata: { + title: libraryItem.media.title, + subtitle: libraryItem.media.subtitle, + authors: libraryItem.media.authors.map(a => a.name), + narrators: libraryItem.media.narrators, + series: libraryItem.media.series.map(se => { + const sequence = se.bookSeries?.sequence || '' + if (!sequence) return se.name + return `${se.name} #${sequence}` + }), + genres: libraryItem.media.genres || [], + publishedYear: libraryItem.media.publishedYear, + publishedDate: libraryItem.media.publishedDate, + publisher: libraryItem.media.publisher, + description: libraryItem.media.description, + isbn: libraryItem.media.isbn, + asin: libraryItem.media.asin, + language: libraryItem.media.language, + explicit: !!libraryItem.media.explicit, + abridged: !!libraryItem.media.abridged + } + } + return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => { + // Add metadata.json to libraryFiles array if it is new + let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) + + return metadataLibraryFile + }).catch((error) => { + libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error) + return null + }) + } else { + // Remove metadata.json if it exists + if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.json`))) { + libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.json for item "${libraryItem.media.title}"`) + await fsExtra.remove(Path.join(metadataPath, `metadata.json`)) + libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.json`))) + } + + return abmetadataGenerator.generateFromNewModel(libraryItem, metadataFilePath).then(async (success) => { + if (!success) { + libraryScan.addLog(LogLevel.ERROR, `Failed saving abmetadata to "${metadataFilePath}"`) + return null + } + // Add metadata.abs to libraryFiles array if it is new + let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath)) + if (storeMetadataWithItem && !metadataLibraryFile) { + const newLibraryFile = new LibraryFile() + await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`) + metadataLibraryFile = newLibraryFile.toJSON() + libraryItem.libraryFiles.push(metadataLibraryFile) + } else if (storeMetadataWithItem) { + const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath) + if (fileTimestamps) { + metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs + metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs + metadataLibraryFile.metadata.size = fileTimestamps.size + metadataLibraryFile.ino = fileTimestamps.ino + } + } + + libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`) + return metadataLibraryFile + }) + } + } } module.exports = new BookScanner() \ No newline at end of file diff --git a/server/scanner/LibraryItemScanData.js b/server/scanner/LibraryItemScanData.js index f8bc29c0..a83aaf61 100644 --- a/server/scanner/LibraryItemScanData.js +++ b/server/scanner/LibraryItemScanData.js @@ -253,7 +253,7 @@ class LibraryItemScanData { for (const key in existingLibraryFile.metadata) { if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) { if (key !== 'path' && key !== 'relPath') { - libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) + libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.relPath}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) } else { libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`) } diff --git a/server/scanner/LibraryScanner.js b/server/scanner/LibraryScanner.js index bdcde4c8..a45d7410 100644 --- a/server/scanner/LibraryScanner.js +++ b/server/scanner/LibraryScanner.js @@ -153,7 +153,6 @@ class LibraryScanner { if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) { const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem) - await oldLibraryItem.saveMetadata() // Save metadata.json oldLibraryItemsUpdated.push(oldLibraryItem) } else { // TODO: Temporary while using old model to socket emit @@ -264,7 +263,6 @@ class LibraryScanner { const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan) if (newLibraryItem) { const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem) - await oldLibraryItem.saveMetadata() // Save metadata.json newOldLibraryItems.push(oldLibraryItem) libraryScan.resultsAdded++ diff --git a/server/utils/generators/abmetadataGenerator.js b/server/utils/generators/abmetadataGenerator.js index 2e64f91a..6795a9ff 100644 --- a/server/utils/generators/abmetadataGenerator.js +++ b/server/utils/generators/abmetadataGenerator.js @@ -41,7 +41,7 @@ const podcastMetadataMapper = { from: (v) => v || null }, genres: { - to: (m) => m.genres.join(', '), + to: (m) => m.genres?.join(', ') || '', from: (v) => commaSeparatedToArray(v) }, feedUrl: { @@ -68,11 +68,15 @@ const bookMetadataMapper = { from: (v) => v || null }, authors: { - to: (m) => m.authorName || '', + to: (m) => { + if (m.authorName !== undefined) return m.authorName + if (!m.authors?.length) return '' + return m.authors.map(au => au.name).join(', ') + }, from: (v) => commaSeparatedToArray(v) }, narrators: { - to: (m) => m.narratorName || '', + to: (m) => m.narrators?.join(', ') || '', from: (v) => commaSeparatedToArray(v) }, publishedYear: { @@ -96,11 +100,19 @@ const bookMetadataMapper = { from: (v) => v || null }, genres: { - to: (m) => m.genres.join(', '), + to: (m) => m.genres?.join(', ') || '', from: (v) => commaSeparatedToArray(v) }, series: { - to: (m) => m.seriesName, + to: (m) => { + if (m.seriesName !== undefined) return m.seriesName + if (!m.series?.length) return '' + return m.series.map((se) => { + const sequence = se.bookSeries?.sequence || '' + if (!sequence) return se.name + return `${se.name} #${sequence}` + }).join(', ') + }, from: (v) => { return commaSeparatedToArray(v).map(series => { // Return array of { name, sequence } let sequence = null @@ -174,6 +186,45 @@ function generate(libraryItem, outputPath) { } module.exports.generate = generate +function generateFromNewModel(libraryItem, outputPath) { + let fileString = `;ABMETADATA${CurrentAbMetadataVersion}\n` + fileString += `#audiobookshelf v${package.version}\n\n` + + const mediaType = libraryItem.mediaType + + fileString += `media=${mediaType}\n` + fileString += `tags=${JSON.stringify(libraryItem.media.tags || '')}\n` + + const metadataMapper = metadataMappers[mediaType] + for (const key in metadataMapper) { + fileString += `${key}=${metadataMapper[key].to(libraryItem.media)}\n` + } + + // Description block + if (libraryItem.media.description) { + fileString += '\n[DESCRIPTION]\n' + fileString += libraryItem.media.description + '\n' + } + + // Book chapters + if (mediaType == 'book' && libraryItem.media.chapters?.length) { + fileString += '\n' + libraryItem.media.chapters.forEach((chapter) => { + fileString += `[CHAPTER]\n` + fileString += `start=${chapter.start}\n` + fileString += `end=${chapter.end}\n` + fileString += `title=${chapter.title}\n` + }) + } + return fs.writeFile(outputPath, fileString).then(() => { + return filePerms.setDefault(outputPath, true).then(() => true) + }).catch((error) => { + Logger.error(`[absMetaFileGenerator] Failed to save abs file`, error) + return false + }) +} +module.exports.generateFromNewModel = generateFromNewModel + function parseSections(lines) { if (!lines || !lines.length || !lines[0].startsWith('[')) { // First line must be section start return []