Update new library scanner to handle metadata file changes

This commit is contained in:
advplyr 2023-09-03 15:14:58 -05:00
parent 9123dcb365
commit e63aab95d8
6 changed files with 265 additions and 69 deletions

View File

@ -325,9 +325,9 @@ class Database {
return Promise.all(oldUsers.map(u => this.updateUser(u)))
}
async removeUser(userId) {
removeUser(userId) {
if (!this.sequelize) return false
await this.models.user.removeById(userId)
return this.models.user.removeById(userId)
}
upsertMediaProgress(oldMediaProgress) {
@ -345,9 +345,9 @@ class Database {
return Promise.all(oldBooks.map(oldBook => this.models.book.saveFromOld(oldBook)))
}
async createLibrary(oldLibrary) {
createLibrary(oldLibrary) {
if (!this.sequelize) return false
await this.models.library.createFromOld(oldLibrary)
return this.models.library.createFromOld(oldLibrary)
}
updateLibrary(oldLibrary) {
@ -355,9 +355,9 @@ class Database {
return this.models.library.updateFromOld(oldLibrary)
}
async removeLibrary(libraryId) {
removeLibrary(libraryId) {
if (!this.sequelize) return false
await this.models.library.removeById(libraryId)
return this.models.library.removeById(libraryId)
}
createBulkCollectionBooks(collectionBooks) {

View File

@ -10,7 +10,7 @@ const Podcast = require('./mediaTypes/Podcast')
const Video = require('./mediaTypes/Video')
const Music = require('./mediaTypes/Music')
const { areEquivalent, copyValue, cleanStringForSearch } = require('../utils/index')
const { filePathToPOSIX } = require('../utils/fileUtils')
const { filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
class LibraryItem {
constructor(libraryItem = null) {
@ -40,6 +40,7 @@ class LibraryItem {
this.mediaType = null
this.media = null
/** @type {LibraryFile[]} */
this.libraryFiles = []
if (libraryItem) {
@ -525,19 +526,20 @@ class LibraryItem {
/**
* Save metadata.json/metadata.abs file
* @returns {boolean} true if saved
* @returns {Promise<LibraryFile>} null if not saved
*/
async saveMetadata() {
if (this.mediaType === 'video' || this.mediaType === 'music') return
if (this.isSavingMetadata) return null
if (this.isSavingMetadata) return
this.isSavingMetadata = true
let metadataPath = Path.join(global.MetadataPath, 'items', this.id)
if (global.ServerSettings.storeMetadataWithItem && !this.isFile) {
let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem
if (storeMetadataWithItem && !this.isFile) {
metadataPath = this.path
} else {
// Make sure metadata book dir exists
storeMetadataWithItem = false
await fs.ensureDir(metadataPath)
}
@ -552,20 +554,29 @@ class LibraryItem {
}
return fs.writeFile(metadataFilePath, JSON.stringify(this.media.toJSONForMetadataFile(), null, 2)).then(async () => {
this.isSavingMetadata = false
// Add metadata.json to libraryFiles array if it is new
if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
this.libraryFiles.push(newLibraryFile)
let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
this.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
return true
return metadataLibraryFile
}).catch((error) => {
this.isSavingMetadata = false
Logger.error(`[LibraryItem] Failed to save json file at "${metadataFilePath}"`, error)
return false
return null
}).finally(() => {
this.isSavingMetadata = false
})
} else {
// Remove metadata.json if it exists
@ -576,19 +587,30 @@ class LibraryItem {
}
return abmetadataGenerator.generate(this, metadataFilePath).then(async (success) => {
this.isSavingMetadata = false
if (!success) Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`)
else {
// Add metadata.abs to libraryFiles array if it is new
if (global.ServerSettings.storeMetadataWithItem && !this.libraryFiles.some(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
this.libraryFiles.push(newLibraryFile)
}
Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
if (!success) {
Logger.error(`[LibraryItem] Failed saving abmetadata to "${metadataFilePath}"`)
return null
}
return success
// Add metadata.abs to libraryFiles array if it is new
let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
this.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
}).finally(() => {
this.isSavingMetadata = false
})
}
}

View File

@ -1,4 +1,5 @@
const uuidv4 = require("uuid").v4
const Path = require('path')
const { Sequelize } = require('sequelize')
const { LogLevel } = require('../utils/constants')
const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
@ -9,9 +10,10 @@ const parseNameString = require('../utils/parsers/parseNameString')
const globals = require('../utils/globals')
const AudioFileScanner = require('./AudioFileScanner')
const Database = require('../Database')
const { readTextFile } = require('../utils/fileUtils')
const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
const AudioFile = require('../objects/files/AudioFile')
const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const fsExtra = require("../libs/fsExtra")
/**
@ -161,26 +163,6 @@ class BookScanner {
hasMediaChanges = true
}
// Check/update the isSupplementary flag on libraryFiles for the LibraryItem
let libraryItemUpdated = false
for (const libraryFile of existingLibraryItem.libraryFiles) {
if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
if (libraryFile.isSupplementary !== false) {
libraryFile.isSupplementary = false
libraryItemUpdated = true
}
} else if (libraryFile.isSupplementary !== true) {
libraryFile.isSupplementary = true
libraryItemUpdated = true
}
}
}
if (libraryItemUpdated) {
existingLibraryItem.changed('libraryFiles', true)
await existingLibraryItem.save()
}
// TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
// TODO: store an additional array of metadata keys that the user has changed manually so we know what not to override
const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan)
@ -317,11 +299,6 @@ class BookScanner {
}
}
// Save Book changes to db
if (hasMediaChanges) {
await media.save()
}
// Load authors/series again if updated (for sending back to client)
if (authorsUpdated) {
media.authors = await media.getAuthors({
@ -340,10 +317,39 @@ class BookScanner {
})
}
existingLibraryItem.media = media
let libraryItemUpdated = false
// Save Book changes to db
if (hasMediaChanges) {
await media.save()
await this.saveMetadataFile(existingLibraryItem, libraryScan)
libraryItemUpdated = global.ServerSettings.storeMetadataWithItem && !existingLibraryItem.isFile
}
// Check/update the isSupplementary flag on libraryFiles for the LibraryItem
for (const libraryFile of existingLibraryItem.libraryFiles) {
if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
if (libraryFile.isSupplementary !== false) {
libraryFile.isSupplementary = false
libraryItemUpdated = true
}
} else if (libraryFile.isSupplementary !== true) {
libraryFile.isSupplementary = true
libraryItemUpdated = true
}
}
}
if (libraryItemUpdated) {
existingLibraryItem.changed('libraryFiles', true)
await existingLibraryItem.save()
}
libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved)
libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved)
existingLibraryItem.media = media
return existingLibraryItem
}
@ -509,6 +515,12 @@ class BookScanner {
]
})
await this.saveMetadataFile(libraryItem, libraryScan)
if (global.ServerSettings.storeMetadataWithItem && !libraryItem.isFile) {
libraryItem.changed('libraryFiles', true)
await libraryItem.save()
}
return libraryItem
}
@ -691,7 +703,7 @@ class BookScanner {
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.relPath}" - preferring`)
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`)
let abMetadata = null
if (!!libraryItemData.metadataJsonLibraryFile) {
abMetadata = abmetadataGenerator.parseJson(metadataText)
@ -707,7 +719,7 @@ class BookScanner {
bookMetadata.chapters = abMetadata.chapters
}
for (const key in abMetadata.metadata) {
if (bookMetadata[key] === undefined || abMetadata.metadata[key] === undefined) continue
if (abMetadata.metadata[key] === undefined) continue
bookMetadata[key] = abMetadata.metadata[key]
}
}
@ -803,7 +815,7 @@ class BookScanner {
// Build chapters from audio files
let currChapterId = 0
let currStartTime = 0
includedAudioFiles.forEach((file) => {
audioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
@ -824,5 +836,118 @@ class BookScanner {
}
return chapters
}
/**
*
* @param {import('../models/LibraryItem')} libraryItem
* @param {import('./LibraryScan')} libraryScan
* @returns {Promise}
*/
async saveMetadataFile(libraryItem, libraryScan) {
let metadataPath = Path.join(global.MetadataPath, 'items', libraryItem.id)
let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem
if (storeMetadataWithItem && !libraryItem.isFile) {
metadataPath = libraryItem.path
} else {
// Make sure metadata book dir exists
storeMetadataWithItem = false
await fsExtra.ensureDir(metadataPath)
}
const metadataFileFormat = global.ServerSettings.metadataFileFormat
const metadataFilePath = Path.join(metadataPath, `metadata.${metadataFileFormat}`)
if (metadataFileFormat === 'json') {
// Remove metadata.abs if it exists
if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.abs`))) {
libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.abs for item "${libraryItem.media.title}"`)
await fsExtra.remove(Path.join(metadataPath, `metadata.abs`))
libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.abs`)))
}
// TODO: Update to not use `metadata` so it fits the updated model
const jsonObject = {
tags: libraryItem.media.tags || [],
chapters: libraryItem.media.chapters?.map(c => ({ ...c })) || [],
metadata: {
title: libraryItem.media.title,
subtitle: libraryItem.media.subtitle,
authors: libraryItem.media.authors.map(a => a.name),
narrators: libraryItem.media.narrators,
series: libraryItem.media.series.map(se => {
const sequence = se.bookSeries?.sequence || ''
if (!sequence) return se.name
return `${se.name} #${sequence}`
}),
genres: libraryItem.media.genres || [],
publishedYear: libraryItem.media.publishedYear,
publishedDate: libraryItem.media.publishedDate,
publisher: libraryItem.media.publisher,
description: libraryItem.media.description,
isbn: libraryItem.media.isbn,
asin: libraryItem.media.asin,
language: libraryItem.media.language,
explicit: !!libraryItem.media.explicit,
abridged: !!libraryItem.media.abridged
}
}
return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
// Add metadata.json to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
}).catch((error) => {
libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error)
return null
})
} else {
// Remove metadata.json if it exists
if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.json`))) {
libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.json for item "${libraryItem.media.title}"`)
await fsExtra.remove(Path.join(metadataPath, `metadata.json`))
libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.json`)))
}
return abmetadataGenerator.generateFromNewModel(libraryItem, metadataFilePath).then(async (success) => {
if (!success) {
libraryScan.addLog(LogLevel.ERROR, `Failed saving abmetadata to "${metadataFilePath}"`)
return null
}
// Add metadata.abs to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
})
}
}
}
module.exports = new BookScanner()

View File

@ -253,7 +253,7 @@ class LibraryItemScanData {
for (const key in existingLibraryFile.metadata) {
if (existingLibraryFile.metadata[key] !== scannedLibraryFile.metadata[key]) {
if (key !== 'path' && key !== 'relPath') {
libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.path}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
libraryScan.addLog(LogLevel.DEBUG, `Library file "${existingLibraryFile.metadata.relPath}" for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library file for library item "${libraryItemPath}" key "${key}" changed from "${existingLibraryFile.metadata[key]}" to "${scannedLibraryFile.metadata[key]}"`)
}

View File

@ -153,7 +153,6 @@ class LibraryScanner {
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
await oldLibraryItem.saveMetadata() // Save metadata.json
oldLibraryItemsUpdated.push(oldLibraryItem)
} else {
// TODO: Temporary while using old model to socket emit
@ -264,7 +263,6 @@ class LibraryScanner {
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
if (newLibraryItem) {
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
await oldLibraryItem.saveMetadata() // Save metadata.json
newOldLibraryItems.push(oldLibraryItem)
libraryScan.resultsAdded++

View File

@ -41,7 +41,7 @@ const podcastMetadataMapper = {
from: (v) => v || null
},
genres: {
to: (m) => m.genres.join(', '),
to: (m) => m.genres?.join(', ') || '',
from: (v) => commaSeparatedToArray(v)
},
feedUrl: {
@ -68,11 +68,15 @@ const bookMetadataMapper = {
from: (v) => v || null
},
authors: {
to: (m) => m.authorName || '',
to: (m) => {
if (m.authorName !== undefined) return m.authorName
if (!m.authors?.length) return ''
return m.authors.map(au => au.name).join(', ')
},
from: (v) => commaSeparatedToArray(v)
},
narrators: {
to: (m) => m.narratorName || '',
to: (m) => m.narrators?.join(', ') || '',
from: (v) => commaSeparatedToArray(v)
},
publishedYear: {
@ -96,11 +100,19 @@ const bookMetadataMapper = {
from: (v) => v || null
},
genres: {
to: (m) => m.genres.join(', '),
to: (m) => m.genres?.join(', ') || '',
from: (v) => commaSeparatedToArray(v)
},
series: {
to: (m) => m.seriesName,
to: (m) => {
if (m.seriesName !== undefined) return m.seriesName
if (!m.series?.length) return ''
return m.series.map((se) => {
const sequence = se.bookSeries?.sequence || ''
if (!sequence) return se.name
return `${se.name} #${sequence}`
}).join(', ')
},
from: (v) => {
return commaSeparatedToArray(v).map(series => { // Return array of { name, sequence }
let sequence = null
@ -174,6 +186,45 @@ function generate(libraryItem, outputPath) {
}
module.exports.generate = generate
function generateFromNewModel(libraryItem, outputPath) {
let fileString = `;ABMETADATA${CurrentAbMetadataVersion}\n`
fileString += `#audiobookshelf v${package.version}\n\n`
const mediaType = libraryItem.mediaType
fileString += `media=${mediaType}\n`
fileString += `tags=${JSON.stringify(libraryItem.media.tags || '')}\n`
const metadataMapper = metadataMappers[mediaType]
for (const key in metadataMapper) {
fileString += `${key}=${metadataMapper[key].to(libraryItem.media)}\n`
}
// Description block
if (libraryItem.media.description) {
fileString += '\n[DESCRIPTION]\n'
fileString += libraryItem.media.description + '\n'
}
// Book chapters
if (mediaType == 'book' && libraryItem.media.chapters?.length) {
fileString += '\n'
libraryItem.media.chapters.forEach((chapter) => {
fileString += `[CHAPTER]\n`
fileString += `start=${chapter.start}\n`
fileString += `end=${chapter.end}\n`
fileString += `title=${chapter.title}\n`
})
}
return fs.writeFile(outputPath, fileString).then(() => {
return filePerms.setDefault(outputPath, true).then(() => true)
}).catch((error) => {
Logger.error(`[absMetaFileGenerator] Failed to save abs file`, error)
return false
})
}
module.exports.generateFromNewModel = generateFromNewModel
function parseSections(lines) {
if (!lines || !lines.length || !lines[0].startsWith('[')) { // First line must be section start
return []