mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-11-07 16:44:16 +01:00
Merge pull request #2773 from mikiher/fix-library-files-inconsistencies
Fix library files inconsistencies
This commit is contained in:
advplyr
GitHub
commit
51ff62356d
@ -93,7 +93,7 @@ class Logger {
|
||||
|
||||
// Save log to file
|
||||
if (level >= this.logLevel) {
|
||||
await this.logManager.logToFile(logObj)
|
||||
await this.logManager?.logToFile(logObj)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
const { DataTypes, Model, WhereOptions } = require('sequelize')
|
||||
const { DataTypes, Model } = require('sequelize')
|
||||
const Logger = require('../Logger')
|
||||
const oldLibraryItem = require('../objects/LibraryItem')
|
||||
const libraryFilters = require('../utils/queries/libraryFilters')
|
||||
@ -116,7 +116,7 @@ class LibraryItem extends Model {
|
||||
|
||||
/**
|
||||
* Currently unused because this is too slow and uses too much mem
|
||||
* @param {[WhereOptions]} where
|
||||
* @param {import('sequelize').WhereOptions} [where]
|
||||
* @returns {Array<objects.LibraryItem>} old library items
|
||||
*/
|
||||
static async getAllOldLibraryItems(where = null) {
|
||||
@ -773,12 +773,14 @@ class LibraryItem extends Model {
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {WhereOptions} where
|
||||
* @param {import('sequelize').WhereOptions} where
|
||||
* @param {import('sequelize').BindOrReplacements} replacements
|
||||
* @returns {Object} oldLibraryItem
|
||||
*/
|
||||
static async findOneOld(where) {
|
||||
static async findOneOld(where, replacements = {}) {
|
||||
const libraryItem = await this.findOne({
|
||||
where,
|
||||
replacements,
|
||||
include: [
|
||||
{
|
||||
model: this.sequelize.models.book,
|
||||
|
||||
@ -20,6 +20,7 @@ const LibraryScan = require("./LibraryScan")
|
||||
const OpfFileScanner = require('./OpfFileScanner')
|
||||
const NfoFileScanner = require('./NfoFileScanner')
|
||||
const AbsMetadataFileScanner = require('./AbsMetadataFileScanner')
|
||||
const EBookFile = require("../objects/files/EBookFile")
|
||||
|
||||
/**
|
||||
* Metadata for books pulled from files
|
||||
@ -84,7 +85,7 @@ class BookScanner {
|
||||
|
||||
// Update audio files that were modified
|
||||
if (libraryItemData.audioLibraryFilesModified.length) {
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified.map(lf => lf.new))
|
||||
media.audioFiles = media.audioFiles.map((audioFileObj) => {
|
||||
let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
|
||||
if (!matchedScannedAudioFile) {
|
||||
@ -138,11 +139,25 @@ class BookScanner {
|
||||
}
|
||||
|
||||
// Check if cover was removed
|
||||
if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath) && !(await fsExtra.pathExists(media.coverPath))) {
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesRemoved.some(lf => lf.metadata.path === media.coverPath) && !(await fsExtra.pathExists(media.coverPath))) {
|
||||
media.coverPath = null
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Update cover if it was modified
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesModified.length) {
|
||||
let coverMatch = libraryItemData.imageLibraryFilesModified.find(iFile => iFile.old.metadata.path === media.coverPath)
|
||||
if (coverMatch) {
|
||||
const coverPath = coverMatch.new.metadata.path
|
||||
if (coverPath !== media.coverPath) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating book cover "${media.coverPath}" => "${coverPath}" for book "${media.title}"`)
|
||||
media.coverPath = coverPath
|
||||
media.changed('coverPath', true)
|
||||
hasMediaChanges = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if cover is not set and image files were found
|
||||
if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
@ -157,6 +172,19 @@ class BookScanner {
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Update ebook if it was modified
|
||||
if (media.ebookFile && libraryItemData.ebookLibraryFilesModified.length) {
|
||||
let ebookMatch = libraryItemData.ebookLibraryFilesModified.find(eFile => eFile.old.metadata.path === media.ebookFile.metadata.path)
|
||||
if (ebookMatch) {
|
||||
const ebookFile = new EBookFile(ebookMatch.new)
|
||||
ebookFile.ebookFormat = ebookFile.metadata.ext.slice(1).toLowerCase()
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating book ebook file "${media.ebookFile.metadata.path}" => "${ebookFile.metadata.path}" for book "${media.title}"`)
|
||||
media.ebookFile = ebookFile.toJSON()
|
||||
media.changed('ebookFile', true)
|
||||
hasMediaChanges = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check if ebook is not set and ebooks were found
|
||||
if (!media.ebookFile && !librarySettings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
|
||||
// Prefer to use an epub ebook then fallback to the first ebook found
|
||||
|
||||
@ -4,6 +4,12 @@ const LibraryItem = require('../models/LibraryItem')
|
||||
const globals = require('../utils/globals')
|
||||
|
||||
class LibraryItemScanData {
|
||||
/**
|
||||
* @typedef LibraryFileModifiedObject
|
||||
* @property {LibraryItem.LibraryFileObject} old
|
||||
* @property {LibraryItem.LibraryFileObject} new
|
||||
*/
|
||||
|
||||
constructor(data) {
|
||||
/** @type {string} */
|
||||
this.libraryFolderId = data.libraryFolderId
|
||||
@ -39,7 +45,7 @@ class LibraryItemScanData {
|
||||
this.libraryFilesRemoved = []
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
this.libraryFilesAdded = []
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
/** @type {LibraryFileModifiedObject[]} */
|
||||
this.libraryFilesModified = []
|
||||
}
|
||||
|
||||
@ -77,9 +83,9 @@ class LibraryItemScanData {
|
||||
return (this.audioLibraryFilesRemoved.length + this.audioLibraryFilesAdded.length + this.audioLibraryFilesModified.length) > 0
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
/** @type {LibraryFileModifiedObject[]} */
|
||||
get audioLibraryFilesModified() {
|
||||
return this.libraryFilesModified.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
return this.libraryFilesModified.filter(lf => globals.SupportedAudioTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
@ -97,12 +103,42 @@ class LibraryItemScanData {
|
||||
return this.libraryFiles.filter(lf => globals.SupportedAudioTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryFileModifiedObject[]} */
|
||||
get imageLibraryFilesModified() {
|
||||
return this.libraryFilesModified.filter(lf => globals.SupportedImageTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get imageLibraryFilesRemoved() {
|
||||
return this.libraryFilesRemoved.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get imageLibraryFilesAdded() {
|
||||
return this.libraryFilesAdded.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get imageLibraryFiles() {
|
||||
return this.libraryFiles.filter(lf => globals.SupportedImageTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {import('../objects/files/LibraryFile')[]} */
|
||||
/** @type {LibraryFileModifiedObject[]} */
|
||||
get ebookLibraryFilesModified() {
|
||||
return this.libraryFilesModified.filter(lf => globals.SupportedEbookTypes.includes(lf.old.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get ebookLibraryFilesRemoved() {
|
||||
return this.libraryFilesRemoved.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get ebookLibraryFilesAdded() {
|
||||
return this.libraryFilesAdded.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
|
||||
/** @type {LibraryItem.LibraryFileObject[]} */
|
||||
get ebookLibraryFiles() {
|
||||
return this.libraryFiles.filter(lf => globals.SupportedEbookTypes.includes(lf.metadata.ext?.slice(1).toLowerCase() || ''))
|
||||
}
|
||||
@ -153,7 +189,7 @@ class LibraryItemScanData {
|
||||
existingLibraryItem[key] = this[key]
|
||||
this.hasChanges = true
|
||||
|
||||
if (key === 'relPath') {
|
||||
if (key === 'relPath' || key === 'path') {
|
||||
this.hasPathChange = true
|
||||
}
|
||||
}
|
||||
@ -202,8 +238,9 @@ class LibraryItemScanData {
|
||||
this.hasChanges = true
|
||||
} else {
|
||||
libraryFilesAdded = libraryFilesAdded.filter(lf => lf !== matchingLibraryFile)
|
||||
let existingLibraryFileBefore = structuredClone(existingLibraryFile)
|
||||
if (this.compareUpdateLibraryFile(existingLibraryItem.path, existingLibraryFile, matchingLibraryFile, libraryScan)) {
|
||||
this.libraryFilesModified.push(existingLibraryFile)
|
||||
this.libraryFilesModified.push({old: existingLibraryFileBefore, new: existingLibraryFile})
|
||||
this.hasChanges = true
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,10 +21,10 @@ class LibraryItemScanner {
|
||||
* Scan single library item
|
||||
*
|
||||
* @param {string} libraryItemId
|
||||
* @param {{relPath:string, path:string}} [renamedPaths] used by watcher when item folder was renamed
|
||||
* @param {{relPath:string, path:string}} [updateLibraryItemDetails] used by watcher when item folder was renamed
|
||||
* @returns {number} ScanResult
|
||||
*/
|
||||
async scanLibraryItem(libraryItemId, renamedPaths = null) {
|
||||
async scanLibraryItem(libraryItemId, updateLibraryItemDetails = null) {
|
||||
// TODO: Add task manager
|
||||
const libraryItem = await Database.libraryItemModel.findByPk(libraryItemId)
|
||||
if (!libraryItem) {
|
||||
@ -32,11 +32,12 @@ class LibraryItemScanner {
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
|
||||
const libraryFolderId = updateLibraryItemDetails?.libraryFolderId || libraryItem.libraryFolderId
|
||||
const library = await Database.libraryModel.findByPk(libraryItem.libraryId, {
|
||||
include: {
|
||||
model: Database.libraryFolderModel,
|
||||
where: {
|
||||
id: libraryItem.libraryFolderId
|
||||
id: libraryFolderId
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -51,11 +52,11 @@ class LibraryItemScanner {
|
||||
|
||||
const scanLogger = new ScanLogger()
|
||||
scanLogger.verbose = true
|
||||
scanLogger.setData('libraryItem', renamedPaths?.relPath || libraryItem.relPath)
|
||||
scanLogger.setData('libraryItem', updateLibraryItemDetails?.relPath || libraryItem.relPath)
|
||||
|
||||
const libraryItemPath = renamedPaths?.path || fileUtils.filePathToPOSIX(libraryItem.path)
|
||||
const libraryItemPath = updateLibraryItemDetails?.path || fileUtils.filePathToPOSIX(libraryItem.path)
|
||||
const folder = library.libraryFolders[0]
|
||||
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false)
|
||||
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, updateLibraryItemDetails?.isFile || false)
|
||||
|
||||
let libraryItemDataUpdated = await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)
|
||||
|
||||
|
||||
@ -154,7 +154,11 @@ class LibraryScanner {
|
||||
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
|
||||
if (!libraryItemData) {
|
||||
// Fallback to finding matching library item with matching inode value
|
||||
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
|
||||
libraryItemData = libraryItemDataFound.find(lid =>
|
||||
ItemToItemInoMatch(lid, existingLibraryItem) ||
|
||||
ItemToFileInoMatch(lid, existingLibraryItem) ||
|
||||
ItemToFileInoMatch(existingLibraryItem, lid)
|
||||
)
|
||||
if (libraryItemData) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
|
||||
}
|
||||
@ -522,22 +526,25 @@ class LibraryScanner {
|
||||
|
||||
// Check if book dir group is already an item
|
||||
let existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
libraryId: library.id,
|
||||
path: potentialChildDirs
|
||||
})
|
||||
|
||||
let renamedPaths = {}
|
||||
let updatedLibraryItemDetails = {}
|
||||
if (!existingLibraryItem) {
|
||||
const dirIno = await fileUtils.getIno(fullPath)
|
||||
existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
ino: dirIno
|
||||
})
|
||||
const isSingleMedia = isSingleMediaFile(fileUpdateGroup, itemDir)
|
||||
existingLibraryItem =
|
||||
await findLibraryItemByItemToItemInoMatch(library.id, fullPath) ||
|
||||
await findLibraryItemByItemToFileInoMatch(library.id, fullPath, isSingleMedia) ||
|
||||
await findLibraryItemByFileToItemInoMatch(library.id, fullPath, isSingleMedia, fileUpdateGroup[itemDir])
|
||||
if (existingLibraryItem) {
|
||||
Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
|
||||
// Update library item paths for scan
|
||||
existingLibraryItem.path = fullPath
|
||||
existingLibraryItem.relPath = itemDir
|
||||
renamedPaths.path = fullPath
|
||||
renamedPaths.relPath = itemDir
|
||||
updatedLibraryItemDetails.path = fullPath
|
||||
updatedLibraryItemDetails.relPath = itemDir
|
||||
updatedLibraryItemDetails.libraryFolderId = folder.id
|
||||
updatedLibraryItemDetails.isFile = isSingleMedia
|
||||
}
|
||||
}
|
||||
if (existingLibraryItem) {
|
||||
@ -554,10 +561,9 @@ class LibraryScanner {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Scan library item for updates
|
||||
Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
|
||||
itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id, renamedPaths)
|
||||
itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id, updatedLibraryItemDetails)
|
||||
continue
|
||||
} else if (library.settings.audiobooksOnly && !hasAudioFiles(fileUpdateGroup, itemDir)) {
|
||||
Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" has no audio files`)
|
||||
@ -594,6 +600,14 @@ class LibraryScanner {
|
||||
}
|
||||
module.exports = new LibraryScanner()
|
||||
|
||||
function ItemToFileInoMatch(libraryItem1, libraryItem2) {
|
||||
return libraryItem1.isFile && libraryItem2.libraryFiles.some(lf => lf.ino === libraryItem1.ino)
|
||||
}
|
||||
|
||||
function ItemToItemInoMatch(libraryItem1, libraryItem2) {
|
||||
return libraryItem1.ino === libraryItem2.ino
|
||||
}
|
||||
|
||||
function hasAudioFiles(fileUpdateGroup, itemDir) {
|
||||
return isSingleMediaFile(fileUpdateGroup, itemDir) ?
|
||||
scanUtils.checkFilepathIsAudioFile(fileUpdateGroup[itemDir]) :
|
||||
@ -603,3 +617,55 @@ function hasAudioFiles(fileUpdateGroup, itemDir) {
|
||||
function isSingleMediaFile(fileUpdateGroup, itemDir) {
|
||||
return itemDir === fileUpdateGroup[itemDir]
|
||||
}
|
||||
|
||||
async function findLibraryItemByItemToItemInoMatch(libraryId, fullPath) {
|
||||
const ino = await fileUtils.getIno(fullPath)
|
||||
if (!ino) return null
|
||||
const existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
libraryId: libraryId,
|
||||
ino: ino
|
||||
})
|
||||
if (existingLibraryItem)
|
||||
Logger.debug(`[LibraryScanner] Found library item with matching inode "${ino}" at path "${existingLibraryItem.path}"`)
|
||||
return existingLibraryItem
|
||||
}
|
||||
|
||||
async function findLibraryItemByItemToFileInoMatch(libraryId, fullPath, isSingleMedia) {
|
||||
if (!isSingleMedia) return null
|
||||
// check if it was moved from another folder by comparing the ino to the library files
|
||||
const ino = await fileUtils.getIno(fullPath)
|
||||
if (!ino) return null
|
||||
const existingLibraryItem = await Database.libraryItemModel.findOneOld([
|
||||
{
|
||||
libraryId: libraryId
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM json_each(libraryFiles) WHERE json_valid(json_each.value) AND json_each.value->>"$.ino" = :inode)'), {
|
||||
[sequelize.Op.gt]: 0
|
||||
})
|
||||
], {
|
||||
inode: ino
|
||||
})
|
||||
if (existingLibraryItem)
|
||||
Logger.debug(`[LibraryScanner] Found library item with a library file matching inode "${ino}" at path "${existingLibraryItem.path}"`)
|
||||
return existingLibraryItem
|
||||
}
|
||||
|
||||
async function findLibraryItemByFileToItemInoMatch(libraryId, fullPath, isSingleMedia, itemFiles) {
|
||||
if (isSingleMedia) return null
|
||||
// check if it was moved from the root folder by comparing the ino to the ino of the scanned files
|
||||
let itemFileInos = []
|
||||
for (const itemFile of itemFiles) {
|
||||
const ino = await fileUtils.getIno(Path.posix.join(fullPath, itemFile))
|
||||
if (ino) itemFileInos.push(ino)
|
||||
}
|
||||
if (!itemFileInos.length) return null
|
||||
const existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
libraryId: libraryId,
|
||||
ino: {
|
||||
[sequelize.Op.in]: itemFileInos
|
||||
}
|
||||
})
|
||||
if (existingLibraryItem)
|
||||
Logger.debug(`[LibraryScanner] Found library item with inode matching one of "${itemFileInos.join(',')}" at path "${existingLibraryItem.path}"`)
|
||||
return existingLibraryItem
|
||||
}
|
||||
@ -71,7 +71,7 @@ class PodcastScanner {
|
||||
|
||||
// Update audio files that were modified
|
||||
if (libraryItemData.audioLibraryFilesModified.length) {
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified.map(lf => lf.new))
|
||||
|
||||
for (const podcastEpisode of existingPodcastEpisodes) {
|
||||
let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === podcastEpisode.audioFile.metadata.path)
|
||||
@ -132,11 +132,25 @@ class PodcastScanner {
|
||||
let hasMediaChanges = false
|
||||
|
||||
// Check if cover was removed
|
||||
if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesRemoved.some(lf => lf.metadata.path === media.coverPath)) {
|
||||
media.coverPath = null
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Update cover if it was modified
|
||||
if (media.coverPath && libraryItemData.imageLibraryFilesModified.length) {
|
||||
let coverMatch = libraryItemData.imageLibraryFilesModified.find(iFile => iFile.old.metadata.path === media.coverPath)
|
||||
if (coverMatch) {
|
||||
const coverPath = coverMatch.new.metadata.path
|
||||
if (coverPath !== media.coverPath) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast cover "${media.coverPath}" => "${coverPath}" for podcast "${media.title}"`)
|
||||
media.coverPath = coverPath
|
||||
media.changed('coverPath', true)
|
||||
hasMediaChanges = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if cover is not set and image files were found
|
||||
if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
|
||||
Loading…
Reference in New Issue
Block a user