mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-11-08 00:54:33 +01:00
413 lines
16 KiB
JavaScript
413 lines
16 KiB
JavaScript
const Path = require('path')
|
|
const packageJson = require('../../package.json')
|
|
const Logger = require('../Logger')
|
|
const SocketAuthority = require('../SocketAuthority')
|
|
const Database = require('../Database')
|
|
const fs = require('../libs/fsExtra')
|
|
const fileUtils = require('../utils/fileUtils')
|
|
const scanUtils = require('../utils/scandir')
|
|
const { ScanResult, LogLevel } = require('../utils/constants')
|
|
const globals = require('../utils/globals')
|
|
const libraryFilters = require('../utils/queries/libraryFilters')
|
|
const AudioFileScanner = require('./AudioFileScanner')
|
|
const ScanOptions = require('./ScanOptions')
|
|
const LibraryScan = require('./LibraryScan')
|
|
const LibraryItemScanData = require('./LibraryItemScanData')
|
|
const AudioFile = require('../objects/files/AudioFile')
|
|
const Book = require('../models/Book')
|
|
const BookScanner = require('./BookScanner')
|
|
|
|
class LibraryScanner {
|
|
constructor(coverManager, taskManager) {
|
|
this.coverManager = coverManager
|
|
this.taskManager = taskManager
|
|
|
|
this.cancelLibraryScan = {}
|
|
this.librariesScanning = []
|
|
}
|
|
|
|
/**
|
|
* @param {string} libraryId
|
|
* @returns {boolean}
|
|
*/
|
|
isLibraryScanning(libraryId) {
|
|
return this.librariesScanning.some(ls => ls.id === libraryId)
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('../objects/Library')} library
|
|
* @param {*} options
|
|
*/
|
|
async scan(library, options = {}) {
|
|
if (this.isLibraryScanning(library.id)) {
|
|
Logger.error(`[Scanner] Already scanning ${library.id}`)
|
|
return
|
|
}
|
|
|
|
if (!library.folders.length) {
|
|
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
|
|
return
|
|
}
|
|
|
|
const scanOptions = new ScanOptions()
|
|
scanOptions.setData(options, Database.serverSettings)
|
|
|
|
const libraryScan = new LibraryScan()
|
|
libraryScan.setData(library, scanOptions)
|
|
libraryScan.verbose = true
|
|
this.librariesScanning.push(libraryScan.getScanEmitData)
|
|
|
|
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
|
|
|
|
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
|
|
|
const canceled = await this.scanLibrary(libraryScan)
|
|
|
|
if (canceled) {
|
|
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
|
|
delete this.cancelLibraryScan[libraryScan.libraryId]
|
|
}
|
|
|
|
libraryScan.setComplete()
|
|
|
|
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
|
|
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
|
|
|
if (canceled && !libraryScan.totalResults) {
|
|
const emitData = libraryScan.getScanEmitData
|
|
emitData.results = null
|
|
SocketAuthority.emitter('scan_complete', emitData)
|
|
return
|
|
}
|
|
|
|
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
|
|
|
|
if (libraryScan.totalResults) {
|
|
libraryScan.saveLog()
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('./LibraryScan')} libraryScan
|
|
*/
|
|
async scanLibrary(libraryScan) {
|
|
// Make sure library filter data is set
|
|
// this is used to check for existing authors & series
|
|
await libraryFilters.getFilterData(libraryScan.library)
|
|
|
|
/** @type {LibraryItemScanData[]} */
|
|
let libraryItemDataFound = []
|
|
|
|
// Scan each library folder
|
|
for (let i = 0; i < libraryScan.folders.length; i++) {
|
|
const folder = libraryScan.folders[i]
|
|
const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
|
|
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
|
|
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
const existingLibraryItems = await Database.libraryItemModel.findAll({
|
|
where: {
|
|
libraryId: libraryScan.libraryId
|
|
},
|
|
attributes: ['id', 'mediaId', 'mediaType', 'path', 'relPath', 'ino', 'isMissing', 'isFile', 'mtime', 'ctime', 'birthtime', 'libraryFiles', 'libraryFolderId', 'size']
|
|
})
|
|
|
|
const libraryItemIdsMissing = []
|
|
for (const existingLibraryItem of existingLibraryItems) {
|
|
// First try to find matching library item with exact file path
|
|
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
|
|
if (!libraryItemData) {
|
|
// Fallback to finding matching library item with matching inode value
|
|
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
|
|
if (libraryItemData) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
|
|
}
|
|
}
|
|
|
|
if (!libraryItemData) {
|
|
// Podcast folder can have no episodes and still be valid
|
|
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
|
|
} else {
|
|
libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
|
|
libraryScan.resultsMissing++
|
|
if (!existingLibraryItem.isMissing) {
|
|
libraryItemIdsMissing.push(existingLibraryItem.id)
|
|
}
|
|
}
|
|
} else {
|
|
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
|
|
await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
|
|
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
|
|
await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update missing library items
|
|
if (libraryItemIdsMissing.length) {
|
|
libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
|
|
await Database.libraryItemModel.update({
|
|
isMissing: true,
|
|
lastScan: Date.now(),
|
|
lastScanVersion: packageJson.version
|
|
}, {
|
|
where: {
|
|
id: libraryItemIdsMissing
|
|
}
|
|
})
|
|
}
|
|
|
|
// Add new library items
|
|
if (libraryItemDataFound.length) {
|
|
for (const libraryItemData of libraryItemDataFound) {
|
|
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
|
|
if (newLibraryItem) {
|
|
libraryScan.resultsAdded++
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO: Socket emitter
|
|
}
|
|
|
|
/**
|
|
* Get scan data for library folder
|
|
* @param {import('../objects/Library')} library
|
|
* @param {import('../objects/Folder')} folder
|
|
* @returns {LibraryItemScanData[]}
|
|
*/
|
|
async scanFolder(library, folder) {
|
|
const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
|
|
|
|
const pathExists = await fs.pathExists(folderPath)
|
|
if (!pathExists) {
|
|
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
|
|
return []
|
|
}
|
|
|
|
const fileItems = await fileUtils.recurseFiles(folderPath)
|
|
const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
|
|
|
|
if (!Object.keys(libraryItemGrouping).length) {
|
|
Logger.error(`Root path has no media folders: ${folderPath}`)
|
|
return []
|
|
}
|
|
|
|
const items = []
|
|
for (const libraryItemPath in libraryItemGrouping) {
|
|
let isFile = false // item is not in a folder
|
|
let libraryItemData = null
|
|
let fileObjs = []
|
|
if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
|
|
// Media file in root only get title
|
|
libraryItemData = {
|
|
mediaMetadata: {
|
|
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
|
|
},
|
|
path: Path.posix.join(folderPath, libraryItemPath),
|
|
relPath: libraryItemPath
|
|
}
|
|
fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
|
|
isFile = true
|
|
} else {
|
|
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
|
|
fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
|
|
}
|
|
|
|
const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
|
|
|
|
if (!libraryItemFolderStats.ino) {
|
|
Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
|
|
continue
|
|
}
|
|
|
|
items.push(new LibraryItemScanData({
|
|
libraryFolderId: folder.id,
|
|
libraryId: folder.libraryId,
|
|
mediaType: library.mediaType,
|
|
ino: libraryItemFolderStats.ino,
|
|
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
|
|
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
|
|
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
|
|
path: libraryItemData.path,
|
|
relPath: libraryItemData.relPath,
|
|
isFile,
|
|
mediaMetadata: libraryItemData.mediaMetadata || null,
|
|
libraryFiles: fileObjs
|
|
}))
|
|
}
|
|
return items
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('../models/LibraryItem')} existingLibraryItem
|
|
* @param {LibraryItemScanData} libraryItemData
|
|
* @param {LibraryScan} libraryScan
|
|
*/
|
|
async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
|
|
if (existingLibraryItem.mediaType === 'book') {
|
|
/** @type {Book} */
|
|
const media = await existingLibraryItem.getMedia({
|
|
include: [
|
|
{
|
|
model: Database.authorModel,
|
|
through: {
|
|
attributes: ['createdAt']
|
|
}
|
|
},
|
|
{
|
|
model: Database.seriesModel,
|
|
through: {
|
|
attributes: ['sequence', 'createdAt']
|
|
}
|
|
}
|
|
]
|
|
})
|
|
|
|
let hasMediaChanges = libraryItemData.hasAudioFileChanges
|
|
if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== media.audioFiles.length) {
|
|
// Filter out audio files that were removed
|
|
media.audioFiles = media.audioFiles.filter(af => libraryItemData.checkAudioFileRemoved(af))
|
|
|
|
// Update audio files that were modified
|
|
if (libraryItemData.audioLibraryFilesModified.length) {
|
|
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
|
|
media.audioFiles = media.audioFiles.map((audioFileObj) => {
|
|
let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === audioFileObj.metadata.path)
|
|
if (!matchedScannedAudioFile) {
|
|
matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === audioFileObj.ino)
|
|
}
|
|
|
|
if (matchedScannedAudioFile) {
|
|
scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
|
|
const audioFile = new AudioFile(audioFileObj)
|
|
audioFile.updateFromScan(matchedScannedAudioFile)
|
|
return audioFile.toJSON()
|
|
}
|
|
return audioFileObj
|
|
})
|
|
// Modified audio files that were not found on the book
|
|
if (scannedAudioFiles.length) {
|
|
media.audioFiles.push(...scannedAudioFiles)
|
|
}
|
|
}
|
|
|
|
// Add new audio files scanned in
|
|
if (libraryItemData.audioLibraryFilesAdded.length) {
|
|
const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded)
|
|
media.audioFiles.push(...scannedAudioFiles)
|
|
}
|
|
|
|
// Add audio library files that are not already set on the book (safety check)
|
|
let audioLibraryFilesToAdd = []
|
|
for (const audioLibraryFile of libraryItemData.audioLibraryFiles) {
|
|
if (!media.audioFiles.some(af => af.ino === audioLibraryFile.ino)) {
|
|
libraryScan.addLog(LogLevel.DEBUG, `Existing audio library file "${audioLibraryFile.metadata.relPath}" was not set on book "${media.title}" so setting it now`)
|
|
audioLibraryFilesToAdd.push(audioLibraryFile)
|
|
}
|
|
}
|
|
if (audioLibraryFilesToAdd.length) {
|
|
const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, audioLibraryFilesToAdd)
|
|
media.audioFiles.push(...scannedAudioFiles)
|
|
}
|
|
|
|
media.audioFiles = AudioFileScanner.runSmartTrackOrder(existingLibraryItem.relPath, media.audioFiles)
|
|
|
|
media.duration = 0
|
|
media.audioFiles.forEach((af) => {
|
|
if (!isNaN(af.duration)) {
|
|
media.duration += af.duration
|
|
}
|
|
})
|
|
|
|
media.changed('audioFiles', true)
|
|
}
|
|
|
|
// Check if cover was removed
|
|
if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
|
|
media.coverPath = null
|
|
hasMediaChanges = true
|
|
}
|
|
|
|
// Check if cover is not set and image files were found
|
|
if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
|
|
// Prefer using a cover image with the name "cover" otherwise use the first image
|
|
const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
|
media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
|
hasMediaChanges = true
|
|
}
|
|
|
|
// Check if ebook was removed
|
|
if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
|
|
media.ebookFile = null
|
|
hasMediaChanges = true
|
|
}
|
|
|
|
// Check if ebook is not set and ebooks were found
|
|
if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
|
|
// Prefer to use an epub ebook then fallback to the first ebook found
|
|
let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
|
|
if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
|
|
// Ebook file is the same as library file except for additional `ebookFormat`
|
|
ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
|
|
media.ebookFile = ebookLibraryFile
|
|
media.changed('ebookFile', true)
|
|
hasMediaChanges = true
|
|
}
|
|
|
|
// Check/update the isSupplementary flag on libraryFiles for the LibraryItem
|
|
let libraryItemUpdated = false
|
|
for (const libraryFile of existingLibraryItem.libraryFiles) {
|
|
if (globals.SupportedEbookTypes.includes(libraryFile.metadata.ext.slice(1).toLowerCase())) {
|
|
if (media.ebookFile && libraryFile.ino === media.ebookFile.ino) {
|
|
if (libraryFile.isSupplementary !== false) {
|
|
libraryFile.isSupplementary = false
|
|
libraryItemUpdated = true
|
|
}
|
|
} else if (libraryFile.isSupplementary !== true) {
|
|
libraryFile.isSupplementary = true
|
|
libraryItemUpdated = true
|
|
}
|
|
}
|
|
}
|
|
if (libraryItemUpdated) {
|
|
existingLibraryItem.changed('libraryFiles', true)
|
|
await existingLibraryItem.save()
|
|
}
|
|
|
|
// TODO: Update chapters & metadata
|
|
|
|
if (hasMediaChanges) {
|
|
await media.save()
|
|
}
|
|
} else {
|
|
// TODO: Scan updated podcast
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {LibraryItemScanData} libraryItemData
|
|
* @param {LibraryScan} libraryScan
|
|
*/
|
|
async scanNewLibraryItem(libraryItemData, libraryScan) {
|
|
if (libraryScan.libraryMediaType === 'book') {
|
|
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
|
|
if (newLibraryItem) {
|
|
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
|
}
|
|
return newLibraryItem
|
|
} else {
|
|
// TODO: Scan new podcast
|
|
return null
|
|
}
|
|
}
|
|
}
|
|
module.exports = LibraryScanner |