mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-11-08 09:04:43 +01:00
294 lines
11 KiB
JavaScript
294 lines
11 KiB
JavaScript
const sequelize = require('sequelize')
|
|
const Path = require('path')
|
|
const packageJson = require('../../package.json')
|
|
const Logger = require('../Logger')
|
|
const SocketAuthority = require('../SocketAuthority')
|
|
const Database = require('../Database')
|
|
const fs = require('../libs/fsExtra')
|
|
const fileUtils = require('../utils/fileUtils')
|
|
const scanUtils = require('../utils/scandir')
|
|
const { LogLevel } = require('../utils/constants')
|
|
const libraryFilters = require('../utils/queries/libraryFilters')
|
|
const LibraryItemScanner = require('./LibraryItemScanner')
|
|
const ScanOptions = require('./ScanOptions')
|
|
const LibraryScan = require('./LibraryScan')
|
|
const LibraryItemScanData = require('./LibraryItemScanData')
|
|
|
|
class LibraryScanner {
|
|
constructor() {
|
|
this.cancelLibraryScan = {}
|
|
this.librariesScanning = []
|
|
}
|
|
|
|
/**
|
|
* @param {string} libraryId
|
|
* @returns {boolean}
|
|
*/
|
|
isLibraryScanning(libraryId) {
|
|
return this.librariesScanning.some(ls => ls.id === libraryId)
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('../objects/Library')} library
|
|
* @param {*} options
|
|
*/
|
|
async scan(library, options = {}) {
|
|
if (this.isLibraryScanning(library.id)) {
|
|
Logger.error(`[Scanner] Already scanning ${library.id}`)
|
|
return
|
|
}
|
|
|
|
if (!library.folders.length) {
|
|
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
|
|
return
|
|
}
|
|
|
|
const scanOptions = new ScanOptions()
|
|
scanOptions.setData(options, Database.serverSettings)
|
|
|
|
const libraryScan = new LibraryScan()
|
|
libraryScan.setData(library, scanOptions)
|
|
libraryScan.verbose = true
|
|
this.librariesScanning.push(libraryScan.getScanEmitData)
|
|
|
|
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
|
|
|
|
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
|
|
|
const canceled = await this.scanLibrary(libraryScan)
|
|
|
|
if (canceled) {
|
|
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
|
|
delete this.cancelLibraryScan[libraryScan.libraryId]
|
|
}
|
|
|
|
libraryScan.setComplete()
|
|
|
|
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
|
|
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
|
|
|
if (canceled && !libraryScan.totalResults) {
|
|
const emitData = libraryScan.getScanEmitData
|
|
emitData.results = null
|
|
SocketAuthority.emitter('scan_complete', emitData)
|
|
return
|
|
}
|
|
|
|
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
|
|
|
|
if (libraryScan.totalResults) {
|
|
libraryScan.saveLog()
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param {import('./LibraryScan')} libraryScan
|
|
* @returns {boolean} true if scan canceled
|
|
*/
|
|
async scanLibrary(libraryScan) {
|
|
// Make sure library filter data is set
|
|
// this is used to check for existing authors & series
|
|
await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId)
|
|
|
|
/** @type {LibraryItemScanData[]} */
|
|
let libraryItemDataFound = []
|
|
|
|
// Scan each library folder
|
|
for (let i = 0; i < libraryScan.folders.length; i++) {
|
|
const folder = libraryScan.folders[i]
|
|
const itemDataFoundInFolder = await this.scanFolder(libraryScan.library, folder)
|
|
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
|
|
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
const existingLibraryItems = await Database.libraryItemModel.findAll({
|
|
where: {
|
|
libraryId: libraryScan.libraryId
|
|
}
|
|
})
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
const libraryItemIdsMissing = []
|
|
let oldLibraryItemsUpdated = []
|
|
for (const existingLibraryItem of existingLibraryItems) {
|
|
// First try to find matching library item with exact file path
|
|
let libraryItemData = libraryItemDataFound.find(lid => lid.path === existingLibraryItem.path)
|
|
if (!libraryItemData) {
|
|
// Fallback to finding matching library item with matching inode value
|
|
libraryItemData = libraryItemDataFound.find(lid => lid.ino === existingLibraryItem.ino)
|
|
if (libraryItemData) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item with path "${existingLibraryItem.path}" was not found, but library item inode "${existingLibraryItem.ino}" was found at path "${libraryItemData.path}"`)
|
|
}
|
|
}
|
|
|
|
if (!libraryItemData) {
|
|
// Podcast folder can have no episodes and still be valid
|
|
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(existingLibraryItem.path)) {
|
|
libraryScan.addLog(LogLevel.INFO, `Library item "${existingLibraryItem.relPath}" folder exists but has no episodes`)
|
|
} else {
|
|
libraryScan.addLog(LogLevel.WARN, `Library Item "${existingLibraryItem.path}" (inode: ${existingLibraryItem.ino}) is missing`)
|
|
libraryScan.resultsMissing++
|
|
if (!existingLibraryItem.isMissing) {
|
|
libraryItemIdsMissing.push(existingLibraryItem.id)
|
|
|
|
// TODO: Temporary while using old model to socket emit
|
|
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
|
|
oldLibraryItem.isMissing = true
|
|
oldLibraryItem.updatedAt = Date.now()
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
}
|
|
}
|
|
} else {
|
|
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
|
|
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
|
|
libraryScan.resultsUpdated++
|
|
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
|
|
const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
|
|
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
} else {
|
|
// TODO: Temporary while using old model to socket emit
|
|
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
|
|
oldLibraryItemsUpdated.push(oldLibraryItem)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Emit item updates in chunks of 10 to client
|
|
if (oldLibraryItemsUpdated.length === 10) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
|
|
oldLibraryItemsUpdated = []
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
}
|
|
// Emit item updates to client
|
|
if (oldLibraryItemsUpdated.length) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
|
|
}
|
|
|
|
// Authors and series that were removed from books should be removed if they are now empty
|
|
await LibraryItemScanner.checkAuthorsAndSeriesRemovedFromBooks(libraryScan.libraryId, libraryScan)
|
|
|
|
// Update missing library items
|
|
if (libraryItemIdsMissing.length) {
|
|
libraryScan.addLog(LogLevel.INFO, `Updating ${libraryItemIdsMissing.length} library items missing`)
|
|
await Database.libraryItemModel.update({
|
|
isMissing: true,
|
|
lastScan: Date.now(),
|
|
lastScanVersion: packageJson.version
|
|
}, {
|
|
where: {
|
|
id: libraryItemIdsMissing
|
|
}
|
|
})
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
|
|
// Add new library items
|
|
if (libraryItemDataFound.length) {
|
|
let newOldLibraryItems = []
|
|
for (const libraryItemData of libraryItemDataFound) {
|
|
const newLibraryItem = await LibraryItemScanner.scanNewLibraryItem(libraryItemData, libraryScan.library.settings, libraryScan)
|
|
if (newLibraryItem) {
|
|
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
|
|
newOldLibraryItems.push(oldLibraryItem)
|
|
|
|
libraryScan.resultsAdded++
|
|
}
|
|
|
|
// Emit new items in chunks of 10 to client
|
|
if (newOldLibraryItems.length === 10) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
|
|
newOldLibraryItems = []
|
|
}
|
|
|
|
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
|
}
|
|
// Emit new items to client
|
|
if (newOldLibraryItems.length) {
|
|
// TODO: Should only emit to clients where library item is accessible
|
|
SocketAuthority.emitter('items_added', newOldLibraryItems.map(li => li.toJSONExpanded()))
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get scan data for library folder
|
|
* @param {import('../objects/Library')} library
|
|
* @param {import('../objects/Folder')} folder
|
|
* @returns {LibraryItemScanData[]}
|
|
*/
|
|
async scanFolder(library, folder) {
|
|
const folderPath = fileUtils.filePathToPOSIX(folder.fullPath)
|
|
|
|
const pathExists = await fs.pathExists(folderPath)
|
|
if (!pathExists) {
|
|
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
|
|
return []
|
|
}
|
|
|
|
const fileItems = await fileUtils.recurseFiles(folderPath)
|
|
const libraryItemGrouping = scanUtils.groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
|
|
|
|
if (!Object.keys(libraryItemGrouping).length) {
|
|
Logger.error(`Root path has no media folders: ${folderPath}`)
|
|
return []
|
|
}
|
|
|
|
const items = []
|
|
for (const libraryItemPath in libraryItemGrouping) {
|
|
let isFile = false // item is not in a folder
|
|
let libraryItemData = null
|
|
let fileObjs = []
|
|
if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
|
|
// Media file in root only get title
|
|
libraryItemData = {
|
|
mediaMetadata: {
|
|
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
|
|
},
|
|
path: Path.posix.join(folderPath, libraryItemPath),
|
|
relPath: libraryItemPath
|
|
}
|
|
fileObjs = await scanUtils.buildLibraryFile(folderPath, [libraryItemPath])
|
|
isFile = true
|
|
} else {
|
|
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
|
|
fileObjs = await scanUtils.buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
|
|
}
|
|
|
|
const libraryItemFolderStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
|
|
|
|
if (!libraryItemFolderStats.ino) {
|
|
Logger.warn(`[LibraryScanner] Library item folder "${libraryItemData.path}" has no inode value`)
|
|
continue
|
|
}
|
|
|
|
items.push(new LibraryItemScanData({
|
|
libraryFolderId: folder.id,
|
|
libraryId: folder.libraryId,
|
|
mediaType: library.mediaType,
|
|
ino: libraryItemFolderStats.ino,
|
|
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
|
|
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
|
|
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
|
|
path: libraryItemData.path,
|
|
relPath: libraryItemData.relPath,
|
|
isFile,
|
|
mediaMetadata: libraryItemData.mediaMetadata || null,
|
|
libraryFiles: fileObjs
|
|
}))
|
|
}
|
|
return items
|
|
}
|
|
}
|
|
module.exports = new LibraryScanner() |