mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-12-24 23:59:08 +01:00
Add new podcast scanner and remove old scanner
This commit is contained in:
advplyr
parent
42ff3d8314
commit
b9da3fa30e
@ -35,6 +35,7 @@ const AudioMetadataMangaer = require('./managers/AudioMetadataManager')
|
||||
const RssFeedManager = require('./managers/RssFeedManager')
|
||||
const CronManager = require('./managers/CronManager')
|
||||
const TaskManager = require('./managers/TaskManager')
|
||||
const LibraryScanner = require('./scanner/LibraryScanner')
|
||||
|
||||
class Server {
|
||||
constructor(SOURCE, PORT, HOST, UID, GID, CONFIG_PATH, METADATA_PATH, ROUTER_BASE_PATH) {
|
||||
@ -76,7 +77,7 @@ class Server {
|
||||
this.rssFeedManager = new RssFeedManager()
|
||||
|
||||
this.scanner = new Scanner(this.coverManager, this.taskManager)
|
||||
this.cronManager = new CronManager(this.scanner, this.podcastManager)
|
||||
this.cronManager = new CronManager(this.podcastManager)
|
||||
|
||||
// Routers
|
||||
this.apiRouter = new ApiRouter(this)
|
||||
@ -92,6 +93,10 @@ class Server {
|
||||
this.auth.authMiddleware(req, res, next)
|
||||
}
|
||||
|
||||
cancelLibraryScan(libraryId) {
|
||||
LibraryScanner.setCancelLibraryScan(libraryId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize database, backups, logs, rss feeds, cron jobs & watcher
|
||||
* Cleanup stale/invalid data
|
||||
@ -122,7 +127,6 @@ class Server {
|
||||
this.watcher.disabled = true
|
||||
} else {
|
||||
this.watcher.initWatcher(libraries)
|
||||
this.watcher.on('files', this.filesChanged.bind(this))
|
||||
}
|
||||
}
|
||||
|
||||
@ -241,11 +245,6 @@ class Server {
|
||||
res.sendStatus(200)
|
||||
}
|
||||
|
||||
async filesChanged(fileUpdates) {
|
||||
Logger.info('[Server]', fileUpdates.length, 'Files Changed')
|
||||
await this.scanner.scanFilesChanged(fileUpdates)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove user media progress for items that no longer exist & remove seriesHideFrom that no longer exist
|
||||
*/
|
||||
|
||||
@ -95,7 +95,7 @@ class SocketAuthority {
|
||||
socket.on('auth', (token) => this.authenticateSocket(socket, token))
|
||||
|
||||
// Scanning
|
||||
socket.on('cancel_scan', this.cancelScan.bind(this))
|
||||
socket.on('cancel_scan', (libraryId) => this.cancelScan(libraryId))
|
||||
|
||||
// Logs
|
||||
socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level))
|
||||
@ -209,7 +209,7 @@ class SocketAuthority {
|
||||
|
||||
cancelScan(id) {
|
||||
Logger.debug('[SocketAuthority] Cancel scan', id)
|
||||
this.Server.scanner.setCancelLibraryScan(id)
|
||||
this.Server.cancelLibraryScan(id)
|
||||
}
|
||||
}
|
||||
module.exports = new SocketAuthority()
|
||||
@ -1,20 +1,29 @@
|
||||
const EventEmitter = require('events')
|
||||
const Watcher = require('./libs/watcher/watcher')
|
||||
const Logger = require('./Logger')
|
||||
const LibraryScanner = require('./scanner/LibraryScanner')
|
||||
|
||||
const { filePathToPOSIX } = require('./utils/fileUtils')
|
||||
|
||||
/**
|
||||
* @typedef PendingFileUpdate
|
||||
* @property {string} path
|
||||
* @property {string} relPath
|
||||
* @property {string} folderId
|
||||
* @property {string} type
|
||||
*/
|
||||
class FolderWatcher extends EventEmitter {
|
||||
constructor() {
|
||||
super()
|
||||
this.paths = [] // Not used
|
||||
this.pendingFiles = [] // Not used
|
||||
|
||||
/** @type {{id:string, name:string, folders:import('./objects/Folder')[], paths:string[], watcher:Watcher[]}[]} */
|
||||
this.libraryWatchers = []
|
||||
/** @type {PendingFileUpdate[]} */
|
||||
this.pendingFileUpdates = []
|
||||
this.pendingDelay = 4000
|
||||
this.pendingTimeout = null
|
||||
|
||||
/** @type {string[]} */
|
||||
this.ignoreDirs = []
|
||||
this.disabled = false
|
||||
}
|
||||
@ -29,11 +38,12 @@ class FolderWatcher extends EventEmitter {
|
||||
return
|
||||
}
|
||||
Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
|
||||
var folderPaths = library.folderPaths
|
||||
|
||||
const folderPaths = library.folderPaths
|
||||
folderPaths.forEach((fp) => {
|
||||
Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
|
||||
})
|
||||
var watcher = new Watcher(folderPaths, {
|
||||
const watcher = new Watcher(folderPaths, {
|
||||
ignored: /(^|[\/\\])\../, // ignore dotfiles
|
||||
renameDetection: true,
|
||||
renameTimeout: 2000,
|
||||
@ -144,6 +154,12 @@ class FolderWatcher extends EventEmitter {
|
||||
this.addFileUpdate(libraryId, pathTo, 'renamed')
|
||||
}
|
||||
|
||||
/**
|
||||
* File update detected from watcher
|
||||
* @param {string} libraryId
|
||||
* @param {string} path
|
||||
* @param {string} type
|
||||
*/
|
||||
addFileUpdate(libraryId, path, type) {
|
||||
path = filePathToPOSIX(path)
|
||||
if (this.pendingFilePaths.includes(path)) return
|
||||
@ -163,9 +179,10 @@ class FolderWatcher extends EventEmitter {
|
||||
}
|
||||
const folderFullPath = filePathToPOSIX(folder.fullPath)
|
||||
|
||||
var relPath = path.replace(folderFullPath, '')
|
||||
const relPath = path.replace(folderFullPath, '')
|
||||
|
||||
var hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
|
||||
// Ignore files/folders starting with "."
|
||||
const hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
|
||||
if (hasDotPath) {
|
||||
Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`)
|
||||
return
|
||||
@ -184,7 +201,8 @@ class FolderWatcher extends EventEmitter {
|
||||
// Notify server of update after "pendingDelay"
|
||||
clearTimeout(this.pendingTimeout)
|
||||
this.pendingTimeout = setTimeout(() => {
|
||||
this.emit('files', this.pendingFileUpdates)
|
||||
// this.emit('files', this.pendingFileUpdates)
|
||||
LibraryScanner.scanFilesChanged(this.pendingFileUpdates)
|
||||
this.pendingFileUpdates = []
|
||||
}, this.pendingDelay)
|
||||
}
|
||||
|
||||
@ -978,12 +978,8 @@ class LibraryController {
|
||||
forceRescan: req.query.force == 1
|
||||
}
|
||||
res.sendStatus(200)
|
||||
if (req.library.mediaType === 'podcast') {
|
||||
// TODO: New library scanner for podcasts
|
||||
await this.scanner.scan(req.library, options)
|
||||
} else {
|
||||
await LibraryScanner.scan(req.library, options)
|
||||
}
|
||||
|
||||
await LibraryScanner.scan(req.library, options)
|
||||
|
||||
await Database.resetLibraryIssuesFilterData(req.library.id)
|
||||
Logger.info('[LibraryController] Scan complete')
|
||||
|
||||
@ -445,7 +445,12 @@ class LibraryItemController {
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
|
||||
const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li)
|
||||
const libraryItems = await Database.libraryItemModel.findAll({
|
||||
where: {
|
||||
id: req.body.libraryItemIds
|
||||
},
|
||||
attributes: ['id', 'libraryId', 'isFile']
|
||||
})
|
||||
if (!libraryItems?.length) {
|
||||
return res.sendStatus(400)
|
||||
}
|
||||
@ -457,7 +462,7 @@ class LibraryItemController {
|
||||
if (libraryItem.isFile) {
|
||||
Logger.warn(`[LibraryItemController] Re-scanning file library items not yet supported`)
|
||||
} else {
|
||||
await this.scanner.scanLibraryItemByRequest(libraryItem)
|
||||
await LibraryItemScanner.scanLibraryItem(libraryItem.id)
|
||||
}
|
||||
}
|
||||
|
||||
@ -476,14 +481,7 @@ class LibraryItemController {
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
|
||||
let result = 0
|
||||
if (req.libraryItem.isPodcast) {
|
||||
// TODO: New library item scanner for podcast
|
||||
result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
|
||||
} else {
|
||||
result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
|
||||
}
|
||||
|
||||
const result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
|
||||
await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId)
|
||||
res.json({
|
||||
result: Object.keys(ScanResult).find(key => ScanResult[key] == result)
|
||||
|
||||
@ -2,10 +2,10 @@ const Sequelize = require('sequelize')
|
||||
const cron = require('../libs/nodeCron')
|
||||
const Logger = require('../Logger')
|
||||
const Database = require('../Database')
|
||||
const LibraryScanner = require('../scanner/LibraryScanner')
|
||||
|
||||
class CronManager {
|
||||
constructor(scanner, podcastManager) {
|
||||
this.scanner = scanner
|
||||
constructor(podcastManager) {
|
||||
this.podcastManager = podcastManager
|
||||
|
||||
this.libraryScanCrons = []
|
||||
@ -39,7 +39,7 @@ class CronManager {
|
||||
Logger.debug(`[CronManager] Init library scan cron for ${library.name} on schedule ${library.settings.autoScanCronExpression}`)
|
||||
const libScanCron = cron.schedule(library.settings.autoScanCronExpression, () => {
|
||||
Logger.debug(`[CronManager] Library scan cron executing for ${library.name}`)
|
||||
this.scanner.scan(library)
|
||||
LibraryScanner.scan(library)
|
||||
})
|
||||
this.libraryScanCrons.push({
|
||||
libraryId: library.id,
|
||||
|
||||
@ -39,7 +39,7 @@ class PodcastEpisode extends Model {
|
||||
this.enclosureType
|
||||
/** @type {Date} */
|
||||
this.publishedAt
|
||||
/** @type {Object} */
|
||||
/** @type {import('./Book').AudioFileObject} */
|
||||
this.audioFile
|
||||
/** @type {ChapterObject[]} */
|
||||
this.chapters
|
||||
|
||||
@ -6,6 +6,7 @@ class AudioFile {
|
||||
constructor(data) {
|
||||
this.index = null
|
||||
this.ino = null
|
||||
/** @type {FileMetadata} */
|
||||
this.metadata = null
|
||||
this.addedAt = null
|
||||
this.updatedAt = null
|
||||
@ -27,6 +28,7 @@ class AudioFile {
|
||||
this.embeddedCoverArt = null
|
||||
|
||||
// Tags scraped from the audio file
|
||||
/** @type {AudioMetaTags} */
|
||||
this.metaTags = null
|
||||
|
||||
this.manuallyVerified = false
|
||||
|
||||
@ -4,10 +4,6 @@ const PodcastMetadata = require('../metadata/PodcastMetadata')
|
||||
const { areEquivalent, copyValue, cleanStringForSearch } = require('../../utils/index')
|
||||
const abmetadataGenerator = require('../../utils/generators/abmetadataGenerator')
|
||||
const { readTextFile, filePathToPOSIX } = require('../../utils/fileUtils')
|
||||
const { createNewSortInstance } = require('../../libs/fastSort')
|
||||
const naturalSort = createNewSortInstance({
|
||||
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
|
||||
})
|
||||
|
||||
class Podcast {
|
||||
constructor(podcast) {
|
||||
|
||||
@ -360,7 +360,7 @@ class BookScanner {
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {import('../models/LibraryItem')}
|
||||
* @returns {Promise<import('../models/LibraryItem')>}
|
||||
*/
|
||||
async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
// Scan audio files found
|
||||
@ -439,6 +439,7 @@ class BookScanner {
|
||||
libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image
|
||||
libraryItemObj.isMissing = false
|
||||
libraryItemObj.isInvalid = false
|
||||
libraryItemObj.extraData = {}
|
||||
|
||||
// Set isSupplementary flag on ebook library files
|
||||
for (const libraryFile of libraryItemObj.libraryFiles) {
|
||||
@ -739,6 +740,8 @@ class BookScanner {
|
||||
bookMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
||||
}
|
||||
|
||||
bookMetadata.titleIgnorePrefix = getTitleIgnorePrefix(bookMetadata.title)
|
||||
|
||||
return bookMetadata
|
||||
}
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ const Database = require('../Database')
|
||||
const LibraryScan = require('./LibraryScan')
|
||||
const LibraryItemScanData = require('./LibraryItemScanData')
|
||||
const BookScanner = require('./BookScanner')
|
||||
const PodcastScanner = require('./PodcastScanner')
|
||||
const ScanLogger = require('./ScanLogger')
|
||||
const LibraryItem = require('../models/LibraryItem')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
@ -158,13 +159,13 @@ class LibraryItemScanner {
|
||||
* @returns {Promise<LibraryItem>}
|
||||
*/
|
||||
async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
|
||||
let newLibraryItem = null
|
||||
if (existingLibraryItem.mediaType === 'book') {
|
||||
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
|
||||
return libraryItem
|
||||
newLibraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
|
||||
} else {
|
||||
// TODO: Scan updated podcast
|
||||
return null
|
||||
newLibraryItem = await PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
|
||||
}
|
||||
return newLibraryItem
|
||||
}
|
||||
|
||||
/**
|
||||
@ -175,16 +176,34 @@ class LibraryItemScanner {
|
||||
* @returns {Promise<LibraryItem>}
|
||||
*/
|
||||
async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
if (libraryScan.libraryMediaType === 'book') {
|
||||
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
|
||||
if (newLibraryItem) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
||||
}
|
||||
return newLibraryItem
|
||||
let newLibraryItem = null
|
||||
if (libraryItemData.mediaType === 'book') {
|
||||
newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
|
||||
} else {
|
||||
// TODO: Scan new podcast
|
||||
return null
|
||||
newLibraryItem = await PodcastScanner.scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan)
|
||||
}
|
||||
if (newLibraryItem) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
||||
}
|
||||
return newLibraryItem
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan library item folder coming from Watcher
|
||||
* @param {string} libraryItemPath
|
||||
* @param {import('../models/Library')} library
|
||||
* @param {import('../models/LibraryFolder')} folder
|
||||
* @param {boolean} isSingleMediaItem
|
||||
* @returns {Promise<LibraryItem>} ScanResult
|
||||
*/
|
||||
async scanPotentialNewLibraryItem(libraryItemPath, library, folder, isSingleMediaItem) {
|
||||
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem)
|
||||
|
||||
const scanLogger = new ScanLogger()
|
||||
scanLogger.verbose = true
|
||||
scanLogger.setData('libraryItem', libraryItemScanData.relPath)
|
||||
|
||||
return this.scanNewLibraryItem(libraryItemScanData, library.settings, scanLogger)
|
||||
}
|
||||
}
|
||||
module.exports = new LibraryItemScanner()
|
||||
@ -7,7 +7,7 @@ const Database = require('../Database')
|
||||
const fs = require('../libs/fsExtra')
|
||||
const fileUtils = require('../utils/fileUtils')
|
||||
const scanUtils = require('../utils/scandir')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { LogLevel, ScanResult } = require('../utils/constants')
|
||||
const libraryFilters = require('../utils/queries/libraryFilters')
|
||||
const LibraryItemScanner = require('./LibraryItemScanner')
|
||||
const ScanOptions = require('./ScanOptions')
|
||||
@ -18,6 +18,10 @@ class LibraryScanner {
|
||||
constructor() {
|
||||
this.cancelLibraryScan = {}
|
||||
this.librariesScanning = []
|
||||
|
||||
this.scanningFilesChanged = false
|
||||
/** @type {import('../Watcher').PendingFileUpdate[][]} */
|
||||
this.pendingFileUpdatesToScan = []
|
||||
}
|
||||
|
||||
/**
|
||||
@ -28,6 +32,16 @@ class LibraryScanner {
|
||||
return this.librariesScanning.some(ls => ls.id === libraryId)
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {string} libraryId
|
||||
*/
|
||||
setCancelLibraryScan(libraryId) {
|
||||
const libraryScanning = this.librariesScanning.find(ls => ls.id === libraryId)
|
||||
if (!libraryScanning) return
|
||||
this.cancelLibraryScan[libraryId] = true
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('../objects/Library')} library
|
||||
@ -290,5 +304,229 @@ class LibraryScanner {
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan files changed from Watcher
|
||||
* @param {import('../Watcher').PendingFileUpdate[]} fileUpdates
|
||||
*/
|
||||
async scanFilesChanged(fileUpdates) {
|
||||
if (!fileUpdates?.length) return
|
||||
|
||||
// If already scanning files from watcher then add these updates to queue
|
||||
if (this.scanningFilesChanged) {
|
||||
this.pendingFileUpdatesToScan.push(fileUpdates)
|
||||
Logger.debug(`[LibraryScanner] Already scanning files from watcher - file updates pushed to queue (size ${this.pendingFileUpdatesToScan.length})`)
|
||||
return
|
||||
}
|
||||
this.scanningFilesChanged = true
|
||||
|
||||
// files grouped by folder
|
||||
const folderGroups = this.getFileUpdatesGrouped(fileUpdates)
|
||||
|
||||
for (const folderId in folderGroups) {
|
||||
const libraryId = folderGroups[folderId].libraryId
|
||||
// const library = await Database.libraryModel.getOldById(libraryId)
|
||||
const library = await Database.libraryModel.findByPk(libraryId, {
|
||||
include: {
|
||||
model: Database.libraryFolderModel,
|
||||
where: {
|
||||
id: folderId
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!library) {
|
||||
Logger.error(`[LibraryScanner] Library "${libraryId}" not found in files changed ${libraryId}`)
|
||||
continue
|
||||
}
|
||||
const folder = library.libraryFolders[0]
|
||||
|
||||
const relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
|
||||
const fileUpdateGroup = scanUtils.groupFilesIntoLibraryItemPaths(library.mediaType, relFilePaths)
|
||||
|
||||
if (!Object.keys(fileUpdateGroup).length) {
|
||||
Logger.info(`[LibraryScanner] No important changes to scan for in folder "${folderId}"`)
|
||||
continue
|
||||
}
|
||||
const folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateGroup)
|
||||
Logger.debug(`[LibraryScanner] Folder scan results`, folderScanResults)
|
||||
|
||||
// If something was updated then reset numIssues filter data for library
|
||||
if (Object.values(folderScanResults).some(scanResult => scanResult !== ScanResult.NOTHING && scanResult !== ScanResult.UPTODATE)) {
|
||||
await Database.resetLibraryIssuesFilterData(libraryId)
|
||||
}
|
||||
}
|
||||
|
||||
this.scanningFilesChanged = false
|
||||
|
||||
if (this.pendingFileUpdatesToScan.length) {
|
||||
Logger.debug(`[LibraryScanner] File updates finished scanning with more updates in queue (${this.pendingFileUpdatesToScan.length})`)
|
||||
this.scanFilesChanged(this.pendingFileUpdatesToScan.shift())
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Group array of PendingFileUpdate from Watcher by folder
|
||||
* @param {import('../Watcher').PendingFileUpdate[]} fileUpdates
|
||||
* @returns {Record<string,{libraryId:string, folderId:string, fileUpdates:import('../Watcher').PendingFileUpdate[]}>}
|
||||
*/
|
||||
getFileUpdatesGrouped(fileUpdates) {
|
||||
const folderGroups = {}
|
||||
fileUpdates.forEach((file) => {
|
||||
if (folderGroups[file.folderId]) {
|
||||
folderGroups[file.folderId].fileUpdates.push(file)
|
||||
} else {
|
||||
folderGroups[file.folderId] = {
|
||||
libraryId: file.libraryId,
|
||||
folderId: file.folderId,
|
||||
fileUpdates: [file]
|
||||
}
|
||||
}
|
||||
})
|
||||
return folderGroups
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan grouped paths for library folder coming from Watcher
|
||||
* @param {import('../models/Library')} library
|
||||
* @param {import('../models/LibraryFolder')} folder
|
||||
* @param {Record<string, string[]>} fileUpdateGroup
|
||||
* @returns {Promise<Record<string,number>>}
|
||||
*/
|
||||
async scanFolderUpdates(library, folder, fileUpdateGroup) {
|
||||
// Make sure library filter data is set
|
||||
// this is used to check for existing authors & series
|
||||
await libraryFilters.getFilterData(library.mediaType, library.id)
|
||||
Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
|
||||
Logger.debug(`[Scanner] scanFolderUpdates fileUpdateGroup`, fileUpdateGroup)
|
||||
|
||||
// First pass - Remove files in parent dirs of items and remap the fileupdate group
|
||||
// Test Case: Moving audio files from library item folder to author folder should trigger a re-scan of the item
|
||||
const updateGroup = { ...fileUpdateGroup }
|
||||
for (const itemDir in updateGroup) {
|
||||
if (itemDir == fileUpdateGroup[itemDir]) continue // Media in root path
|
||||
|
||||
const itemDirNestedFiles = fileUpdateGroup[itemDir].filter(b => b.includes('/'))
|
||||
if (!itemDirNestedFiles.length) continue
|
||||
|
||||
const firstNest = itemDirNestedFiles[0].split('/').shift()
|
||||
const altDir = `${itemDir}/${firstNest}`
|
||||
|
||||
const fullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir)
|
||||
const childLibraryItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[sequelize.Op.not]: fullPath
|
||||
},
|
||||
path: {
|
||||
[sequelize.Op.startsWith]: fullPath
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!childLibraryItem) {
|
||||
continue
|
||||
}
|
||||
|
||||
const altFullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), altDir)
|
||||
const altChildLibraryItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[sequelize.Op.not]: altFullPath
|
||||
},
|
||||
path: {
|
||||
[sequelize.Op.startsWith]: altFullPath
|
||||
}
|
||||
}
|
||||
})
|
||||
if (altChildLibraryItem) {
|
||||
continue
|
||||
}
|
||||
|
||||
delete fileUpdateGroup[itemDir]
|
||||
fileUpdateGroup[altDir] = itemDirNestedFiles.map((f) => f.split('/').slice(1).join('/'))
|
||||
Logger.warn(`[LibraryScanner] Some files were modified in a parent directory of a library item "${childLibraryItem.path}" - ignoring`)
|
||||
}
|
||||
|
||||
// Second pass: Check for new/updated/removed items
|
||||
const itemGroupingResults = {}
|
||||
for (const itemDir in fileUpdateGroup) {
|
||||
const fullPath = Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir)
|
||||
const dirIno = await fileUtils.getIno(fullPath)
|
||||
|
||||
const itemDirParts = itemDir.split('/').slice(0, -1)
|
||||
const potentialChildDirs = []
|
||||
for (let i = 0; i < itemDirParts.length; i++) {
|
||||
potentialChildDirs.push(Path.posix.join(fileUtils.filePathToPOSIX(folder.path), itemDir.split('/').slice(0, -1 - i).join('/')))
|
||||
}
|
||||
|
||||
// Check if book dir group is already an item
|
||||
let existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
path: potentialChildDirs
|
||||
})
|
||||
|
||||
if (!existingLibraryItem) {
|
||||
existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
ino: dirIno
|
||||
})
|
||||
if (existingLibraryItem) {
|
||||
Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
|
||||
// Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData
|
||||
existingLibraryItem.path = fullPath
|
||||
existingLibraryItem.relPath = itemDir
|
||||
}
|
||||
}
|
||||
if (existingLibraryItem) {
|
||||
// Is the item exactly - check if was deleted
|
||||
if (existingLibraryItem.path === fullPath) {
|
||||
const exists = await fs.pathExists(fullPath)
|
||||
if (!exists) {
|
||||
Logger.info(`[LibraryScanner] Scanning file update group and library item was deleted "${existingLibraryItem.media.metadata.title}" - marking as missing`)
|
||||
existingLibraryItem.setMissing()
|
||||
await Database.updateLibraryItem(existingLibraryItem)
|
||||
SocketAuthority.emitter('item_updated', existingLibraryItem.toJSONExpanded())
|
||||
|
||||
itemGroupingResults[itemDir] = ScanResult.REMOVED
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Scan library item for updates
|
||||
Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
|
||||
// itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem)
|
||||
itemGroupingResults[itemDir] = await LibraryItemScanner.scanLibraryItem(existingLibraryItem.id)
|
||||
continue
|
||||
} else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(scanUtils.checkFilepathIsAudioFile)) {
|
||||
Logger.debug(`[LibraryScanner] Folder update for relative path "${itemDir}" has no audio files`)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if a library item is a subdirectory of this dir
|
||||
const childItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[sequelize.Op.startsWith]: fullPath + '/'
|
||||
}
|
||||
}
|
||||
})
|
||||
if (childItem) {
|
||||
Logger.warn(`[LibraryScanner] Files were modified in a parent directory of a library item "${childItem.path}" - ignoring`)
|
||||
itemGroupingResults[itemDir] = ScanResult.NOTHING
|
||||
continue
|
||||
}
|
||||
|
||||
Logger.debug(`[LibraryScanner] Folder update group must be a new item "${itemDir}" in library "${library.name}"`)
|
||||
const isSingleMediaItem = itemDir === fileUpdateGroup[itemDir]
|
||||
const newLibraryItem = await LibraryItemScanner.scanPotentialNewLibraryItem(fullPath, library, folder, isSingleMediaItem)
|
||||
if (newLibraryItem) {
|
||||
const oldNewLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
|
||||
SocketAuthority.emitter('item_added', oldNewLibraryItem.toJSONExpanded())
|
||||
}
|
||||
itemGroupingResults[itemDir] = newLibraryItem ? ScanResult.ADDED : ScanResult.NOTHING
|
||||
}
|
||||
|
||||
return itemGroupingResults
|
||||
}
|
||||
}
|
||||
module.exports = new LibraryScanner()
|
||||
626
server/scanner/PodcastScanner.js
Normal file
626
server/scanner/PodcastScanner.js
Normal file
@ -0,0 +1,626 @@
|
||||
const uuidv4 = require("uuid").v4
|
||||
const Path = require('path')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { getTitleIgnorePrefix } = require('../utils/index')
|
||||
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
|
||||
const AudioFileScanner = require('./AudioFileScanner')
|
||||
const Database = require('../Database')
|
||||
const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../utils/fileUtils')
|
||||
const AudioFile = require('../objects/files/AudioFile')
|
||||
const CoverManager = require('../managers/CoverManager')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
const fsExtra = require("../libs/fsExtra")
|
||||
const PodcastEpisode = require("../models/PodcastEpisode")
|
||||
|
||||
/**
|
||||
* Metadata for podcasts pulled from files
|
||||
* @typedef PodcastMetadataObject
|
||||
* @property {string} title
|
||||
* @property {string} titleIgnorePrefix
|
||||
* @property {string} author
|
||||
* @property {string} releaseDate
|
||||
* @property {string} feedURL
|
||||
* @property {string} imageURL
|
||||
* @property {string} description
|
||||
* @property {string} itunesPageURL
|
||||
* @property {string} itunesId
|
||||
* @property {string} language
|
||||
* @property {string} podcastType
|
||||
* @property {string[]} genres
|
||||
* @property {string[]} tags
|
||||
* @property {boolean} explicit
|
||||
*/
|
||||
|
||||
class PodcastScanner {
|
||||
constructor() { }
|
||||
|
||||
/**
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise<import('../models/LibraryItem')>}
|
||||
*/
|
||||
async rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
|
||||
/** @type {import('../models/Podcast')} */
|
||||
const media = await existingLibraryItem.getMedia({
|
||||
include: [
|
||||
{
|
||||
model: Database.podcastEpisodeModel
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
/** @type {import('../models/PodcastEpisode')[]} */
|
||||
let existingPodcastEpisodes = media.podcastEpisodes
|
||||
|
||||
/** @type {AudioFile[]} */
|
||||
let newAudioFiles = []
|
||||
|
||||
if (libraryItemData.hasAudioFileChanges || libraryItemData.audioLibraryFiles.length !== existingPodcastEpisodes.length) {
|
||||
// Filter out and destroy episodes that were removed
|
||||
existingPodcastEpisodes = await Promise.all(existingPodcastEpisodes.filter(async ep => {
|
||||
if (libraryItemData.checkAudioFileRemoved(ep.audioFile)) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Podcast episode "${ep.title}" audio file was removed`)
|
||||
// TODO: Should clean up other data linked to this episode
|
||||
await ep.destroy()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}))
|
||||
|
||||
// Update audio files that were modified
|
||||
if (libraryItemData.audioLibraryFilesModified.length) {
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesModified)
|
||||
|
||||
for (const podcastEpisode of existingPodcastEpisodes) {
|
||||
let matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.metadata.path === podcastEpisode.audioFile.metadata.path)
|
||||
if (!matchedScannedAudioFile) {
|
||||
matchedScannedAudioFile = scannedAudioFiles.find(saf => saf.ino === podcastEpisode.audioFile.ino)
|
||||
}
|
||||
|
||||
if (matchedScannedAudioFile) {
|
||||
scannedAudioFiles = scannedAudioFiles.filter(saf => saf !== matchedScannedAudioFile)
|
||||
const audioFile = new AudioFile(podcastEpisode.audioFile)
|
||||
audioFile.updateFromScan(matchedScannedAudioFile)
|
||||
podcastEpisode.audioFile = audioFile.toJSON()
|
||||
podcastEpisode.changed('audioFile', true)
|
||||
|
||||
// Set metadata and save episode
|
||||
this.setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, libraryScan)
|
||||
libraryScan.addLog(LogLevel.INFO, `Podcast episode "${podcastEpisode.title}" keys changed [${podcastEpisode.changed()?.join(', ')}]`)
|
||||
await podcastEpisode.save()
|
||||
}
|
||||
}
|
||||
|
||||
// Modified audio files that were not found as a podcast episode
|
||||
if (scannedAudioFiles.length) {
|
||||
newAudioFiles.push(...scannedAudioFiles)
|
||||
}
|
||||
}
|
||||
|
||||
// Add new audio files scanned in
|
||||
if (libraryItemData.audioLibraryFilesAdded.length) {
|
||||
const scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(existingLibraryItem.mediaType, libraryItemData, libraryItemData.audioLibraryFilesAdded)
|
||||
newAudioFiles.push(...scannedAudioFiles)
|
||||
}
|
||||
|
||||
// Create new podcast episodes from new found audio files
|
||||
for (const newAudioFile of newAudioFiles) {
|
||||
const newEpisode = {
|
||||
title: newAudioFile.metaTags.tagTitle || newAudioFile.metadata.filenameNoExt,
|
||||
subtitle: null,
|
||||
season: null,
|
||||
episode: null,
|
||||
episodeType: null,
|
||||
pubDate: null,
|
||||
publishedAt: null,
|
||||
description: null,
|
||||
audioFile: newAudioFile.toJSON(),
|
||||
chapters: newAudioFile.chapters || [],
|
||||
podcastId: media.id
|
||||
}
|
||||
const newPodcastEpisode = Database.podcastEpisodeModel.build(newEpisode)
|
||||
// Set metadata and save new episode
|
||||
this.setPodcastEpisodeMetadataFromAudioFile(newPodcastEpisode, libraryScan)
|
||||
libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newPodcastEpisode.title}" added`)
|
||||
await newPodcastEpisode.save()
|
||||
existingPodcastEpisodes.push(newPodcastEpisode)
|
||||
}
|
||||
}
|
||||
|
||||
let hasMediaChanges = false
|
||||
|
||||
// Check if cover was removed
|
||||
if (media.coverPath && !libraryItemData.imageLibraryFiles.some(lf => lf.metadata.path === media.coverPath)) {
|
||||
media.coverPath = null
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Check if cover is not set and image files were found
|
||||
if (!media.coverPath && libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
media.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// TODO: When metadata file is stored in /metadata/items/{libraryItemId}.[abs|json] we should load this
|
||||
const podcastMetadata = await this.getPodcastMetadataFromScanData(existingPodcastEpisodes, libraryItemData, libraryScan)
|
||||
|
||||
for (const key in podcastMetadata) {
|
||||
// Ignore unset metadata and empty arrays
|
||||
if (podcastMetadata[key] === undefined || (Array.isArray(podcastMetadata[key]) && !podcastMetadata[key].length)) continue
|
||||
|
||||
if (key === 'genres') {
|
||||
const existingGenres = media.genres || []
|
||||
if (podcastMetadata.genres.some(g => !existingGenres.includes(g)) || existingGenres.some(g => !podcastMetadata.genres.includes(g))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast genres "${existingGenres.join(',')}" => "${podcastMetadata.genres.join(',')}" for podcast "${podcastMetadata.title}"`)
|
||||
media.genres = podcastMetadata.genres
|
||||
media.changed('genres', true)
|
||||
hasMediaChanges = true
|
||||
}
|
||||
} else if (key === 'tags') {
|
||||
const existingTags = media.tags || []
|
||||
if (podcastMetadata.tags.some(t => !existingTags.includes(t)) || existingTags.some(t => !podcastMetadata.tags.includes(t))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast tags "${existingTags.join(',')}" => "${podcastMetadata.tags.join(',')}" for podcast "${podcastMetadata.title}"`)
|
||||
media.tags = podcastMetadata.tags
|
||||
media.changed('tags', true)
|
||||
hasMediaChanges = true
|
||||
}
|
||||
} else if (podcastMetadata[key] !== media[key]) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast ${key} "${media[key]}" => "${podcastMetadata[key]}" for podcast "${podcastMetadata.title}"`)
|
||||
media[key] = podcastMetadata[key]
|
||||
hasMediaChanges = true
|
||||
}
|
||||
}
|
||||
|
||||
// If no cover then extract cover from audio file if available
|
||||
if (!media.coverPath && existingPodcastEpisodes.length) {
|
||||
const audioFiles = existingPodcastEpisodes.map(ep => ep.audioFile)
|
||||
const extractedCoverPath = await CoverManager.saveEmbeddedCoverArtNew(audioFiles, existingLibraryItem.id, existingLibraryItem.path)
|
||||
if (extractedCoverPath) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating podcast "${podcastMetadata.title}" extracted embedded cover art from audio file to path "${extractedCoverPath}"`)
|
||||
media.coverPath = extractedCoverPath
|
||||
hasMediaChanges = true
|
||||
}
|
||||
}
|
||||
|
||||
existingLibraryItem.media = media
|
||||
|
||||
let libraryItemUpdated = false
|
||||
|
||||
// Save Podcast changes to db
|
||||
if (hasMediaChanges) {
|
||||
await media.save()
|
||||
await this.saveMetadataFile(existingLibraryItem, libraryScan)
|
||||
libraryItemUpdated = global.ServerSettings.storeMetadataWithItem
|
||||
}
|
||||
|
||||
if (libraryItemUpdated) {
|
||||
existingLibraryItem.changed('libraryFiles', true)
|
||||
await existingLibraryItem.save()
|
||||
}
|
||||
|
||||
return existingLibraryItem
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise<import('../models/LibraryItem')>}
|
||||
*/
|
||||
async scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
// Scan audio files found
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryItemData.mediaType, libraryItemData, libraryItemData.audioLibraryFiles)
|
||||
|
||||
// Do not add library items that have no valid audio files
|
||||
if (!scannedAudioFiles.length) {
|
||||
libraryScan.addLog(LogLevel.WARN, `Library item at path "${libraryItemData.relPath}" has no audio files - ignoring`)
|
||||
return null
|
||||
}
|
||||
|
||||
const newPodcastEpisodes = []
|
||||
|
||||
// Create podcast episodes from audio files
|
||||
for (const audioFile of scannedAudioFiles) {
|
||||
const newEpisode = {
|
||||
title: audioFile.metaTags.tagTitle || audioFile.metadata.filenameNoExt,
|
||||
subtitle: null,
|
||||
season: null,
|
||||
episode: null,
|
||||
episodeType: null,
|
||||
pubDate: null,
|
||||
publishedAt: null,
|
||||
description: null,
|
||||
audioFile: audioFile.toJSON(),
|
||||
chapters: audioFile.chapters || []
|
||||
}
|
||||
|
||||
// Set metadata and save new episode
|
||||
this.setPodcastEpisodeMetadataFromAudioFile(newEpisode, libraryScan)
|
||||
libraryScan.addLog(LogLevel.INFO, `New Podcast episode "${newEpisode.title}" found`)
|
||||
newPodcastEpisodes.push(newEpisode)
|
||||
}
|
||||
|
||||
const podcastMetadata = await this.getPodcastMetadataFromScanData(newPodcastEpisodes, libraryItemData, libraryScan)
|
||||
podcastMetadata.explicit = !!podcastMetadata.explicit // Ensure boolean
|
||||
|
||||
// Set cover image from library file
|
||||
if (libraryItemData.imageLibraryFiles.length) {
|
||||
// Prefer using a cover image with the name "cover" otherwise use the first image
|
||||
const coverMatch = libraryItemData.imageLibraryFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
|
||||
podcastMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
|
||||
}
|
||||
|
||||
// Set default podcastType to episodic
|
||||
if (!podcastMetadata.podcastType) {
|
||||
podcastMetadata.podcastType = 'episodic'
|
||||
}
|
||||
|
||||
const podcastObject = {
|
||||
...podcastMetadata,
|
||||
autoDownloadEpisodes: false,
|
||||
autoDownloadSchedule: '0 * * * *',
|
||||
lastEpisodeCheck: 0,
|
||||
maxEpisodesToKeep: 0,
|
||||
maxNewEpisodesToDownload: 3,
|
||||
podcastEpisodes: newPodcastEpisodes
|
||||
}
|
||||
|
||||
const libraryItemObj = libraryItemData.libraryItemObject
|
||||
libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image
|
||||
libraryItemObj.isMissing = false
|
||||
libraryItemObj.isInvalid = false
|
||||
libraryItemObj.extraData = {}
|
||||
|
||||
// If cover was not found in folder then check embedded covers in audio files
|
||||
if (!podcastObject.coverPath && scannedAudioFiles.length) {
|
||||
// Extract and save embedded cover art
|
||||
podcastObject.coverPath = await CoverManager.saveEmbeddedCoverArtNew(scannedAudioFiles, libraryItemObj.id, libraryItemObj.path)
|
||||
}
|
||||
|
||||
libraryItemObj.podcast = podcastObject
|
||||
const libraryItem = await Database.libraryItemModel.create(libraryItemObj, {
|
||||
include: {
|
||||
model: Database.podcastModel,
|
||||
include: Database.podcastEpisodeModel
|
||||
}
|
||||
})
|
||||
|
||||
Database.addGenresToFilterData(libraryItemData.libraryId, libraryItem.podcast.genres)
|
||||
Database.addTagsToFilterData(libraryItemData.libraryId, libraryItem.podcast.tags)
|
||||
|
||||
// Load for emitting to client
|
||||
libraryItem.media = await libraryItem.getMedia({
|
||||
include: Database.podcastEpisodeModel
|
||||
})
|
||||
|
||||
await this.saveMetadataFile(libraryItem, libraryScan)
|
||||
if (global.ServerSettings.storeMetadataWithItem) {
|
||||
libraryItem.changed('libraryFiles', true)
|
||||
await libraryItem.save()
|
||||
}
|
||||
|
||||
return libraryItem
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {PodcastEpisode[]} podcastEpisodes Not the models for new podcasts
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise<PodcastMetadataObject>}
|
||||
*/
|
||||
async getPodcastMetadataFromScanData(podcastEpisodes, libraryItemData, libraryScan) {
|
||||
const podcastMetadata = {
|
||||
title: libraryItemData.mediaMetadata.title,
|
||||
titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
|
||||
author: undefined,
|
||||
releaseDate: undefined,
|
||||
feedURL: undefined,
|
||||
imageURL: undefined,
|
||||
description: undefined,
|
||||
itunesPageURL: undefined,
|
||||
itunesId: undefined,
|
||||
itunesArtistId: undefined,
|
||||
language: undefined,
|
||||
podcastType: undefined,
|
||||
explicit: undefined,
|
||||
tags: [],
|
||||
genres: []
|
||||
}
|
||||
|
||||
if (podcastEpisodes.length) {
|
||||
const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags
|
||||
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
|
||||
|
||||
const MetadataMapArray = [
|
||||
{
|
||||
tag: 'tagAlbum',
|
||||
altTag: 'tagSeries',
|
||||
key: 'title'
|
||||
},
|
||||
{
|
||||
tag: 'tagArtist',
|
||||
key: 'author'
|
||||
},
|
||||
{
|
||||
tag: 'tagGenre',
|
||||
key: 'genres'
|
||||
},
|
||||
{
|
||||
tag: 'tagLanguage',
|
||||
key: 'language'
|
||||
},
|
||||
{
|
||||
tag: 'tagItunesId',
|
||||
key: 'itunesId'
|
||||
},
|
||||
{
|
||||
tag: 'tagPodcastType',
|
||||
key: 'podcastType',
|
||||
}
|
||||
]
|
||||
|
||||
MetadataMapArray.forEach((mapping) => {
|
||||
let value = audioFileMetaTags[mapping.tag]
|
||||
let tagToUse = mapping.tag
|
||||
if (!value && mapping.altTag) {
|
||||
value = audioFileMetaTags[mapping.altTag]
|
||||
tagToUse = mapping.altTag
|
||||
}
|
||||
|
||||
if (value && typeof value === 'string') {
|
||||
value = value.trim() // Trim whitespace
|
||||
|
||||
if (mapping.key === 'genres' && (!podcastMetadata.genres.length || overrideExistingDetails)) {
|
||||
podcastMetadata.genres = this.parseGenresString(value)
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`)
|
||||
} else if (!podcastMetadata[mapping.key] || overrideExistingDetails) {
|
||||
podcastMetadata[mapping.key] = value
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// If metadata.json or metadata.abs use this for metadata
|
||||
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
|
||||
const metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
|
||||
if (metadataText) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataLibraryFile.metadata.path}" - preferring`)
|
||||
let abMetadata = null
|
||||
if (!!libraryItemData.metadataJsonLibraryFile) {
|
||||
abMetadata = abmetadataGenerator.parseJson(metadataText)
|
||||
} else {
|
||||
abMetadata = abmetadataGenerator.parse(metadataText, 'podcast')
|
||||
}
|
||||
|
||||
if (abMetadata) {
|
||||
if (abMetadata.tags?.length) {
|
||||
podcastMetadata.tags = abMetadata.tags
|
||||
}
|
||||
for (const key in abMetadata.metadata) {
|
||||
if (abMetadata.metadata[key] === undefined) continue
|
||||
|
||||
// TODO: New podcast model changed some keys, need to update the abmetadataGenerator
|
||||
let newModelKey = key
|
||||
if (key === 'feedUrl') newModelKey = 'feedURL'
|
||||
else if (key === 'imageUrl') newModelKey = 'imageURL'
|
||||
else if (key === 'itunesPageUrl') newModelKey = 'itunesPageURL'
|
||||
else if (key === 'type') newModelKey = 'podcastType'
|
||||
|
||||
podcastMetadata[newModelKey] = abMetadata.metadata[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
podcastMetadata.titleIgnorePrefix = getTitleIgnorePrefix(podcastMetadata.title)
|
||||
|
||||
return podcastMetadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a genre string into multiple genres
|
||||
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
|
||||
* @param {string} genreTag
|
||||
* @returns {string[]}
|
||||
*/
|
||||
parseGenresString(genreTag) {
|
||||
if (!genreTag?.length) return []
|
||||
const separators = ['/', '//', ';']
|
||||
for (let i = 0; i < separators.length; i++) {
|
||||
if (genreTag.includes(separators[i])) {
|
||||
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
|
||||
}
|
||||
}
|
||||
return [genreTag]
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('../models/LibraryItem')} libraryItem
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {Promise}
|
||||
*/
|
||||
async saveMetadataFile(libraryItem, libraryScan) {
|
||||
let metadataPath = Path.join(global.MetadataPath, 'items', libraryItem.id)
|
||||
let storeMetadataWithItem = global.ServerSettings.storeMetadataWithItem
|
||||
if (storeMetadataWithItem) {
|
||||
metadataPath = libraryItem.path
|
||||
} else {
|
||||
// Make sure metadata book dir exists
|
||||
storeMetadataWithItem = false
|
||||
await fsExtra.ensureDir(metadataPath)
|
||||
}
|
||||
|
||||
const metadataFileFormat = global.ServerSettings.metadataFileFormat
|
||||
const metadataFilePath = Path.join(metadataPath, `metadata.${metadataFileFormat}`)
|
||||
if (metadataFileFormat === 'json') {
|
||||
// Remove metadata.abs if it exists
|
||||
if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.abs`))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.abs for item "${libraryItem.media.title}"`)
|
||||
await fsExtra.remove(Path.join(metadataPath, `metadata.abs`))
|
||||
libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.abs`)))
|
||||
}
|
||||
|
||||
// TODO: Update to not use `metadata` so it fits the updated model
|
||||
const jsonObject = {
|
||||
tags: libraryItem.media.tags || [],
|
||||
metadata: {
|
||||
title: libraryItem.media.title,
|
||||
author: libraryItem.media.author,
|
||||
description: libraryItem.media.description,
|
||||
releaseDate: libraryItem.media.releaseDate,
|
||||
genres: libraryItem.media.genres || [],
|
||||
feedUrl: libraryItem.media.feedURL,
|
||||
imageUrl: libraryItem.media.imageURL,
|
||||
itunesPageUrl: libraryItem.media.itunesPageURL,
|
||||
itunesId: libraryItem.media.itunesId,
|
||||
itunesArtistId: libraryItem.media.itunesArtistId,
|
||||
asin: libraryItem.media.asin,
|
||||
language: libraryItem.media.language,
|
||||
explicit: !!libraryItem.media.explicit,
|
||||
type: libraryItem.media.podcastType
|
||||
}
|
||||
}
|
||||
return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
|
||||
// Add metadata.json to libraryFiles array if it is new
|
||||
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
|
||||
if (storeMetadataWithItem && !metadataLibraryFile) {
|
||||
const newLibraryFile = new LibraryFile()
|
||||
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
|
||||
metadataLibraryFile = newLibraryFile.toJSON()
|
||||
libraryItem.libraryFiles.push(metadataLibraryFile)
|
||||
} else if (storeMetadataWithItem) {
|
||||
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
|
||||
if (fileTimestamps) {
|
||||
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
|
||||
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
|
||||
metadataLibraryFile.metadata.size = fileTimestamps.size
|
||||
metadataLibraryFile.ino = fileTimestamps.ino
|
||||
}
|
||||
}
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
|
||||
|
||||
return metadataLibraryFile
|
||||
}).catch((error) => {
|
||||
libraryScan.addLog(LogLevel.ERROR, `Failed to save json file at "${metadataFilePath}"`, error)
|
||||
return null
|
||||
})
|
||||
} else {
|
||||
// Remove metadata.json if it exists
|
||||
if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.json`))) {
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Removing metadata.json for item "${libraryItem.media.title}"`)
|
||||
await fsExtra.remove(Path.join(metadataPath, `metadata.json`))
|
||||
libraryItem.libraryFiles = libraryItem.libraryFiles.filter(lf => lf.metadata.path !== filePathToPOSIX(Path.join(metadataPath, `metadata.json`)))
|
||||
}
|
||||
|
||||
return abmetadataGenerator.generateFromNewModel(libraryItem, metadataFilePath).then(async (success) => {
|
||||
if (!success) {
|
||||
libraryScan.addLog(LogLevel.ERROR, `Failed saving abmetadata to "${metadataFilePath}"`)
|
||||
return null
|
||||
}
|
||||
// Add metadata.abs to libraryFiles array if it is new
|
||||
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
|
||||
if (storeMetadataWithItem && !metadataLibraryFile) {
|
||||
const newLibraryFile = new LibraryFile()
|
||||
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
|
||||
metadataLibraryFile = newLibraryFile.toJSON()
|
||||
libraryItem.libraryFiles.push(metadataLibraryFile)
|
||||
} else if (storeMetadataWithItem) {
|
||||
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
|
||||
if (fileTimestamps) {
|
||||
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
|
||||
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
|
||||
metadataLibraryFile.metadata.size = fileTimestamps.size
|
||||
metadataLibraryFile.ino = fileTimestamps.ino
|
||||
}
|
||||
}
|
||||
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
|
||||
return metadataLibraryFile
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {PodcastEpisode} podcastEpisode Not the model when creating new podcast
|
||||
* @param {import('./ScanLogger')} scanLogger
|
||||
*/
|
||||
setPodcastEpisodeMetadataFromAudioFile(podcastEpisode, scanLogger) {
|
||||
const MetadataMapArray = [
|
||||
{
|
||||
tag: 'tagComment',
|
||||
altTag: 'tagSubtitle',
|
||||
key: 'description'
|
||||
},
|
||||
{
|
||||
tag: 'tagSubtitle',
|
||||
key: 'subtitle'
|
||||
},
|
||||
{
|
||||
tag: 'tagDate',
|
||||
key: 'pubDate'
|
||||
},
|
||||
{
|
||||
tag: 'tagDisc',
|
||||
key: 'season',
|
||||
},
|
||||
{
|
||||
tag: 'tagTrack',
|
||||
altTag: 'tagSeriesPart',
|
||||
key: 'episode'
|
||||
},
|
||||
{
|
||||
tag: 'tagTitle',
|
||||
key: 'title'
|
||||
},
|
||||
{
|
||||
tag: 'tagEpisodeType',
|
||||
key: 'episodeType'
|
||||
}
|
||||
]
|
||||
|
||||
const audioFileMetaTags = podcastEpisode.audioFile.metaTags
|
||||
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
|
||||
MetadataMapArray.forEach((mapping) => {
|
||||
let value = audioFileMetaTags[mapping.tag]
|
||||
let tagToUse = mapping.tag
|
||||
if (!value && mapping.altTag) {
|
||||
tagToUse = mapping.altTag
|
||||
value = audioFileMetaTags[mapping.altTag]
|
||||
}
|
||||
|
||||
if (value && typeof value === 'string') {
|
||||
value = value.trim() // Trim whitespace
|
||||
|
||||
if (mapping.key === 'pubDate' && (!podcastEpisode.pubDate || overrideExistingDetails)) {
|
||||
const pubJsDate = new Date(value)
|
||||
if (pubJsDate && !isNaN(pubJsDate)) {
|
||||
podcastEpisode.publishedAt = pubJsDate.valueOf()
|
||||
podcastEpisode.pubDate = value
|
||||
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
|
||||
} else {
|
||||
scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`)
|
||||
}
|
||||
} else if (mapping.key === 'episodeType' && (!podcastEpisode.episodeType || overrideExistingDetails)) {
|
||||
if (['full', 'trailer', 'bonus'].includes(value)) {
|
||||
podcastEpisode.episodeType = value
|
||||
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
|
||||
} else {
|
||||
scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`)
|
||||
}
|
||||
} else if (!podcastEpisode[mapping.key] || overrideExistingDetails) {
|
||||
podcastEpisode[mapping.key] = value
|
||||
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
module.exports = new PodcastScanner()
|
||||
@ -1,27 +1,17 @@
|
||||
const Sequelize = require('sequelize')
|
||||
const fs = require('../libs/fsExtra')
|
||||
const Path = require('path')
|
||||
const Logger = require('../Logger')
|
||||
const SocketAuthority = require('../SocketAuthority')
|
||||
const Database = require('../Database')
|
||||
|
||||
// Utils
|
||||
const { groupFilesIntoLibraryItemPaths, getLibraryItemFileData, scanFolder, checkFilepathIsAudioFile } = require('../utils/scandir')
|
||||
const { comparePaths } = require('../utils/index')
|
||||
const { getIno, filePathToPOSIX } = require('../utils/fileUtils')
|
||||
const { ScanResult, LogLevel } = require('../utils/constants')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils')
|
||||
|
||||
const MediaFileScanner = require('./MediaFileScanner')
|
||||
const BookFinder = require('../finders/BookFinder')
|
||||
const PodcastFinder = require('../finders/PodcastFinder')
|
||||
const LibraryItem = require('../objects/LibraryItem')
|
||||
const LibraryScan = require('./LibraryScan')
|
||||
const ScanOptions = require('./ScanOptions')
|
||||
|
||||
const Author = require('../objects/entities/Author')
|
||||
const Series = require('../objects/entities/Series')
|
||||
const Task = require('../objects/Task')
|
||||
|
||||
class Scanner {
|
||||
constructor(coverManager, taskManager) {
|
||||
@ -31,710 +21,10 @@ class Scanner {
|
||||
this.cancelLibraryScan = {}
|
||||
this.librariesScanning = []
|
||||
|
||||
// Watcher file update scan vars
|
||||
this.pendingFileUpdatesToScan = []
|
||||
this.scanningFilesChanged = false
|
||||
|
||||
this.bookFinder = new BookFinder()
|
||||
this.podcastFinder = new PodcastFinder()
|
||||
}
|
||||
|
||||
isLibraryScanning(libraryId) {
|
||||
return this.librariesScanning.find(ls => ls.id === libraryId)
|
||||
}
|
||||
|
||||
setCancelLibraryScan(libraryId) {
|
||||
var libraryScanning = this.librariesScanning.find(ls => ls.id === libraryId)
|
||||
if (!libraryScanning) return
|
||||
this.cancelLibraryScan[libraryId] = true
|
||||
}
|
||||
|
||||
getScanResultDescription(result) {
|
||||
switch (result) {
|
||||
case ScanResult.ADDED:
|
||||
return 'Added to library'
|
||||
case ScanResult.NOTHING:
|
||||
return 'No updates necessary'
|
||||
case ScanResult.REMOVED:
|
||||
return 'Removed from library'
|
||||
case ScanResult.UPDATED:
|
||||
return 'Item was updated'
|
||||
case ScanResult.UPTODATE:
|
||||
return 'No updates necessary'
|
||||
default:
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
async scanLibraryItemByRequest(libraryItem) {
|
||||
const library = await Database.libraryModel.getOldById(libraryItem.libraryId)
|
||||
if (!library) {
|
||||
Logger.error(`[Scanner] Scan libraryItem by id library not found "${libraryItem.libraryId}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
const folder = library.folders.find(f => f.id === libraryItem.folderId)
|
||||
if (!folder) {
|
||||
Logger.error(`[Scanner] Scan libraryItem by id folder not found "${libraryItem.folderId}" in library "${library.name}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
Logger.info(`[Scanner] Scanning Library Item "${libraryItem.media.metadata.title}"`)
|
||||
|
||||
const task = new Task()
|
||||
task.setData('scan-item', `Scan ${libraryItem.media.metadata.title}`, '', true, {
|
||||
libraryItemId: libraryItem.id,
|
||||
libraryId: library.id,
|
||||
mediaType: library.mediaType
|
||||
})
|
||||
this.taskManager.addTask(task)
|
||||
|
||||
const result = await this.scanLibraryItem(library, folder, libraryItem)
|
||||
|
||||
task.setFinished(this.getScanResultDescription(result))
|
||||
this.taskManager.taskFinished(task)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async scanLibraryItem(library, folder, libraryItem) {
|
||||
const libraryMediaType = library.mediaType
|
||||
|
||||
// TODO: Support for single media item
|
||||
const libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, libraryItem.path, false)
|
||||
if (!libraryItemData) {
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
let hasUpdated = false
|
||||
|
||||
const checkRes = libraryItem.checkScanData(libraryItemData)
|
||||
if (checkRes.updated) hasUpdated = true
|
||||
|
||||
// Sync other files first so that local images are used as cover art
|
||||
if (await libraryItem.syncFiles(Database.serverSettings.scannerPreferOpfMetadata, library.settings)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// Scan all audio files
|
||||
if (libraryItem.hasAudioFiles) {
|
||||
const libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio')
|
||||
if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItem)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) {
|
||||
const coverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem)
|
||||
if (coverPath) {
|
||||
Logger.debug(`[Scanner] Saved embedded cover art "${coverPath}"`)
|
||||
hasUpdated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.createNewAuthorsAndSeries(libraryItem)
|
||||
|
||||
// Library Item is invalid - (a book has no audio files or ebook files)
|
||||
if (!libraryItem.hasMediaEntities && libraryItem.mediaType !== 'podcast') {
|
||||
libraryItem.setInvalid()
|
||||
hasUpdated = true
|
||||
} else if (libraryItem.isInvalid) {
|
||||
libraryItem.isInvalid = false
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
if (hasUpdated) {
|
||||
await Database.updateLibraryItem(libraryItem)
|
||||
SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
|
||||
return ScanResult.UPDATED
|
||||
}
|
||||
return ScanResult.UPTODATE
|
||||
}
|
||||
|
||||
async scan(library, options = {}) {
|
||||
if (this.isLibraryScanning(library.id)) {
|
||||
Logger.error(`[Scanner] Already scanning ${library.id}`)
|
||||
return
|
||||
}
|
||||
|
||||
if (!library.folders.length) {
|
||||
Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
|
||||
return
|
||||
}
|
||||
|
||||
const scanOptions = new ScanOptions()
|
||||
scanOptions.setData(options, Database.serverSettings)
|
||||
|
||||
const libraryScan = new LibraryScan()
|
||||
libraryScan.setData(library, scanOptions)
|
||||
libraryScan.verbose = false
|
||||
this.librariesScanning.push(libraryScan.getScanEmitData)
|
||||
|
||||
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
|
||||
|
||||
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
||||
|
||||
const canceled = await this.scanLibrary(libraryScan)
|
||||
|
||||
if (canceled) {
|
||||
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
|
||||
delete this.cancelLibraryScan[libraryScan.libraryId]
|
||||
}
|
||||
|
||||
libraryScan.setComplete()
|
||||
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
|
||||
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
||||
|
||||
if (canceled && !libraryScan.totalResults) {
|
||||
const emitData = libraryScan.getScanEmitData
|
||||
emitData.results = null
|
||||
SocketAuthority.emitter('scan_complete', emitData)
|
||||
return
|
||||
}
|
||||
|
||||
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
|
||||
|
||||
if (libraryScan.totalResults) {
|
||||
libraryScan.saveLog()
|
||||
}
|
||||
}
|
||||
|
||||
async scanLibrary(libraryScan) {
|
||||
let libraryItemDataFound = []
|
||||
|
||||
// Scan each library
|
||||
for (let i = 0; i < libraryScan.folders.length; i++) {
|
||||
const folder = libraryScan.folders[i]
|
||||
const itemDataFoundInFolder = await scanFolder(libraryScan.library, folder)
|
||||
libraryScan.addLog(LogLevel.INFO, `${itemDataFoundInFolder.length} item data found in folder "${folder.fullPath}"`)
|
||||
libraryItemDataFound = libraryItemDataFound.concat(itemDataFoundInFolder)
|
||||
}
|
||||
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
|
||||
// Remove items with no inode
|
||||
libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino)
|
||||
const libraryItemsInLibrary = Database.libraryItems.filter(li => li.libraryId === libraryScan.libraryId)
|
||||
|
||||
const MaxSizePerChunk = 2.5e9
|
||||
const itemDataToRescanChunks = []
|
||||
const newItemDataToScanChunks = []
|
||||
let itemsToUpdate = []
|
||||
let itemDataToRescan = []
|
||||
let itemDataToRescanSize = 0
|
||||
let newItemDataToScan = []
|
||||
let newItemDataToScanSize = 0
|
||||
const itemsToFindCovers = []
|
||||
|
||||
// Check for existing & removed library items
|
||||
for (let i = 0; i < libraryItemsInLibrary.length; i++) {
|
||||
const libraryItem = libraryItemsInLibrary[i]
|
||||
// Find library item folder with matching inode or matching path
|
||||
const dataFound = libraryItemDataFound.find(lid => lid.ino === libraryItem.ino || comparePaths(lid.relPath, libraryItem.relPath))
|
||||
if (!dataFound) {
|
||||
// Podcast folder can have no episodes and still be valid
|
||||
if (libraryScan.libraryMediaType === 'podcast' && await fs.pathExists(libraryItem.path)) {
|
||||
Logger.info(`[Scanner] Library item "${libraryItem.media.metadata.title}" folder exists but has no episodes`)
|
||||
if (libraryItem.isMissing) {
|
||||
libraryScan.resultsUpdated++
|
||||
libraryItem.isMissing = false
|
||||
libraryItem.setLastScan()
|
||||
itemsToUpdate.push(libraryItem)
|
||||
}
|
||||
} else {
|
||||
libraryScan.addLog(LogLevel.WARN, `Library Item "${libraryItem.media.metadata.title}" is missing`)
|
||||
Logger.warn(`[Scanner] Library item "${libraryItem.media.metadata.title}" is missing (inode "${libraryItem.ino}")`)
|
||||
libraryScan.resultsMissing++
|
||||
libraryItem.setMissing()
|
||||
itemsToUpdate.push(libraryItem)
|
||||
}
|
||||
} else {
|
||||
const checkRes = libraryItem.checkScanData(dataFound)
|
||||
if (checkRes.newLibraryFiles.length || libraryScan.scanOptions.forceRescan) { // Item has new files
|
||||
checkRes.libraryItem = libraryItem
|
||||
checkRes.scanData = dataFound
|
||||
|
||||
// If this item will go over max size then push current chunk
|
||||
if (libraryItem.audioFileTotalSize + itemDataToRescanSize > MaxSizePerChunk && itemDataToRescan.length > 0) {
|
||||
itemDataToRescanChunks.push(itemDataToRescan)
|
||||
itemDataToRescanSize = 0
|
||||
itemDataToRescan = []
|
||||
}
|
||||
|
||||
itemDataToRescan.push(checkRes)
|
||||
itemDataToRescanSize += libraryItem.audioFileTotalSize
|
||||
if (itemDataToRescanSize >= MaxSizePerChunk) {
|
||||
itemDataToRescanChunks.push(itemDataToRescan)
|
||||
itemDataToRescanSize = 0
|
||||
itemDataToRescan = []
|
||||
}
|
||||
|
||||
} else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) { // Search cover
|
||||
libraryScan.resultsUpdated++
|
||||
itemsToFindCovers.push(libraryItem)
|
||||
itemsToUpdate.push(libraryItem)
|
||||
} else if (checkRes.updated) { // Updated but no scan required
|
||||
libraryScan.resultsUpdated++
|
||||
itemsToUpdate.push(libraryItem)
|
||||
}
|
||||
libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino !== dataFound.ino)
|
||||
}
|
||||
}
|
||||
if (itemDataToRescan.length) itemDataToRescanChunks.push(itemDataToRescan)
|
||||
|
||||
// Potential NEW Library Items
|
||||
for (let i = 0; i < libraryItemDataFound.length; i++) {
|
||||
const dataFound = libraryItemDataFound[i]
|
||||
|
||||
const hasMediaFile = dataFound.libraryFiles.some(lf => lf.isMediaFile)
|
||||
if (!hasMediaFile) {
|
||||
libraryScan.addLog(LogLevel.WARN, `Item found "${libraryItemDataFound.path}" has no media files`)
|
||||
} else {
|
||||
// If this item will go over max size then push current chunk
|
||||
let mediaFileSize = 0
|
||||
dataFound.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video').forEach(lf => mediaFileSize += lf.metadata.size)
|
||||
if (mediaFileSize + newItemDataToScanSize > MaxSizePerChunk && newItemDataToScan.length > 0) {
|
||||
newItemDataToScanChunks.push(newItemDataToScan)
|
||||
newItemDataToScanSize = 0
|
||||
newItemDataToScan = []
|
||||
}
|
||||
|
||||
newItemDataToScan.push(dataFound)
|
||||
newItemDataToScanSize += mediaFileSize
|
||||
|
||||
if (newItemDataToScanSize >= MaxSizePerChunk) {
|
||||
newItemDataToScanChunks.push(newItemDataToScan)
|
||||
newItemDataToScanSize = 0
|
||||
newItemDataToScan = []
|
||||
}
|
||||
}
|
||||
}
|
||||
if (newItemDataToScan.length) newItemDataToScanChunks.push(newItemDataToScan)
|
||||
|
||||
// Library Items not requiring a scan but require a search for cover
|
||||
for (let i = 0; i < itemsToFindCovers.length; i++) {
|
||||
const libraryItem = itemsToFindCovers[i]
|
||||
const updatedCover = await this.searchForCover(libraryItem, libraryScan)
|
||||
libraryItem.media.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
|
||||
if (itemsToUpdate.length) {
|
||||
await this.updateLibraryItemChunk(itemsToUpdate)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
}
|
||||
|
||||
// Chunking will be removed when legacy single threaded scanner is removed
|
||||
for (let i = 0; i < itemDataToRescanChunks.length; i++) {
|
||||
await this.rescanLibraryItemDataChunk(itemDataToRescanChunks[i], libraryScan)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
}
|
||||
for (let i = 0; i < newItemDataToScanChunks.length; i++) {
|
||||
await this.scanNewLibraryItemDataChunk(newItemDataToScanChunks[i], libraryScan)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
}
|
||||
}
|
||||
|
||||
async updateLibraryItemChunk(itemsToUpdate) {
|
||||
await Database.updateBulkLibraryItems(itemsToUpdate)
|
||||
SocketAuthority.emitter('items_updated', itemsToUpdate.map(li => li.toJSONExpanded()))
|
||||
}
|
||||
|
||||
async rescanLibraryItemDataChunk(itemDataToRescan, libraryScan) {
|
||||
var itemsUpdated = await Promise.all(itemDataToRescan.map((lid) => {
|
||||
return this.rescanLibraryItem(lid, libraryScan)
|
||||
}))
|
||||
|
||||
itemsUpdated = itemsUpdated.filter(li => li) // Filter out nulls
|
||||
|
||||
for (const libraryItem of itemsUpdated) {
|
||||
// Temp authors & series are inserted - create them if found
|
||||
await this.createNewAuthorsAndSeries(libraryItem)
|
||||
}
|
||||
|
||||
if (itemsUpdated.length) {
|
||||
libraryScan.resultsUpdated += itemsUpdated.length
|
||||
await Database.updateBulkLibraryItems(itemsUpdated)
|
||||
SocketAuthority.emitter('items_updated', itemsUpdated.map(li => li.toJSONExpanded()))
|
||||
}
|
||||
}
|
||||
|
||||
async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) {
|
||||
let newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => {
|
||||
return this.scanNewLibraryItem(lid, libraryScan.library, libraryScan)
|
||||
}))
|
||||
newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls
|
||||
|
||||
for (const libraryItem of newLibraryItems) {
|
||||
// Temp authors & series are inserted - create them if found
|
||||
await this.createNewAuthorsAndSeries(libraryItem)
|
||||
}
|
||||
|
||||
libraryScan.resultsAdded += newLibraryItems.length
|
||||
await Database.createBulkLibraryItems(newLibraryItems)
|
||||
SocketAuthority.emitter('items_added', newLibraryItems.map(li => li.toJSONExpanded()))
|
||||
}
|
||||
|
||||
async rescanLibraryItem(libraryItemCheckData, libraryScan) {
|
||||
const { newLibraryFiles, filesRemoved, existingLibraryFiles, libraryItem, scanData, updated } = libraryItemCheckData
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${libraryItem.path}"`)
|
||||
let hasUpdated = updated
|
||||
|
||||
// Sync other files first to use local images as cover before extracting audio file cover
|
||||
if (await libraryItem.syncFiles(libraryScan.preferOpfMetadata, libraryScan.library.settings)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// forceRescan all existing audio files - will probe and update ID3 tag metadata
|
||||
const existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio')
|
||||
if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) {
|
||||
if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, libraryItem, libraryScan)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
}
|
||||
// Scan new audio files
|
||||
const newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio')
|
||||
const removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio')
|
||||
if (newAudioFiles.length || removedAudioFiles.length) {
|
||||
if (await MediaFileScanner.scanMediaFiles(newAudioFiles, libraryItem, libraryScan)) {
|
||||
hasUpdated = true
|
||||
}
|
||||
}
|
||||
// If an audio file has embedded cover art and no cover is set yet, extract & use it
|
||||
if (newAudioFiles.length || libraryScan.scanOptions.forceRescan) {
|
||||
if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) {
|
||||
const savedCoverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem)
|
||||
if (savedCoverPath) {
|
||||
hasUpdated = true
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${savedCoverPath}"`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Library Item is invalid - (a book has no audio files or ebook files)
|
||||
if (!libraryItem.hasMediaEntities && libraryItem.mediaType !== 'podcast') {
|
||||
libraryItem.setInvalid()
|
||||
hasUpdated = true
|
||||
} else if (libraryItem.isInvalid) {
|
||||
libraryItem.isInvalid = false
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
// Scan for cover if enabled and has no cover (and author or title has changed OR has been 7 days since last lookup)
|
||||
if (libraryScan.findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) {
|
||||
const updatedCover = await this.searchForCover(libraryItem, libraryScan)
|
||||
libraryItem.media.updateLastCoverSearch(updatedCover)
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
return hasUpdated ? libraryItem : null
|
||||
}
|
||||
|
||||
async scanNewLibraryItem(libraryItemData, library, libraryScan = null) {
|
||||
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`)
|
||||
else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`)
|
||||
|
||||
const preferOpfMetadata = libraryScan ? !!libraryScan.preferOpfMetadata : !!global.ServerSettings.scannerPreferOpfMetadata
|
||||
const findCovers = libraryScan ? !!libraryScan.findCovers : !!global.ServerSettings.scannerFindCovers
|
||||
|
||||
const libraryItem = new LibraryItem()
|
||||
libraryItem.setData(library.mediaType, libraryItemData)
|
||||
libraryItem.setLastScan()
|
||||
|
||||
const mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video')
|
||||
if (mediaFiles.length) {
|
||||
await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItem, libraryScan)
|
||||
}
|
||||
|
||||
await libraryItem.syncFiles(preferOpfMetadata, library.settings)
|
||||
|
||||
if (!libraryItem.hasMediaEntities) {
|
||||
Logger.warn(`[Scanner] Library item has no media files "${libraryItemData.path}"`)
|
||||
return null
|
||||
}
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (libraryItem.media.hasEmbeddedCoverArt && !libraryItem.media.coverPath) {
|
||||
const coverPath = await this.coverManager.saveEmbeddedCoverArt(libraryItem)
|
||||
if (coverPath) {
|
||||
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${coverPath}"`)
|
||||
else Logger.debug(`[Scanner] Saved embedded cover art "${coverPath}"`)
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for cover if enabled and has no cover
|
||||
if (library.isBook) {
|
||||
if (libraryItem && findCovers && !libraryItem.media.coverPath && libraryItem.media.shouldSearchForCover) {
|
||||
const updatedCover = await this.searchForCover(libraryItem, libraryScan)
|
||||
libraryItem.media.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
}
|
||||
|
||||
return libraryItem
|
||||
}
|
||||
|
||||
// Any series or author object on library item with an id starting with "new"
|
||||
// will create a new author/series OR find a matching author/series
|
||||
async createNewAuthorsAndSeries(libraryItem) {
|
||||
if (libraryItem.mediaType !== 'book') return
|
||||
|
||||
// Create or match all new authors and series
|
||||
if (libraryItem.media.metadata.authors.some(au => au.id.startsWith('new'))) {
|
||||
const newAuthors = []
|
||||
libraryItem.media.metadata.authors = Promise.all(libraryItem.media.metadata.authors.map(async (tempMinAuthor) => {
|
||||
let _author = await Database.authorModel.getOldByNameAndLibrary(tempMinAuthor.name, libraryItem.libraryId)
|
||||
if (!_author) _author = newAuthors.find(au => au.libraryId === libraryItem.libraryId && au.checkNameEquals(tempMinAuthor.name)) // Check new unsaved authors
|
||||
if (!_author) { // Must create new author
|
||||
_author = new Author()
|
||||
_author.setData(tempMinAuthor, libraryItem.libraryId)
|
||||
newAuthors.push(_author)
|
||||
// Update filter data
|
||||
Database.addAuthorToFilterData(libraryItem.libraryId, _author.name, _author.id)
|
||||
}
|
||||
|
||||
return {
|
||||
id: _author.id,
|
||||
name: _author.name
|
||||
}
|
||||
}))
|
||||
if (newAuthors.length) {
|
||||
await Database.createBulkAuthors(newAuthors)
|
||||
SocketAuthority.emitter('authors_added', newAuthors.map(au => au.toJSON()))
|
||||
}
|
||||
}
|
||||
if (libraryItem.media.metadata.series.some(se => se.id.startsWith('new'))) {
|
||||
const newSeries = []
|
||||
libraryItem.media.metadata.series = await Promise.all(libraryItem.media.metadata.series.map(async (tempMinSeries) => {
|
||||
let _series = await Database.seriesModel.getOldByNameAndLibrary(tempMinSeries.name, libraryItem.libraryId)
|
||||
if (!_series) {
|
||||
// Check new unsaved series
|
||||
_series = newSeries.find(se => se.libraryId === libraryItem.libraryId && se.checkNameEquals(tempMinSeries.name))
|
||||
}
|
||||
|
||||
if (!_series) { // Must create new series
|
||||
_series = new Series()
|
||||
_series.setData(tempMinSeries, libraryItem.libraryId)
|
||||
newSeries.push(_series)
|
||||
// Update filter data
|
||||
Database.addSeriesToFilterData(libraryItem.libraryId, _series.name, _series.id)
|
||||
}
|
||||
return {
|
||||
id: _series.id,
|
||||
name: _series.name,
|
||||
sequence: tempMinSeries.sequence
|
||||
}
|
||||
}))
|
||||
if (newSeries.length) {
|
||||
await Database.createBulkSeries(newSeries)
|
||||
SocketAuthority.emitter('multiple_series_added', newSeries.map(se => se.toJSON()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getFileUpdatesGrouped(fileUpdates) {
|
||||
var folderGroups = {}
|
||||
fileUpdates.forEach((file) => {
|
||||
if (folderGroups[file.folderId]) {
|
||||
folderGroups[file.folderId].fileUpdates.push(file)
|
||||
} else {
|
||||
folderGroups[file.folderId] = {
|
||||
libraryId: file.libraryId,
|
||||
folderId: file.folderId,
|
||||
fileUpdates: [file]
|
||||
}
|
||||
}
|
||||
})
|
||||
return folderGroups
|
||||
}
|
||||
|
||||
async scanFilesChanged(fileUpdates) {
|
||||
if (!fileUpdates?.length) return
|
||||
|
||||
// If already scanning files from watcher then add these updates to queue
|
||||
if (this.scanningFilesChanged) {
|
||||
this.pendingFileUpdatesToScan.push(fileUpdates)
|
||||
Logger.debug(`[Scanner] Already scanning files from watcher - file updates pushed to queue (size ${this.pendingFileUpdatesToScan.length})`)
|
||||
return
|
||||
}
|
||||
this.scanningFilesChanged = true
|
||||
|
||||
// files grouped by folder
|
||||
const folderGroups = this.getFileUpdatesGrouped(fileUpdates)
|
||||
|
||||
for (const folderId in folderGroups) {
|
||||
const libraryId = folderGroups[folderId].libraryId
|
||||
const library = await Database.libraryModel.getOldById(libraryId)
|
||||
if (!library) {
|
||||
Logger.error(`[Scanner] Library not found in files changed ${libraryId}`)
|
||||
continue
|
||||
}
|
||||
const folder = library.getFolderById(folderId)
|
||||
if (!folder) {
|
||||
Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
|
||||
continue
|
||||
}
|
||||
const relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
|
||||
const fileUpdateGroup = groupFilesIntoLibraryItemPaths(library.mediaType, relFilePaths, false)
|
||||
|
||||
if (!Object.keys(fileUpdateGroup).length) {
|
||||
Logger.info(`[Scanner] No important changes to scan for in folder "${folderId}"`)
|
||||
continue
|
||||
}
|
||||
const folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateGroup)
|
||||
Logger.debug(`[Scanner] Folder scan results`, folderScanResults)
|
||||
|
||||
// If something was updated then reset numIssues filter data for library
|
||||
if (Object.values(folderScanResults).some(scanResult => scanResult !== ScanResult.NOTHING && scanResult !== ScanResult.UPTODATE)) {
|
||||
await Database.resetLibraryIssuesFilterData(libraryId)
|
||||
}
|
||||
}
|
||||
|
||||
this.scanningFilesChanged = false
|
||||
|
||||
if (this.pendingFileUpdatesToScan.length) {
|
||||
Logger.debug(`[Scanner] File updates finished scanning with more updates in queue (${this.pendingFileUpdatesToScan.length})`)
|
||||
this.scanFilesChanged(this.pendingFileUpdatesToScan.shift())
|
||||
}
|
||||
}
|
||||
|
||||
async scanFolderUpdates(library, folder, fileUpdateGroup) {
|
||||
Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
|
||||
Logger.debug(`[Scanner] scanFolderUpdates fileUpdateGroup`, fileUpdateGroup)
|
||||
|
||||
// First pass - Remove files in parent dirs of items and remap the fileupdate group
|
||||
// Test Case: Moving audio files from library item folder to author folder should trigger a re-scan of the item
|
||||
const updateGroup = { ...fileUpdateGroup }
|
||||
for (const itemDir in updateGroup) {
|
||||
if (itemDir == fileUpdateGroup[itemDir]) continue // Media in root path
|
||||
|
||||
const itemDirNestedFiles = fileUpdateGroup[itemDir].filter(b => b.includes('/'))
|
||||
if (!itemDirNestedFiles.length) continue
|
||||
|
||||
const firstNest = itemDirNestedFiles[0].split('/').shift()
|
||||
const altDir = `${itemDir}/${firstNest}`
|
||||
|
||||
const fullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir)
|
||||
const childLibraryItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[Sequelize.Op.not]: fullPath
|
||||
},
|
||||
path: {
|
||||
[Sequelize.Op.startsWith]: fullPath
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!childLibraryItem) {
|
||||
continue
|
||||
}
|
||||
|
||||
const altFullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), altDir)
|
||||
const altChildLibraryItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[Sequelize.Op.not]: altFullPath
|
||||
},
|
||||
path: {
|
||||
[Sequelize.Op.startsWith]: altFullPath
|
||||
}
|
||||
}
|
||||
})
|
||||
if (altChildLibraryItem) {
|
||||
continue
|
||||
}
|
||||
|
||||
delete fileUpdateGroup[itemDir]
|
||||
fileUpdateGroup[altDir] = itemDirNestedFiles.map((f) => f.split('/').slice(1).join('/'))
|
||||
Logger.warn(`[Scanner] Some files were modified in a parent directory of a library item "${childLibraryItem.path}" - ignoring`)
|
||||
}
|
||||
|
||||
// Second pass: Check for new/updated/removed items
|
||||
const itemGroupingResults = {}
|
||||
for (const itemDir in fileUpdateGroup) {
|
||||
const fullPath = Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir)
|
||||
const dirIno = await getIno(fullPath)
|
||||
|
||||
const itemDirParts = itemDir.split('/').slice(0, -1)
|
||||
const potentialChildDirs = []
|
||||
for (let i = 0; i < itemDirParts.length; i++) {
|
||||
potentialChildDirs.push(Path.posix.join(filePathToPOSIX(folder.fullPath), itemDir.split('/').slice(0, -1 - i).join('/')))
|
||||
}
|
||||
|
||||
// Check if book dir group is already an item
|
||||
let existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
path: potentialChildDirs
|
||||
})
|
||||
|
||||
if (!existingLibraryItem) {
|
||||
existingLibraryItem = await Database.libraryItemModel.findOneOld({
|
||||
ino: dirIno
|
||||
})
|
||||
if (existingLibraryItem) {
|
||||
Logger.debug(`[Scanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
|
||||
// Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData
|
||||
existingLibraryItem.path = fullPath
|
||||
existingLibraryItem.relPath = itemDir
|
||||
}
|
||||
}
|
||||
if (existingLibraryItem) {
|
||||
// Is the item exactly - check if was deleted
|
||||
if (existingLibraryItem.path === fullPath) {
|
||||
const exists = await fs.pathExists(fullPath)
|
||||
if (!exists) {
|
||||
Logger.info(`[Scanner] Scanning file update group and library item was deleted "${existingLibraryItem.media.metadata.title}" - marking as missing`)
|
||||
existingLibraryItem.setMissing()
|
||||
await Database.updateLibraryItem(existingLibraryItem)
|
||||
SocketAuthority.emitter('item_updated', existingLibraryItem.toJSONExpanded())
|
||||
|
||||
itemGroupingResults[itemDir] = ScanResult.REMOVED
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Scan library item for updates
|
||||
Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" is in library item "${existingLibraryItem.media.metadata.title}" - scan for updates`)
|
||||
itemGroupingResults[itemDir] = await this.scanLibraryItem(library, folder, existingLibraryItem)
|
||||
continue
|
||||
} else if (library.settings.audiobooksOnly && !fileUpdateGroup[itemDir].some?.(checkFilepathIsAudioFile)) {
|
||||
Logger.debug(`[Scanner] Folder update for relative path "${itemDir}" has no audio files`)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if a library item is a subdirectory of this dir
|
||||
const childItem = await Database.libraryItemModel.findOne({
|
||||
attributes: ['id', 'path'],
|
||||
where: {
|
||||
path: {
|
||||
[Sequelize.Op.startsWith]: fullPath + '/'
|
||||
}
|
||||
}
|
||||
})
|
||||
if (childItem) {
|
||||
Logger.warn(`[Scanner] Files were modified in a parent directory of a library item "${childItem.path}" - ignoring`)
|
||||
itemGroupingResults[itemDir] = ScanResult.NOTHING
|
||||
continue
|
||||
}
|
||||
|
||||
Logger.debug(`[Scanner] Folder update group must be a new item "${itemDir}" in library "${library.name}"`)
|
||||
var isSingleMediaItem = itemDir === fileUpdateGroup[itemDir]
|
||||
var newLibraryItem = await this.scanPotentialNewLibraryItem(library, folder, fullPath, isSingleMediaItem)
|
||||
if (newLibraryItem) {
|
||||
await this.createNewAuthorsAndSeries(newLibraryItem)
|
||||
await Database.createLibraryItem(newLibraryItem)
|
||||
SocketAuthority.emitter('item_added', newLibraryItem.toJSONExpanded())
|
||||
}
|
||||
itemGroupingResults[itemDir] = newLibraryItem ? ScanResult.ADDED : ScanResult.NOTHING
|
||||
}
|
||||
|
||||
return itemGroupingResults
|
||||
}
|
||||
|
||||
async scanPotentialNewLibraryItem(library, folder, fullPath, isSingleMediaItem = false) {
|
||||
const libraryItemData = await getLibraryItemFileData(library.mediaType, folder, fullPath, isSingleMediaItem)
|
||||
if (!libraryItemData) return null
|
||||
return this.scanNewLibraryItem(libraryItemData, library)
|
||||
}
|
||||
|
||||
async searchForCover(libraryItem, libraryScan = null) {
|
||||
const options = {
|
||||
titleDistance: 2,
|
||||
@ -1032,11 +322,6 @@ class Scanner {
|
||||
return
|
||||
}
|
||||
|
||||
if (this.isLibraryScanning(library.id)) {
|
||||
Logger.error(`[Scanner] matchLibraryItems: Already scanning ${library.id}`)
|
||||
return
|
||||
}
|
||||
|
||||
const itemsInLibrary = Database.libraryItems.filter(li => li.libraryId === library.id)
|
||||
if (!itemsInLibrary.length) {
|
||||
Logger.error(`[Scanner] matchLibraryItems: Library has no items ${library.id}`)
|
||||
|
||||
@ -303,7 +303,7 @@ module.exports = {
|
||||
* @returns {{podcast:object[], tags:object[]}}
|
||||
*/
|
||||
async search(oldUser, oldLibrary, query, limit, offset) {
|
||||
const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user)
|
||||
const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser)
|
||||
// Search title, author, itunesId, itunesArtistId
|
||||
const podcasts = await Database.podcastModel.findAll({
|
||||
where: [
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
const Path = require('path')
|
||||
const fs = require('../libs/fsExtra')
|
||||
const Logger = require('../Logger')
|
||||
const { recurseFiles, getFileTimestampsWithIno, filePathToPOSIX } = require('./fileUtils')
|
||||
const { filePathToPOSIX } = require('./fileUtils')
|
||||
const globals = require('./globals')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
|
||||
@ -198,73 +196,6 @@ function buildLibraryFile(libraryItemPath, files) {
|
||||
}
|
||||
module.exports.buildLibraryFile = buildLibraryFile
|
||||
|
||||
// Scan folder
|
||||
async function scanFolder(library, folder) {
|
||||
const folderPath = filePathToPOSIX(folder.fullPath)
|
||||
|
||||
const pathExists = await fs.pathExists(folderPath)
|
||||
if (!pathExists) {
|
||||
Logger.error(`[scandir] Invalid folder path does not exist "${folderPath}"`)
|
||||
return []
|
||||
}
|
||||
|
||||
const fileItems = await recurseFiles(folderPath)
|
||||
const libraryItemGrouping = groupFileItemsIntoLibraryItemDirs(library.mediaType, fileItems, library.settings.audiobooksOnly)
|
||||
|
||||
if (!Object.keys(libraryItemGrouping).length) {
|
||||
Logger.error(`Root path has no media folders: ${folderPath}`)
|
||||
return []
|
||||
}
|
||||
|
||||
const items = []
|
||||
for (const libraryItemPath in libraryItemGrouping) {
|
||||
let isFile = false // item is not in a folder
|
||||
let libraryItemData = null
|
||||
let fileObjs = []
|
||||
if (library.mediaType === 'music') {
|
||||
libraryItemData = {
|
||||
path: Path.posix.join(folderPath, libraryItemPath),
|
||||
relPath: libraryItemPath
|
||||
}
|
||||
fileObjs = await buildLibraryFile(folderPath, [libraryItemPath])
|
||||
isFile = true
|
||||
} else if (libraryItemPath === libraryItemGrouping[libraryItemPath]) {
|
||||
// Media file in root only get title
|
||||
libraryItemData = {
|
||||
mediaMetadata: {
|
||||
title: Path.basename(libraryItemPath, Path.extname(libraryItemPath))
|
||||
},
|
||||
path: Path.posix.join(folderPath, libraryItemPath),
|
||||
relPath: libraryItemPath
|
||||
}
|
||||
fileObjs = await buildLibraryFile(folderPath, [libraryItemPath])
|
||||
isFile = true
|
||||
} else {
|
||||
libraryItemData = getDataFromMediaDir(library.mediaType, folderPath, libraryItemPath)
|
||||
fileObjs = await buildLibraryFile(libraryItemData.path, libraryItemGrouping[libraryItemPath])
|
||||
}
|
||||
|
||||
const libraryItemFolderStats = await getFileTimestampsWithIno(libraryItemData.path)
|
||||
items.push({
|
||||
folderId: folder.id,
|
||||
libraryId: folder.libraryId,
|
||||
ino: libraryItemFolderStats.ino,
|
||||
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
|
||||
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
|
||||
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
|
||||
path: libraryItemData.path,
|
||||
relPath: libraryItemData.relPath,
|
||||
isFile,
|
||||
media: {
|
||||
metadata: libraryItemData.mediaMetadata || null
|
||||
},
|
||||
libraryFiles: fileObjs
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
module.exports.scanFolder = scanFolder
|
||||
|
||||
// Input relative filepath, output all details that can be parsed
|
||||
function getBookDataFromDir(folderPath, relPath, parseSubtitle = false) {
|
||||
relPath = filePathToPOSIX(relPath)
|
||||
@ -381,60 +312,3 @@ function getDataFromMediaDir(libraryMediaType, folderPath, relPath) {
|
||||
}
|
||||
}
|
||||
module.exports.getDataFromMediaDir = getDataFromMediaDir
|
||||
|
||||
// Called from Scanner.js
|
||||
async function getLibraryItemFileData(libraryMediaType, folder, libraryItemPath, isSingleMediaItem) {
|
||||
libraryItemPath = filePathToPOSIX(libraryItemPath)
|
||||
const folderFullPath = filePathToPOSIX(folder.fullPath)
|
||||
|
||||
const libraryItemDir = libraryItemPath.replace(folderFullPath, '').slice(1)
|
||||
let libraryItemData = {}
|
||||
|
||||
let fileItems = []
|
||||
|
||||
if (isSingleMediaItem) { // Single media item in root of folder
|
||||
fileItems = [
|
||||
{
|
||||
fullpath: libraryItemPath,
|
||||
path: libraryItemDir // actually the relPath (only filename here)
|
||||
}
|
||||
]
|
||||
libraryItemData = {
|
||||
path: libraryItemPath, // full path
|
||||
relPath: libraryItemDir, // only filename
|
||||
mediaMetadata: {
|
||||
title: Path.basename(libraryItemDir, Path.extname(libraryItemDir))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fileItems = await recurseFiles(libraryItemPath)
|
||||
libraryItemData = getDataFromMediaDir(libraryMediaType, folderFullPath, libraryItemDir)
|
||||
}
|
||||
|
||||
const libraryItemDirStats = await getFileTimestampsWithIno(libraryItemData.path)
|
||||
const libraryItem = {
|
||||
ino: libraryItemDirStats.ino,
|
||||
mtimeMs: libraryItemDirStats.mtimeMs || 0,
|
||||
ctimeMs: libraryItemDirStats.ctimeMs || 0,
|
||||
birthtimeMs: libraryItemDirStats.birthtimeMs || 0,
|
||||
folderId: folder.id,
|
||||
libraryId: folder.libraryId,
|
||||
path: libraryItemData.path,
|
||||
relPath: libraryItemData.relPath,
|
||||
isFile: isSingleMediaItem,
|
||||
media: {
|
||||
metadata: libraryItemData.mediaMetadata || null
|
||||
},
|
||||
libraryFiles: []
|
||||
}
|
||||
|
||||
for (let i = 0; i < fileItems.length; i++) {
|
||||
const fileItem = fileItems[i]
|
||||
const newLibraryFile = new LibraryFile()
|
||||
// fileItem.path is the relative path
|
||||
await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path)
|
||||
libraryItem.libraryFiles.push(newLibraryFile)
|
||||
}
|
||||
return libraryItem
|
||||
}
|
||||
module.exports.getLibraryItemFileData = getLibraryItemFileData
|
||||
|
||||
Loading…
Reference in New Issue
Block a user