Add new library item scanner

This commit is contained in:
advplyr 2023-09-03 17:51:58 -05:00
parent e63aab95d8
commit 42ff3d8314
11 changed files with 421 additions and 154 deletions

View File

@ -45,6 +45,11 @@ class Database {
return this.models.library
}
/** @type {typeof import('./models/LibraryFolder')} */
get libraryFolderModel() {
return this.models.libraryFolder
}
/** @type {typeof import('./models/Author')} */
get authorModel() {
return this.models.author

View File

@ -16,7 +16,6 @@ const Logger = require('./Logger')
const Auth = require('./Auth')
const Watcher = require('./Watcher')
const Scanner = require('./scanner/Scanner')
const LibraryScanner = require('./scanner/LibraryScanner')
const Database = require('./Database')
const SocketAuthority = require('./SocketAuthority')
@ -77,7 +76,6 @@ class Server {
this.rssFeedManager = new RssFeedManager()
this.scanner = new Scanner(this.coverManager, this.taskManager)
this.libraryScanner = new LibraryScanner(this.coverManager, this.taskManager)
this.cronManager = new CronManager(this.scanner, this.podcastManager)
// Routers

View File

@ -9,11 +9,13 @@ const libraryHelpers = require('../utils/libraryHelpers')
const libraryItemsBookFilters = require('../utils/queries/libraryItemsBookFilters')
const libraryItemFilters = require('../utils/queries/libraryItemFilters')
const seriesFilters = require('../utils/queries/seriesFilters')
const fileUtils = require('../utils/fileUtils')
const { sort, createNewSortInstance } = require('../libs/fastSort')
const naturalSort = createNewSortInstance({
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
})
const LibraryScanner = require('../scanner/LibraryScanner')
const Database = require('../Database')
const libraryFilters = require('../utils/queries/libraryFilters')
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
@ -33,7 +35,7 @@ class LibraryController {
// Validate folder paths exist or can be created & resolve rel paths
// returns 400 if a folder fails to access
newLibraryPayload.folders = newLibraryPayload.folders.map(f => {
f.fullPath = Path.resolve(f.fullPath)
f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath))
return f
})
for (const folder of newLibraryPayload.folders) {
@ -87,7 +89,7 @@ class LibraryController {
async findOne(req, res) {
const includeArray = (req.query.include || '').split(',')
if (includeArray.includes('filterdata')) {
const filterdata = await libraryFilters.getFilterData(req.library)
const filterdata = await libraryFilters.getFilterData(req.library.mediaType, req.library.id)
return res.json({
filterdata,
@ -119,7 +121,7 @@ class LibraryController {
const newFolderPaths = []
req.body.folders = req.body.folders.map(f => {
if (!f.id) {
f.fullPath = Path.resolve(f.fullPath)
f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath))
newFolderPaths.push(f.fullPath)
}
return f
@ -670,7 +672,7 @@ class LibraryController {
* @param {import('express').Response} res
*/
async getLibraryFilterData(req, res) {
const filterData = await libraryFilters.getFilterData(req.library)
const filterData = await libraryFilters.getFilterData(req.library.mediaType, req.library.id)
res.json(filterData)
}
@ -976,9 +978,13 @@ class LibraryController {
forceRescan: req.query.force == 1
}
res.sendStatus(200)
await this.scanner.scan(req.library, options)
// TODO: New library scanner
// await this.libraryScanner.scan(req.library, options)
if (req.library.mediaType === 'podcast') {
// TODO: New library scanner for podcasts
await this.scanner.scan(req.library, options)
} else {
await LibraryScanner.scan(req.library, options)
}
await Database.resetLibraryIssuesFilterData(req.library.id)
Logger.info('[LibraryController] Scan complete')
}

View File

@ -8,6 +8,7 @@ const zipHelpers = require('../utils/zipHelpers')
const { reqSupportsWebp } = require('../utils/index')
const { ScanResult } = require('../utils/constants')
const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils')
const LibraryItemScanner = require('../scanner/LibraryItemScanner')
class LibraryItemController {
constructor() { }
@ -475,7 +476,14 @@ class LibraryItemController {
return res.sendStatus(500)
}
const result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
let result = 0
if (req.libraryItem.isPodcast) {
// TODO: New library item scanner for podcast
result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
} else {
result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
}
await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId)
res.json({
result: Object.keys(ScanResult).find(key => ScanResult[key] == result)

View File

@ -2,6 +2,16 @@ const { DataTypes, Model } = require('sequelize')
const Logger = require('../Logger')
const oldLibrary = require('../objects/Library')
/**
* @typedef LibrarySettingsObject
* @property {number} coverAspectRatio BookCoverAspectRatio
* @property {boolean} disableWatcher
* @property {boolean} skipMatchingMediaWithAsin
* @property {boolean} skipMatchingMediaWithIsbn
* @property {string} autoScanCronExpression
* @property {boolean} audiobooksOnly
* @property {boolean} hideSingleBookSeries Do not show series that only have 1 book
*/
class Library extends Model {
constructor(values, options) {
@ -23,7 +33,7 @@ class Library extends Model {
this.lastScan
/** @type {string} */
this.lastScanVersion
/** @type {Object} */
/** @type {LibrarySettingsObject} */
this.settings
/** @type {Object} */
this.extraData

View File

@ -41,7 +41,6 @@ class ApiRouter {
constructor(Server) {
this.auth = Server.auth
this.scanner = Server.scanner
this.libraryScanner = Server.libraryScanner
this.playbackSessionManager = Server.playbackSessionManager
this.abMergeManager = Server.abMergeManager
this.backupManager = Server.backupManager

View File

@ -1,6 +1,6 @@
const uuidv4 = require("uuid").v4
const Path = require('path')
const { Sequelize } = require('sequelize')
const sequelize = require('sequelize')
const { LogLevel } = require('../utils/constants')
const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
@ -14,6 +14,7 @@ const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../
const AudioFile = require('../objects/files/AudioFile')
const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority')
const fsExtra = require("../libs/fsExtra")
/**
@ -45,10 +46,11 @@ class BookScanner {
/**
* @param {import('../models/LibraryItem')} existingLibraryItem
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {import('./LibraryScan')} libraryScan
* @returns {import('../models/LibraryItem')}
* @returns {Promise<import('../models/LibraryItem')>}
*/
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
/** @type {import('../models/Book')} */
const media = await existingLibraryItem.getMedia({
include: [
@ -146,13 +148,13 @@ class BookScanner {
}
// Check if ebook was removed
if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
if (media.ebookFile && (librarySettings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
media.ebookFile = null
hasMediaChanges = true
}
// Check if ebook is not set and ebooks were found
if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
if (!media.ebookFile && !librarySettings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
// Prefer to use an epub ebook then fallback to the first ebook found
let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
@ -179,7 +181,7 @@ class BookScanner {
// Check for authors added
for (const authorName of bookMetadata.authors) {
if (!media.authors.some(au => au.name === authorName)) {
const existingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName)
const existingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName)
if (existingAuthor) {
await Database.bookAuthorModel.create({
bookId: media.id,
@ -191,10 +193,10 @@ class BookScanner {
const newAuthor = await Database.authorModel.create({
name: authorName,
lastFirst: parseNameString.nameToLastFirst(authorName),
libraryId: libraryScan.libraryId
libraryId: libraryItemData.libraryId
})
await media.addAuthor(newAuthor)
Database.addAuthorToFilterData(libraryScan.libraryId, newAuthor.name, newAuthor.id)
Database.addAuthorToFilterData(libraryItemData.libraryId, newAuthor.name, newAuthor.id)
libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`)
authorsUpdated = true
}
@ -213,7 +215,7 @@ class BookScanner {
// Check for series added
for (const seriesObj of bookMetadata.series) {
if (!media.series.some(se => se.name === seriesObj.name)) {
const existingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name)
const existingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name)
if (existingSeries) {
await Database.bookSeriesModel.create({
bookId: media.id,
@ -226,10 +228,10 @@ class BookScanner {
const newSeries = await Database.seriesModel.create({
name: seriesObj.name,
nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name),
libraryId: libraryScan.libraryId
libraryId: libraryItemData.libraryId
})
await media.addSeries(newSeries)
Database.addSeriesToFilterData(libraryScan.libraryId, newSeries.name, newSeries.id)
await media.addSeries(newSeries, { through: { sequence: seriesObj.sequence } })
Database.addSeriesToFilterData(libraryItemData.libraryId, newSeries.name, newSeries.id)
libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`)
seriesUpdated = true
}
@ -304,7 +306,7 @@ class BookScanner {
media.authors = await media.getAuthors({
joinTableAttributes: ['createdAt'],
order: [
Sequelize.literal(`bookAuthor.createdAt ASC`)
sequelize.literal(`bookAuthor.createdAt ASC`)
]
})
}
@ -312,7 +314,7 @@ class BookScanner {
media.series = await media.getSeries({
joinTableAttributes: ['sequence', 'createdAt'],
order: [
Sequelize.literal(`bookSeries.createdAt ASC`)
sequelize.literal(`bookSeries.createdAt ASC`)
]
})
}
@ -356,16 +358,17 @@ class BookScanner {
/**
*
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {import('./LibraryScan')} libraryScan
* @returns {import('../models/LibraryItem')}
*/
async scanNewBookLibraryItem(libraryItemData, libraryScan) {
async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) {
// Scan audio files found
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryScan.libraryMediaType, libraryItemData, libraryItemData.audioLibraryFiles)
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryItemData.mediaType, libraryItemData, libraryItemData.audioLibraryFiles)
scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles)
// Find ebook file (prefer epub)
let ebookLibraryFile = libraryScan.library.settings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
let ebookLibraryFile = librarySettings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
// Do not add library items that have no valid audio files and no ebook file
if (!ebookLibraryFile && !scannedAudioFiles.length) {
@ -393,7 +396,7 @@ class BookScanner {
}
if (bookMetadata.authors.length) {
for (const authorName of bookMetadata.authors) {
const matchingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName)
const matchingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName)
if (matchingAuthor) {
bookObject.bookAuthors.push({
authorId: matchingAuthor.id
@ -402,7 +405,7 @@ class BookScanner {
// New author
bookObject.bookAuthors.push({
author: {
libraryId: libraryScan.libraryId,
libraryId: libraryItemData.libraryId,
name: authorName,
lastFirst: parseNameString.nameToLastFirst(authorName)
}
@ -413,7 +416,7 @@ class BookScanner {
if (bookMetadata.series.length) {
for (const seriesObj of bookMetadata.series) {
if (!seriesObj.name) continue
const matchingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name)
const matchingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name)
if (matchingSeries) {
bookObject.bookSeries.push({
seriesId: matchingSeries.id,
@ -425,7 +428,7 @@ class BookScanner {
series: {
name: seriesObj.name,
nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name),
libraryId: libraryScan.libraryId
libraryId: libraryItemData.libraryId
}
})
}
@ -476,22 +479,22 @@ class BookScanner {
if (libraryItem.book.bookSeries?.length) {
for (const bs of libraryItem.book.bookSeries) {
if (bs.series) {
Database.addSeriesToFilterData(libraryScan.libraryId, bs.series.name, bs.series.id)
Database.addSeriesToFilterData(libraryItemData.libraryId, bs.series.name, bs.series.id)
}
}
}
if (libraryItem.book.bookAuthors?.length) {
for (const ba of libraryItem.book.bookAuthors) {
if (ba.author) {
Database.addAuthorToFilterData(libraryScan.libraryId, ba.author.name, ba.author.id)
Database.addAuthorToFilterData(libraryItemData.libraryId, ba.author.name, ba.author.id)
}
}
}
Database.addNarratorsToFilterData(libraryScan.libraryId, libraryItem.book.narrators)
Database.addGenresToFilterData(libraryScan.libraryId, libraryItem.book.genres)
Database.addTagsToFilterData(libraryScan.libraryId, libraryItem.book.tags)
Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher)
Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language)
Database.addNarratorsToFilterData(libraryItemData.libraryId, libraryItem.book.narrators)
Database.addGenresToFilterData(libraryItemData.libraryId, libraryItem.book.genres)
Database.addTagsToFilterData(libraryItemData.libraryId, libraryItem.book.tags)
Database.addPublisherToFilterData(libraryItemData.libraryId, libraryItem.book.publisher)
Database.addLanguageToFilterData(libraryItemData.libraryId, libraryItem.book.language)
// Load for emitting to client
libraryItem.media = await libraryItem.getMedia({
@ -949,5 +952,78 @@ class BookScanner {
})
}
}
/**
* Check authors that were removed from a book and remove them if they no longer have any books
* keep authors without books that have a asin, description or imagePath
* @param {string} libraryId
* @param {import('./ScanLogger')} scanLogger
* @returns {Promise}
*/
async checkAuthorsRemovedFromBooks(libraryId, scanLogger) {
const bookAuthorsToRemove = (await Database.authorModel.findAll({
where: [
{
id: scanLogger.authorsRemovedFromBooks,
asin: {
[sequelize.Op.or]: [null, ""]
},
description: {
[sequelize.Op.or]: [null, ""]
},
imagePath: {
[sequelize.Op.or]: [null, ""]
}
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
],
attributes: ['id'],
raw: true
})).map(au => au.id)
if (bookAuthorsToRemove.length) {
await Database.authorModel.destroy({
where: {
id: bookAuthorsToRemove
}
})
bookAuthorsToRemove.forEach((authorId) => {
Database.removeAuthorFromFilterData(libraryId, authorId)
// TODO: Clients were expecting full author in payload but its unnecessary
SocketAuthority.emitter('author_removed', { id: authorId, libraryId })
})
scanLogger.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
}
}
/**
* Check series that were removed from books and remove them if they no longer have any books
* @param {string} libraryId
* @param {import('./ScanLogger')} scanLogger
* @returns {Promise}
*/
async checkSeriesRemovedFromBooks(libraryId, scanLogger) {
const bookSeriesToRemove = (await Database.seriesModel.findAll({
where: [
{
id: scanLogger.seriesRemovedFromBooks
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0)
],
attributes: ['id'],
raw: true
})).map(se => se.id)
if (bookSeriesToRemove.length) {
await Database.seriesModel.destroy({
where: {
id: bookSeriesToRemove
}
})
bookSeriesToRemove.forEach((seriesId) => {
Database.removeSeriesFromFilterData(libraryId, seriesId)
SocketAuthority.emitter('series_removed', { id: seriesId, libraryId })
})
scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
}
}
}
module.exports = new BookScanner()

View File

@ -0,0 +1,190 @@
const Path = require('path')
const { LogLevel, ScanResult } = require('../utils/constants')
const fileUtils = require('../utils/fileUtils')
const scanUtils = require('../utils/scandir')
const libraryFilters = require('../utils/queries/libraryFilters')
const Database = require('../Database')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
const BookScanner = require('./BookScanner')
const ScanLogger = require('./ScanLogger')
const LibraryItem = require('../models/LibraryItem')
const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority')
class LibraryItemScanner {
constructor() { }
/**
* Scan single library item
*
* @param {string} libraryItemId
* @returns {number} ScanResult
*/
async scanLibraryItem(libraryItemId) {
// TODO: Add task manager
const libraryItem = await Database.libraryItemModel.findByPk(libraryItemId)
if (!libraryItem) {
Logger.error(`[LibraryItemScanner] Library item not found "${libraryItemId}"`)
return ScanResult.NOTHING
}
const library = await Database.libraryModel.findByPk(libraryItem.libraryId, {
include: {
model: Database.libraryFolderModel,
where: {
id: libraryItem.libraryFolderId
}
}
})
if (!library) {
Logger.error(`[LibraryItemScanner] Library "${libraryItem.libraryId}" not found for library item "${libraryItem.id}"`)
return ScanResult.NOTHING
}
// Make sure library filter data is set
// this is used to check for existing authors & series
await libraryFilters.getFilterData(library.mediaType, library.id)
const scanLogger = new ScanLogger()
scanLogger.verbose = true
scanLogger.setData('libraryItem', libraryItemId)
const libraryItemPath = fileUtils.filePathToPOSIX(libraryItem.path)
const folder = library.libraryFolders[0]
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false)
if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) {
if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) {
const expandedLibraryItem = await this.rescanLibraryItem(libraryItem, libraryItemScanData, library.settings, scanLogger)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger)
} else {
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(libraryItem.id)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
}
return ScanResult.UPDATED
}
return ScanResult.UPTODATE
}
/**
* Remove empty authors and series
* @param {string} libraryId
* @param {ScanLogger} scanLogger
* @returns {Promise}
*/
async checkAuthorsAndSeriesRemovedFromBooks(libraryId, scanLogger) {
if (scanLogger.authorsRemovedFromBooks.length) {
await BookScanner.checkAuthorsRemovedFromBooks(libraryId, scanLogger)
}
if (scanLogger.seriesRemovedFromBooks.length) {
await BookScanner.checkSeriesRemovedFromBooks(libraryId, scanLogger)
}
}
/**
*
* @param {string} libraryItemPath
* @param {import('../models/Library')} library
* @param {import('../models/LibraryFolder')} folder
* @param {boolean} isSingleMediaItem
* @returns {Promise<LibraryItemScanData>}
*/
async getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem) {
const libraryFolderPath = fileUtils.filePathToPOSIX(folder.path)
const libraryItemDir = libraryItemPath.replace(libraryFolderPath, '').slice(1)
let libraryItemData = {}
let fileItems = []
if (isSingleMediaItem) { // Single media item in root of folder
fileItems = [
{
fullpath: libraryItemPath,
path: libraryItemDir // actually the relPath (only filename here)
}
]
libraryItemData = {
path: libraryItemPath, // full path
relPath: libraryItemDir, // only filename
mediaMetadata: {
title: Path.basename(libraryItemDir, Path.extname(libraryItemDir))
}
}
} else {
fileItems = await fileUtils.recurseFiles(libraryItemPath)
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, libraryFolderPath, libraryItemDir)
}
const libraryFiles = []
for (let i = 0; i < fileItems.length; i++) {
const fileItem = fileItems[i]
const newLibraryFile = new LibraryFile()
// fileItem.path is the relative path
await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path)
libraryFiles.push(newLibraryFile)
}
const libraryItemStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
return new LibraryItemScanData({
libraryFolderId: folder.id,
libraryId: library.id,
mediaType: library.mediaType,
ino: libraryItemStats.ino,
mtimeMs: libraryItemStats.mtimeMs || 0,
ctimeMs: libraryItemStats.ctimeMs || 0,
birthtimeMs: libraryItemStats.birthtimeMs || 0,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
isFile: isSingleMediaItem,
mediaMetadata: libraryItemData.mediaMetadata || null,
libraryFiles
})
}
/**
*
* @param {import('../models/LibraryItem')} existingLibraryItem
* @param {LibraryItemScanData} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<LibraryItem>}
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
if (existingLibraryItem.mediaType === 'book') {
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
return libraryItem
} else {
// TODO: Scan updated podcast
return null
}
}
/**
*
* @param {LibraryItemScanData} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<LibraryItem>}
*/
async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) {
if (libraryScan.libraryMediaType === 'book') {
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
if (newLibraryItem) {
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
}
return newLibraryItem
} else {
// TODO: Scan new podcast
return null
}
}
}
module.exports = new LibraryItemScanner()

View File

@ -7,18 +7,15 @@ const Database = require('../Database')
const fs = require('../libs/fsExtra')
const fileUtils = require('../utils/fileUtils')
const scanUtils = require('../utils/scandir')
const { ScanResult, LogLevel } = require('../utils/constants')
const { LogLevel } = require('../utils/constants')
const libraryFilters = require('../utils/queries/libraryFilters')
const LibraryItemScanner = require('./LibraryItemScanner')
const ScanOptions = require('./ScanOptions')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
const BookScanner = require('./BookScanner')
class LibraryScanner {
constructor(coverManager, taskManager) {
this.coverManager = coverManager
this.taskManager = taskManager
constructor() {
this.cancelLibraryScan = {}
this.librariesScanning = []
}
@ -93,7 +90,7 @@ class LibraryScanner {
async scanLibrary(libraryScan) {
// Make sure library filter data is set
// this is used to check for existing authors & series
await libraryFilters.getFilterData(libraryScan.library)
await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId)
/** @type {LibraryItemScanData[]} */
let libraryItemDataFound = []
@ -151,7 +148,7 @@ class LibraryScanner {
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
libraryScan.resultsUpdated++
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
oldLibraryItemsUpdated.push(oldLibraryItem)
} else {
@ -177,68 +174,8 @@ class LibraryScanner {
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
}
// Check authors that were removed from a book and remove them if they no longer have any books
// keep authors without books that have a asin, description or imagePath
if (libraryScan.authorsRemovedFromBooks.length) {
const bookAuthorsToRemove = (await Database.authorModel.findAll({
where: [
{
id: libraryScan.authorsRemovedFromBooks,
asin: {
[sequelize.Op.or]: [null, ""]
},
description: {
[sequelize.Op.or]: [null, ""]
},
imagePath: {
[sequelize.Op.or]: [null, ""]
}
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
],
attributes: ['id'],
raw: true
})).map(au => au.id)
if (bookAuthorsToRemove.length) {
await Database.authorModel.destroy({
where: {
id: bookAuthorsToRemove
}
})
bookAuthorsToRemove.forEach((authorId) => {
Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId)
// TODO: Clients were expecting full author in payload but its unnecessary
SocketAuthority.emitter('author_removed', { id: authorId, libraryId: libraryScan.libraryId })
})
libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
}
}
// Check series that were removed from books and remove them if they no longer have any books
if (libraryScan.seriesRemovedFromBooks.length) {
const bookSeriesToRemove = (await Database.seriesModel.findAll({
where: [
{
id: libraryScan.seriesRemovedFromBooks
},
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0)
],
attributes: ['id'],
raw: true
})).map(se => se.id)
if (bookSeriesToRemove.length) {
await Database.seriesModel.destroy({
where: {
id: bookSeriesToRemove
}
})
bookSeriesToRemove.forEach((seriesId) => {
Database.removeSeriesFromFilterData(libraryScan.libraryId, seriesId)
SocketAuthority.emitter('series_removed', { id: seriesId, libraryId: libraryScan.libraryId })
})
libraryScan.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
}
}
// Authors and series that were removed from books should be removed if they are now empty
await LibraryItemScanner.checkAuthorsAndSeriesRemovedFromBooks(libraryScan.libraryId, libraryScan)
// Update missing library items
if (libraryItemIdsMissing.length) {
@ -260,7 +197,7 @@ class LibraryScanner {
if (libraryItemDataFound.length) {
let newOldLibraryItems = []
for (const libraryItemData of libraryItemDataFound) {
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
const newLibraryItem = await LibraryItemScanner.scanNewLibraryItem(libraryItemData, libraryScan.library.settings, libraryScan)
if (newLibraryItem) {
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
newOldLibraryItems.push(oldLibraryItem)
@ -353,38 +290,5 @@ class LibraryScanner {
}
return items
}
/**
*
* @param {import('../models/LibraryItem')} existingLibraryItem
* @param {LibraryItemScanData} libraryItemData
* @param {LibraryScan} libraryScan
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
if (existingLibraryItem.mediaType === 'book') {
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
return libraryItem
} else {
// TODO: Scan updated podcast
}
}
/**
*
* @param {LibraryItemScanData} libraryItemData
* @param {LibraryScan} libraryScan
*/
async scanNewLibraryItem(libraryItemData, libraryScan) {
if (libraryScan.libraryMediaType === 'book') {
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
if (newLibraryItem) {
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
}
return newLibraryItem
} else {
// TODO: Scan new podcast
return null
}
}
}
module.exports = LibraryScanner
module.exports = new LibraryScanner()

View File

@ -0,0 +1,70 @@
const uuidv4 = require("uuid").v4
const Logger = require('../Logger')
const { LogLevel } = require('../utils/constants')
class ScanLogger {
constructor() {
this.id = null
this.type = null
this.name = null
this.verbose = false
this.startedAt = null
this.finishedAt = null
this.elapsed = null
/** @type {string[]} */
this.authorsRemovedFromBooks = []
/** @type {string[]} */
this.seriesRemovedFromBooks = []
this.logs = []
}
toJSON() {
return {
id: this.id,
type: this.type,
name: this.name,
startedAt: this.startedAt,
finishedAt: this.finishedAt,
elapsed: this.elapsed
}
}
setData(type, name) {
this.id = uuidv4()
this.type = type
this.name = name
this.startedAt = Date.now()
}
setComplete() {
this.finishedAt = Date.now()
this.elapsed = this.finishedAt - this.startedAt
}
getLogLevelString(level) {
for (const key in LogLevel) {
if (LogLevel[key] === level) {
return key
}
}
return 'UNKNOWN'
}
addLog(level, ...args) {
const logObj = {
timestamp: (new Date()).toISOString(),
message: args.join(' '),
levelName: this.getLogLevelString(level),
level
}
if (this.verbose) {
Logger.debug(`[Scan] "${this.name}":`, ...args)
}
this.logs.push(logObj)
}
}
module.exports = ScanLogger

View File

@ -390,11 +390,12 @@ module.exports = {
/**
* Get filter data used in filter menus
* @param {import('../../objects/Library')} oldLibrary
* @param {string} mediaType
* @param {string} libraryId
* @returns {Promise<object>}
*/
async getFilterData(oldLibrary) {
const cachedFilterData = Database.libraryFilterData[oldLibrary.id]
async getFilterData(mediaType, libraryId) {
const cachedFilterData = Database.libraryFilterData[libraryId]
if (cachedFilterData) {
const cacheElapsed = Date.now() - cachedFilterData.loadedAt
// Cache library filters for 30 mins
@ -416,13 +417,13 @@ module.exports = {
numIssues: 0
}
if (oldLibrary.isPodcast) {
if (mediaType === 'podcast') {
const podcasts = await Database.podcastModel.findAll({
include: {
model: Database.libraryItemModel,
attributes: [],
where: {
libraryId: oldLibrary.id
libraryId: libraryId
}
},
attributes: ['tags', 'genres']
@ -441,7 +442,7 @@ module.exports = {
model: Database.libraryItemModel,
attributes: ['isMissing', 'isInvalid'],
where: {
libraryId: oldLibrary.id
libraryId: libraryId
}
},
attributes: ['tags', 'genres', 'publisher', 'narrators', 'language']
@ -463,7 +464,7 @@ module.exports = {
const series = await Database.seriesModel.findAll({
where: {
libraryId: oldLibrary.id
libraryId: libraryId
},
attributes: ['id', 'name']
})
@ -471,7 +472,7 @@ module.exports = {
const authors = await Database.authorModel.findAll({
where: {
libraryId: oldLibrary.id
libraryId: libraryId
},
attributes: ['id', 'name']
})
@ -486,7 +487,7 @@ module.exports = {
data.publishers = naturalSort([...data.publishers]).asc()
data.languages = naturalSort([...data.languages]).asc()
data.loadedAt = Date.now()
Database.libraryFilterData[oldLibrary.id] = data
Database.libraryFilterData[libraryId] = data
Logger.debug(`Loaded filterdata in ${((Date.now() - start) / 1000).toFixed(2)}s`)
return data