mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-10-05 01:32:08 +02:00
Add new library item scanner
This commit is contained in:
advplyr
parent
e63aab95d8
commit
42ff3d8314
@ -45,6 +45,11 @@ class Database {
|
||||
return this.models.library
|
||||
}
|
||||
|
||||
/** @type {typeof import('./models/LibraryFolder')} */
|
||||
get libraryFolderModel() {
|
||||
return this.models.libraryFolder
|
||||
}
|
||||
|
||||
/** @type {typeof import('./models/Author')} */
|
||||
get authorModel() {
|
||||
return this.models.author
|
||||
|
||||
@ -16,7 +16,6 @@ const Logger = require('./Logger')
|
||||
const Auth = require('./Auth')
|
||||
const Watcher = require('./Watcher')
|
||||
const Scanner = require('./scanner/Scanner')
|
||||
const LibraryScanner = require('./scanner/LibraryScanner')
|
||||
const Database = require('./Database')
|
||||
const SocketAuthority = require('./SocketAuthority')
|
||||
|
||||
@ -77,7 +76,6 @@ class Server {
|
||||
this.rssFeedManager = new RssFeedManager()
|
||||
|
||||
this.scanner = new Scanner(this.coverManager, this.taskManager)
|
||||
this.libraryScanner = new LibraryScanner(this.coverManager, this.taskManager)
|
||||
this.cronManager = new CronManager(this.scanner, this.podcastManager)
|
||||
|
||||
// Routers
|
||||
|
||||
@ -9,11 +9,13 @@ const libraryHelpers = require('../utils/libraryHelpers')
|
||||
const libraryItemsBookFilters = require('../utils/queries/libraryItemsBookFilters')
|
||||
const libraryItemFilters = require('../utils/queries/libraryItemFilters')
|
||||
const seriesFilters = require('../utils/queries/seriesFilters')
|
||||
const fileUtils = require('../utils/fileUtils')
|
||||
const { sort, createNewSortInstance } = require('../libs/fastSort')
|
||||
const naturalSort = createNewSortInstance({
|
||||
comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
|
||||
})
|
||||
|
||||
const LibraryScanner = require('../scanner/LibraryScanner')
|
||||
const Database = require('../Database')
|
||||
const libraryFilters = require('../utils/queries/libraryFilters')
|
||||
const libraryItemsPodcastFilters = require('../utils/queries/libraryItemsPodcastFilters')
|
||||
@ -33,7 +35,7 @@ class LibraryController {
|
||||
// Validate folder paths exist or can be created & resolve rel paths
|
||||
// returns 400 if a folder fails to access
|
||||
newLibraryPayload.folders = newLibraryPayload.folders.map(f => {
|
||||
f.fullPath = Path.resolve(f.fullPath)
|
||||
f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath))
|
||||
return f
|
||||
})
|
||||
for (const folder of newLibraryPayload.folders) {
|
||||
@ -87,7 +89,7 @@ class LibraryController {
|
||||
async findOne(req, res) {
|
||||
const includeArray = (req.query.include || '').split(',')
|
||||
if (includeArray.includes('filterdata')) {
|
||||
const filterdata = await libraryFilters.getFilterData(req.library)
|
||||
const filterdata = await libraryFilters.getFilterData(req.library.mediaType, req.library.id)
|
||||
|
||||
return res.json({
|
||||
filterdata,
|
||||
@ -119,7 +121,7 @@ class LibraryController {
|
||||
const newFolderPaths = []
|
||||
req.body.folders = req.body.folders.map(f => {
|
||||
if (!f.id) {
|
||||
f.fullPath = Path.resolve(f.fullPath)
|
||||
f.fullPath = fileUtils.filePathToPOSIX(Path.resolve(f.fullPath))
|
||||
newFolderPaths.push(f.fullPath)
|
||||
}
|
||||
return f
|
||||
@ -670,7 +672,7 @@ class LibraryController {
|
||||
* @param {import('express').Response} res
|
||||
*/
|
||||
async getLibraryFilterData(req, res) {
|
||||
const filterData = await libraryFilters.getFilterData(req.library)
|
||||
const filterData = await libraryFilters.getFilterData(req.library.mediaType, req.library.id)
|
||||
res.json(filterData)
|
||||
}
|
||||
|
||||
@ -976,9 +978,13 @@ class LibraryController {
|
||||
forceRescan: req.query.force == 1
|
||||
}
|
||||
res.sendStatus(200)
|
||||
await this.scanner.scan(req.library, options)
|
||||
// TODO: New library scanner
|
||||
// await this.libraryScanner.scan(req.library, options)
|
||||
if (req.library.mediaType === 'podcast') {
|
||||
// TODO: New library scanner for podcasts
|
||||
await this.scanner.scan(req.library, options)
|
||||
} else {
|
||||
await LibraryScanner.scan(req.library, options)
|
||||
}
|
||||
|
||||
await Database.resetLibraryIssuesFilterData(req.library.id)
|
||||
Logger.info('[LibraryController] Scan complete')
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ const zipHelpers = require('../utils/zipHelpers')
|
||||
const { reqSupportsWebp } = require('../utils/index')
|
||||
const { ScanResult } = require('../utils/constants')
|
||||
const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils')
|
||||
const LibraryItemScanner = require('../scanner/LibraryItemScanner')
|
||||
|
||||
class LibraryItemController {
|
||||
constructor() { }
|
||||
@ -475,7 +476,14 @@ class LibraryItemController {
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
|
||||
const result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
|
||||
let result = 0
|
||||
if (req.libraryItem.isPodcast) {
|
||||
// TODO: New library item scanner for podcast
|
||||
result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
|
||||
} else {
|
||||
result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
|
||||
}
|
||||
|
||||
await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId)
|
||||
res.json({
|
||||
result: Object.keys(ScanResult).find(key => ScanResult[key] == result)
|
||||
|
||||
@ -2,6 +2,16 @@ const { DataTypes, Model } = require('sequelize')
|
||||
const Logger = require('../Logger')
|
||||
const oldLibrary = require('../objects/Library')
|
||||
|
||||
/**
|
||||
* @typedef LibrarySettingsObject
|
||||
* @property {number} coverAspectRatio BookCoverAspectRatio
|
||||
* @property {boolean} disableWatcher
|
||||
* @property {boolean} skipMatchingMediaWithAsin
|
||||
* @property {boolean} skipMatchingMediaWithIsbn
|
||||
* @property {string} autoScanCronExpression
|
||||
* @property {boolean} audiobooksOnly
|
||||
* @property {boolean} hideSingleBookSeries Do not show series that only have 1 book
|
||||
*/
|
||||
|
||||
class Library extends Model {
|
||||
constructor(values, options) {
|
||||
@ -23,7 +33,7 @@ class Library extends Model {
|
||||
this.lastScan
|
||||
/** @type {string} */
|
||||
this.lastScanVersion
|
||||
/** @type {Object} */
|
||||
/** @type {LibrarySettingsObject} */
|
||||
this.settings
|
||||
/** @type {Object} */
|
||||
this.extraData
|
||||
|
||||
@ -41,7 +41,6 @@ class ApiRouter {
|
||||
constructor(Server) {
|
||||
this.auth = Server.auth
|
||||
this.scanner = Server.scanner
|
||||
this.libraryScanner = Server.libraryScanner
|
||||
this.playbackSessionManager = Server.playbackSessionManager
|
||||
this.abMergeManager = Server.abMergeManager
|
||||
this.backupManager = Server.backupManager
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
const uuidv4 = require("uuid").v4
|
||||
const Path = require('path')
|
||||
const { Sequelize } = require('sequelize')
|
||||
const sequelize = require('sequelize')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
|
||||
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
|
||||
@ -14,6 +14,7 @@ const { readTextFile, filePathToPOSIX, getFileTimestampsWithIno } = require('../
|
||||
const AudioFile = require('../objects/files/AudioFile')
|
||||
const CoverManager = require('../managers/CoverManager')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
const SocketAuthority = require('../SocketAuthority')
|
||||
const fsExtra = require("../libs/fsExtra")
|
||||
|
||||
/**
|
||||
@ -45,10 +46,11 @@ class BookScanner {
|
||||
/**
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {import('../models/LibraryItem')}
|
||||
* @returns {Promise<import('../models/LibraryItem')>}
|
||||
*/
|
||||
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
|
||||
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
|
||||
/** @type {import('../models/Book')} */
|
||||
const media = await existingLibraryItem.getMedia({
|
||||
include: [
|
||||
@ -146,13 +148,13 @@ class BookScanner {
|
||||
}
|
||||
|
||||
// Check if ebook was removed
|
||||
if (media.ebookFile && (libraryScan.library.settings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
|
||||
if (media.ebookFile && (librarySettings.audiobooksOnly || libraryItemData.checkEbookFileRemoved(media.ebookFile))) {
|
||||
media.ebookFile = null
|
||||
hasMediaChanges = true
|
||||
}
|
||||
|
||||
// Check if ebook is not set and ebooks were found
|
||||
if (!media.ebookFile && !libraryScan.library.settings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
|
||||
if (!media.ebookFile && !librarySettings.audiobooksOnly && libraryItemData.ebookLibraryFiles.length) {
|
||||
// Prefer to use an epub ebook then fallback to the first ebook found
|
||||
let ebookLibraryFile = libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub')
|
||||
if (!ebookLibraryFile) ebookLibraryFile = libraryItemData.ebookLibraryFiles[0]
|
||||
@ -179,7 +181,7 @@ class BookScanner {
|
||||
// Check for authors added
|
||||
for (const authorName of bookMetadata.authors) {
|
||||
if (!media.authors.some(au => au.name === authorName)) {
|
||||
const existingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName)
|
||||
const existingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName)
|
||||
if (existingAuthor) {
|
||||
await Database.bookAuthorModel.create({
|
||||
bookId: media.id,
|
||||
@ -191,10 +193,10 @@ class BookScanner {
|
||||
const newAuthor = await Database.authorModel.create({
|
||||
name: authorName,
|
||||
lastFirst: parseNameString.nameToLastFirst(authorName),
|
||||
libraryId: libraryScan.libraryId
|
||||
libraryId: libraryItemData.libraryId
|
||||
})
|
||||
await media.addAuthor(newAuthor)
|
||||
Database.addAuthorToFilterData(libraryScan.libraryId, newAuthor.name, newAuthor.id)
|
||||
Database.addAuthorToFilterData(libraryItemData.libraryId, newAuthor.name, newAuthor.id)
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new author "${authorName}"`)
|
||||
authorsUpdated = true
|
||||
}
|
||||
@ -213,7 +215,7 @@ class BookScanner {
|
||||
// Check for series added
|
||||
for (const seriesObj of bookMetadata.series) {
|
||||
if (!media.series.some(se => se.name === seriesObj.name)) {
|
||||
const existingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name)
|
||||
const existingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name)
|
||||
if (existingSeries) {
|
||||
await Database.bookSeriesModel.create({
|
||||
bookId: media.id,
|
||||
@ -226,10 +228,10 @@ class BookScanner {
|
||||
const newSeries = await Database.seriesModel.create({
|
||||
name: seriesObj.name,
|
||||
nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name),
|
||||
libraryId: libraryScan.libraryId
|
||||
libraryId: libraryItemData.libraryId
|
||||
})
|
||||
await media.addSeries(newSeries)
|
||||
Database.addSeriesToFilterData(libraryScan.libraryId, newSeries.name, newSeries.id)
|
||||
await media.addSeries(newSeries, { through: { sequence: seriesObj.sequence } })
|
||||
Database.addSeriesToFilterData(libraryItemData.libraryId, newSeries.name, newSeries.id)
|
||||
libraryScan.addLog(LogLevel.DEBUG, `Updating book "${bookMetadata.title}" added new series "${seriesObj.name}"${seriesObj.sequence ? ` with sequence "${seriesObj.sequence}"` : ''}`)
|
||||
seriesUpdated = true
|
||||
}
|
||||
@ -304,7 +306,7 @@ class BookScanner {
|
||||
media.authors = await media.getAuthors({
|
||||
joinTableAttributes: ['createdAt'],
|
||||
order: [
|
||||
Sequelize.literal(`bookAuthor.createdAt ASC`)
|
||||
sequelize.literal(`bookAuthor.createdAt ASC`)
|
||||
]
|
||||
})
|
||||
}
|
||||
@ -312,7 +314,7 @@ class BookScanner {
|
||||
media.series = await media.getSeries({
|
||||
joinTableAttributes: ['sequence', 'createdAt'],
|
||||
order: [
|
||||
Sequelize.literal(`bookSeries.createdAt ASC`)
|
||||
sequelize.literal(`bookSeries.createdAt ASC`)
|
||||
]
|
||||
})
|
||||
}
|
||||
@ -356,16 +358,17 @@ class BookScanner {
|
||||
/**
|
||||
*
|
||||
* @param {import('./LibraryItemScanData')} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {import('./LibraryScan')} libraryScan
|
||||
* @returns {import('../models/LibraryItem')}
|
||||
*/
|
||||
async scanNewBookLibraryItem(libraryItemData, libraryScan) {
|
||||
async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
// Scan audio files found
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryScan.libraryMediaType, libraryItemData, libraryItemData.audioLibraryFiles)
|
||||
let scannedAudioFiles = await AudioFileScanner.executeMediaFileScans(libraryItemData.mediaType, libraryItemData, libraryItemData.audioLibraryFiles)
|
||||
scannedAudioFiles = AudioFileScanner.runSmartTrackOrder(libraryItemData.relPath, scannedAudioFiles)
|
||||
|
||||
// Find ebook file (prefer epub)
|
||||
let ebookLibraryFile = libraryScan.library.settings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
|
||||
let ebookLibraryFile = librarySettings.audiobooksOnly ? null : libraryItemData.ebookLibraryFiles.find(lf => lf.metadata.ext.slice(1).toLowerCase() === 'epub') || libraryItemData.ebookLibraryFiles[0]
|
||||
|
||||
// Do not add library items that have no valid audio files and no ebook file
|
||||
if (!ebookLibraryFile && !scannedAudioFiles.length) {
|
||||
@ -393,7 +396,7 @@ class BookScanner {
|
||||
}
|
||||
if (bookMetadata.authors.length) {
|
||||
for (const authorName of bookMetadata.authors) {
|
||||
const matchingAuthor = Database.libraryFilterData[libraryScan.libraryId].authors.find(au => au.name === authorName)
|
||||
const matchingAuthor = Database.libraryFilterData[libraryItemData.libraryId].authors.find(au => au.name === authorName)
|
||||
if (matchingAuthor) {
|
||||
bookObject.bookAuthors.push({
|
||||
authorId: matchingAuthor.id
|
||||
@ -402,7 +405,7 @@ class BookScanner {
|
||||
// New author
|
||||
bookObject.bookAuthors.push({
|
||||
author: {
|
||||
libraryId: libraryScan.libraryId,
|
||||
libraryId: libraryItemData.libraryId,
|
||||
name: authorName,
|
||||
lastFirst: parseNameString.nameToLastFirst(authorName)
|
||||
}
|
||||
@ -413,7 +416,7 @@ class BookScanner {
|
||||
if (bookMetadata.series.length) {
|
||||
for (const seriesObj of bookMetadata.series) {
|
||||
if (!seriesObj.name) continue
|
||||
const matchingSeries = Database.libraryFilterData[libraryScan.libraryId].series.find(se => se.name === seriesObj.name)
|
||||
const matchingSeries = Database.libraryFilterData[libraryItemData.libraryId].series.find(se => se.name === seriesObj.name)
|
||||
if (matchingSeries) {
|
||||
bookObject.bookSeries.push({
|
||||
seriesId: matchingSeries.id,
|
||||
@ -425,7 +428,7 @@ class BookScanner {
|
||||
series: {
|
||||
name: seriesObj.name,
|
||||
nameIgnorePrefix: getTitleIgnorePrefix(seriesObj.name),
|
||||
libraryId: libraryScan.libraryId
|
||||
libraryId: libraryItemData.libraryId
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -476,22 +479,22 @@ class BookScanner {
|
||||
if (libraryItem.book.bookSeries?.length) {
|
||||
for (const bs of libraryItem.book.bookSeries) {
|
||||
if (bs.series) {
|
||||
Database.addSeriesToFilterData(libraryScan.libraryId, bs.series.name, bs.series.id)
|
||||
Database.addSeriesToFilterData(libraryItemData.libraryId, bs.series.name, bs.series.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (libraryItem.book.bookAuthors?.length) {
|
||||
for (const ba of libraryItem.book.bookAuthors) {
|
||||
if (ba.author) {
|
||||
Database.addAuthorToFilterData(libraryScan.libraryId, ba.author.name, ba.author.id)
|
||||
Database.addAuthorToFilterData(libraryItemData.libraryId, ba.author.name, ba.author.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
Database.addNarratorsToFilterData(libraryScan.libraryId, libraryItem.book.narrators)
|
||||
Database.addGenresToFilterData(libraryScan.libraryId, libraryItem.book.genres)
|
||||
Database.addTagsToFilterData(libraryScan.libraryId, libraryItem.book.tags)
|
||||
Database.addPublisherToFilterData(libraryScan.libraryId, libraryItem.book.publisher)
|
||||
Database.addLanguageToFilterData(libraryScan.libraryId, libraryItem.book.language)
|
||||
Database.addNarratorsToFilterData(libraryItemData.libraryId, libraryItem.book.narrators)
|
||||
Database.addGenresToFilterData(libraryItemData.libraryId, libraryItem.book.genres)
|
||||
Database.addTagsToFilterData(libraryItemData.libraryId, libraryItem.book.tags)
|
||||
Database.addPublisherToFilterData(libraryItemData.libraryId, libraryItem.book.publisher)
|
||||
Database.addLanguageToFilterData(libraryItemData.libraryId, libraryItem.book.language)
|
||||
|
||||
// Load for emitting to client
|
||||
libraryItem.media = await libraryItem.getMedia({
|
||||
@ -949,5 +952,78 @@ class BookScanner {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check authors that were removed from a book and remove them if they no longer have any books
|
||||
* keep authors without books that have a asin, description or imagePath
|
||||
* @param {string} libraryId
|
||||
* @param {import('./ScanLogger')} scanLogger
|
||||
* @returns {Promise}
|
||||
*/
|
||||
async checkAuthorsRemovedFromBooks(libraryId, scanLogger) {
|
||||
const bookAuthorsToRemove = (await Database.authorModel.findAll({
|
||||
where: [
|
||||
{
|
||||
id: scanLogger.authorsRemovedFromBooks,
|
||||
asin: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
},
|
||||
description: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
},
|
||||
imagePath: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
}
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
|
||||
],
|
||||
attributes: ['id'],
|
||||
raw: true
|
||||
})).map(au => au.id)
|
||||
if (bookAuthorsToRemove.length) {
|
||||
await Database.authorModel.destroy({
|
||||
where: {
|
||||
id: bookAuthorsToRemove
|
||||
}
|
||||
})
|
||||
bookAuthorsToRemove.forEach((authorId) => {
|
||||
Database.removeAuthorFromFilterData(libraryId, authorId)
|
||||
// TODO: Clients were expecting full author in payload but its unnecessary
|
||||
SocketAuthority.emitter('author_removed', { id: authorId, libraryId })
|
||||
})
|
||||
scanLogger.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check series that were removed from books and remove them if they no longer have any books
|
||||
* @param {string} libraryId
|
||||
* @param {import('./ScanLogger')} scanLogger
|
||||
* @returns {Promise}
|
||||
*/
|
||||
async checkSeriesRemovedFromBooks(libraryId, scanLogger) {
|
||||
const bookSeriesToRemove = (await Database.seriesModel.findAll({
|
||||
where: [
|
||||
{
|
||||
id: scanLogger.seriesRemovedFromBooks
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0)
|
||||
],
|
||||
attributes: ['id'],
|
||||
raw: true
|
||||
})).map(se => se.id)
|
||||
if (bookSeriesToRemove.length) {
|
||||
await Database.seriesModel.destroy({
|
||||
where: {
|
||||
id: bookSeriesToRemove
|
||||
}
|
||||
})
|
||||
bookSeriesToRemove.forEach((seriesId) => {
|
||||
Database.removeSeriesFromFilterData(libraryId, seriesId)
|
||||
SocketAuthority.emitter('series_removed', { id: seriesId, libraryId })
|
||||
})
|
||||
scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
|
||||
}
|
||||
}
|
||||
}
|
||||
module.exports = new BookScanner()
|
||||
190
server/scanner/LibraryItemScanner.js
Normal file
190
server/scanner/LibraryItemScanner.js
Normal file
@ -0,0 +1,190 @@
|
||||
const Path = require('path')
|
||||
const { LogLevel, ScanResult } = require('../utils/constants')
|
||||
|
||||
const fileUtils = require('../utils/fileUtils')
|
||||
const scanUtils = require('../utils/scandir')
|
||||
const libraryFilters = require('../utils/queries/libraryFilters')
|
||||
const Database = require('../Database')
|
||||
const LibraryScan = require('./LibraryScan')
|
||||
const LibraryItemScanData = require('./LibraryItemScanData')
|
||||
const BookScanner = require('./BookScanner')
|
||||
const ScanLogger = require('./ScanLogger')
|
||||
const LibraryItem = require('../models/LibraryItem')
|
||||
const LibraryFile = require('../objects/files/LibraryFile')
|
||||
const SocketAuthority = require('../SocketAuthority')
|
||||
|
||||
class LibraryItemScanner {
|
||||
constructor() { }
|
||||
|
||||
/**
|
||||
* Scan single library item
|
||||
*
|
||||
* @param {string} libraryItemId
|
||||
* @returns {number} ScanResult
|
||||
*/
|
||||
async scanLibraryItem(libraryItemId) {
|
||||
// TODO: Add task manager
|
||||
const libraryItem = await Database.libraryItemModel.findByPk(libraryItemId)
|
||||
if (!libraryItem) {
|
||||
Logger.error(`[LibraryItemScanner] Library item not found "${libraryItemId}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
|
||||
const library = await Database.libraryModel.findByPk(libraryItem.libraryId, {
|
||||
include: {
|
||||
model: Database.libraryFolderModel,
|
||||
where: {
|
||||
id: libraryItem.libraryFolderId
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!library) {
|
||||
Logger.error(`[LibraryItemScanner] Library "${libraryItem.libraryId}" not found for library item "${libraryItem.id}"`)
|
||||
return ScanResult.NOTHING
|
||||
}
|
||||
|
||||
// Make sure library filter data is set
|
||||
// this is used to check for existing authors & series
|
||||
await libraryFilters.getFilterData(library.mediaType, library.id)
|
||||
|
||||
const scanLogger = new ScanLogger()
|
||||
scanLogger.verbose = true
|
||||
scanLogger.setData('libraryItem', libraryItemId)
|
||||
|
||||
const libraryItemPath = fileUtils.filePathToPOSIX(libraryItem.path)
|
||||
const folder = library.libraryFolders[0]
|
||||
const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, false)
|
||||
|
||||
if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) {
|
||||
if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) {
|
||||
const expandedLibraryItem = await this.rescanLibraryItem(libraryItem, libraryItemScanData, library.settings, scanLogger)
|
||||
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
|
||||
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
|
||||
|
||||
await this.checkAuthorsAndSeriesRemovedFromBooks(library.id, scanLogger)
|
||||
} else {
|
||||
// TODO: Temporary while using old model to socket emit
|
||||
const oldLibraryItem = await Database.libraryItemModel.getOldById(libraryItem.id)
|
||||
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
|
||||
}
|
||||
|
||||
return ScanResult.UPDATED
|
||||
}
|
||||
return ScanResult.UPTODATE
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove empty authors and series
|
||||
* @param {string} libraryId
|
||||
* @param {ScanLogger} scanLogger
|
||||
* @returns {Promise}
|
||||
*/
|
||||
async checkAuthorsAndSeriesRemovedFromBooks(libraryId, scanLogger) {
|
||||
if (scanLogger.authorsRemovedFromBooks.length) {
|
||||
await BookScanner.checkAuthorsRemovedFromBooks(libraryId, scanLogger)
|
||||
}
|
||||
if (scanLogger.seriesRemovedFromBooks.length) {
|
||||
await BookScanner.checkSeriesRemovedFromBooks(libraryId, scanLogger)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {string} libraryItemPath
|
||||
* @param {import('../models/Library')} library
|
||||
* @param {import('../models/LibraryFolder')} folder
|
||||
* @param {boolean} isSingleMediaItem
|
||||
* @returns {Promise<LibraryItemScanData>}
|
||||
*/
|
||||
async getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem) {
|
||||
const libraryFolderPath = fileUtils.filePathToPOSIX(folder.path)
|
||||
const libraryItemDir = libraryItemPath.replace(libraryFolderPath, '').slice(1)
|
||||
|
||||
let libraryItemData = {}
|
||||
|
||||
let fileItems = []
|
||||
|
||||
if (isSingleMediaItem) { // Single media item in root of folder
|
||||
fileItems = [
|
||||
{
|
||||
fullpath: libraryItemPath,
|
||||
path: libraryItemDir // actually the relPath (only filename here)
|
||||
}
|
||||
]
|
||||
libraryItemData = {
|
||||
path: libraryItemPath, // full path
|
||||
relPath: libraryItemDir, // only filename
|
||||
mediaMetadata: {
|
||||
title: Path.basename(libraryItemDir, Path.extname(libraryItemDir))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fileItems = await fileUtils.recurseFiles(libraryItemPath)
|
||||
libraryItemData = scanUtils.getDataFromMediaDir(library.mediaType, libraryFolderPath, libraryItemDir)
|
||||
}
|
||||
|
||||
const libraryFiles = []
|
||||
for (let i = 0; i < fileItems.length; i++) {
|
||||
const fileItem = fileItems[i]
|
||||
const newLibraryFile = new LibraryFile()
|
||||
// fileItem.path is the relative path
|
||||
await newLibraryFile.setDataFromPath(fileItem.fullpath, fileItem.path)
|
||||
libraryFiles.push(newLibraryFile)
|
||||
}
|
||||
|
||||
const libraryItemStats = await fileUtils.getFileTimestampsWithIno(libraryItemData.path)
|
||||
return new LibraryItemScanData({
|
||||
libraryFolderId: folder.id,
|
||||
libraryId: library.id,
|
||||
mediaType: library.mediaType,
|
||||
ino: libraryItemStats.ino,
|
||||
mtimeMs: libraryItemStats.mtimeMs || 0,
|
||||
ctimeMs: libraryItemStats.ctimeMs || 0,
|
||||
birthtimeMs: libraryItemStats.birthtimeMs || 0,
|
||||
path: libraryItemData.path,
|
||||
relPath: libraryItemData.relPath,
|
||||
isFile: isSingleMediaItem,
|
||||
mediaMetadata: libraryItemData.mediaMetadata || null,
|
||||
libraryFiles
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {LibraryItemScanData} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {LibraryScan} libraryScan
|
||||
* @returns {Promise<LibraryItem>}
|
||||
*/
|
||||
async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
|
||||
if (existingLibraryItem.mediaType === 'book') {
|
||||
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
|
||||
return libraryItem
|
||||
} else {
|
||||
// TODO: Scan updated podcast
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {LibraryItemScanData} libraryItemData
|
||||
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
|
||||
* @param {LibraryScan} libraryScan
|
||||
* @returns {Promise<LibraryItem>}
|
||||
*/
|
||||
async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) {
|
||||
if (libraryScan.libraryMediaType === 'book') {
|
||||
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
|
||||
if (newLibraryItem) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
||||
}
|
||||
return newLibraryItem
|
||||
} else {
|
||||
// TODO: Scan new podcast
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
module.exports = new LibraryItemScanner()
|
||||
@ -7,18 +7,15 @@ const Database = require('../Database')
|
||||
const fs = require('../libs/fsExtra')
|
||||
const fileUtils = require('../utils/fileUtils')
|
||||
const scanUtils = require('../utils/scandir')
|
||||
const { ScanResult, LogLevel } = require('../utils/constants')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
const libraryFilters = require('../utils/queries/libraryFilters')
|
||||
const LibraryItemScanner = require('./LibraryItemScanner')
|
||||
const ScanOptions = require('./ScanOptions')
|
||||
const LibraryScan = require('./LibraryScan')
|
||||
const LibraryItemScanData = require('./LibraryItemScanData')
|
||||
const BookScanner = require('./BookScanner')
|
||||
|
||||
class LibraryScanner {
|
||||
constructor(coverManager, taskManager) {
|
||||
this.coverManager = coverManager
|
||||
this.taskManager = taskManager
|
||||
|
||||
constructor() {
|
||||
this.cancelLibraryScan = {}
|
||||
this.librariesScanning = []
|
||||
}
|
||||
@ -93,7 +90,7 @@ class LibraryScanner {
|
||||
async scanLibrary(libraryScan) {
|
||||
// Make sure library filter data is set
|
||||
// this is used to check for existing authors & series
|
||||
await libraryFilters.getFilterData(libraryScan.library)
|
||||
await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId)
|
||||
|
||||
/** @type {LibraryItemScanData[]} */
|
||||
let libraryItemDataFound = []
|
||||
@ -151,7 +148,7 @@ class LibraryScanner {
|
||||
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
|
||||
libraryScan.resultsUpdated++
|
||||
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
|
||||
const libraryItem = await this.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
|
||||
const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
|
||||
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
|
||||
oldLibraryItemsUpdated.push(oldLibraryItem)
|
||||
} else {
|
||||
@ -177,68 +174,8 @@ class LibraryScanner {
|
||||
SocketAuthority.emitter('items_updated', oldLibraryItemsUpdated.map(li => li.toJSONExpanded()))
|
||||
}
|
||||
|
||||
// Check authors that were removed from a book and remove them if they no longer have any books
|
||||
// keep authors without books that have a asin, description or imagePath
|
||||
if (libraryScan.authorsRemovedFromBooks.length) {
|
||||
const bookAuthorsToRemove = (await Database.authorModel.findAll({
|
||||
where: [
|
||||
{
|
||||
id: libraryScan.authorsRemovedFromBooks,
|
||||
asin: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
},
|
||||
description: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
},
|
||||
imagePath: {
|
||||
[sequelize.Op.or]: [null, ""]
|
||||
}
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 0)
|
||||
],
|
||||
attributes: ['id'],
|
||||
raw: true
|
||||
})).map(au => au.id)
|
||||
if (bookAuthorsToRemove.length) {
|
||||
await Database.authorModel.destroy({
|
||||
where: {
|
||||
id: bookAuthorsToRemove
|
||||
}
|
||||
})
|
||||
bookAuthorsToRemove.forEach((authorId) => {
|
||||
Database.removeAuthorFromFilterData(libraryScan.libraryId, authorId)
|
||||
// TODO: Clients were expecting full author in payload but its unnecessary
|
||||
SocketAuthority.emitter('author_removed', { id: authorId, libraryId: libraryScan.libraryId })
|
||||
})
|
||||
libraryScan.addLog(LogLevel.INFO, `Removed ${bookAuthorsToRemove.length} authors`)
|
||||
}
|
||||
}
|
||||
|
||||
// Check series that were removed from books and remove them if they no longer have any books
|
||||
if (libraryScan.seriesRemovedFromBooks.length) {
|
||||
const bookSeriesToRemove = (await Database.seriesModel.findAll({
|
||||
where: [
|
||||
{
|
||||
id: libraryScan.seriesRemovedFromBooks
|
||||
},
|
||||
sequelize.where(sequelize.literal('(SELECT count(*) FROM bookSeries bs WHERE bs.seriesId = series.id)'), 0)
|
||||
],
|
||||
attributes: ['id'],
|
||||
raw: true
|
||||
})).map(se => se.id)
|
||||
if (bookSeriesToRemove.length) {
|
||||
await Database.seriesModel.destroy({
|
||||
where: {
|
||||
id: bookSeriesToRemove
|
||||
}
|
||||
})
|
||||
bookSeriesToRemove.forEach((seriesId) => {
|
||||
Database.removeSeriesFromFilterData(libraryScan.libraryId, seriesId)
|
||||
SocketAuthority.emitter('series_removed', { id: seriesId, libraryId: libraryScan.libraryId })
|
||||
})
|
||||
libraryScan.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
|
||||
}
|
||||
}
|
||||
// Authors and series that were removed from books should be removed if they are now empty
|
||||
await LibraryItemScanner.checkAuthorsAndSeriesRemovedFromBooks(libraryScan.libraryId, libraryScan)
|
||||
|
||||
// Update missing library items
|
||||
if (libraryItemIdsMissing.length) {
|
||||
@ -260,7 +197,7 @@ class LibraryScanner {
|
||||
if (libraryItemDataFound.length) {
|
||||
let newOldLibraryItems = []
|
||||
for (const libraryItemData of libraryItemDataFound) {
|
||||
const newLibraryItem = await this.scanNewLibraryItem(libraryItemData, libraryScan)
|
||||
const newLibraryItem = await LibraryItemScanner.scanNewLibraryItem(libraryItemData, libraryScan.library.settings, libraryScan)
|
||||
if (newLibraryItem) {
|
||||
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(newLibraryItem)
|
||||
newOldLibraryItems.push(oldLibraryItem)
|
||||
@ -353,38 +290,5 @@ class LibraryScanner {
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('../models/LibraryItem')} existingLibraryItem
|
||||
* @param {LibraryItemScanData} libraryItemData
|
||||
* @param {LibraryScan} libraryScan
|
||||
*/
|
||||
async rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan) {
|
||||
if (existingLibraryItem.mediaType === 'book') {
|
||||
const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, libraryScan)
|
||||
return libraryItem
|
||||
} else {
|
||||
// TODO: Scan updated podcast
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {LibraryItemScanData} libraryItemData
|
||||
* @param {LibraryScan} libraryScan
|
||||
*/
|
||||
async scanNewLibraryItem(libraryItemData, libraryScan) {
|
||||
if (libraryScan.libraryMediaType === 'book') {
|
||||
const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, libraryScan)
|
||||
if (newLibraryItem) {
|
||||
libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
|
||||
}
|
||||
return newLibraryItem
|
||||
} else {
|
||||
// TODO: Scan new podcast
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
module.exports = LibraryScanner
|
||||
module.exports = new LibraryScanner()
|
||||
70
server/scanner/ScanLogger.js
Normal file
70
server/scanner/ScanLogger.js
Normal file
@ -0,0 +1,70 @@
|
||||
const uuidv4 = require("uuid").v4
|
||||
const Logger = require('../Logger')
|
||||
const { LogLevel } = require('../utils/constants')
|
||||
|
||||
class ScanLogger {
|
||||
constructor() {
|
||||
this.id = null
|
||||
this.type = null
|
||||
this.name = null
|
||||
this.verbose = false
|
||||
|
||||
this.startedAt = null
|
||||
this.finishedAt = null
|
||||
this.elapsed = null
|
||||
|
||||
/** @type {string[]} */
|
||||
this.authorsRemovedFromBooks = []
|
||||
/** @type {string[]} */
|
||||
this.seriesRemovedFromBooks = []
|
||||
|
||||
this.logs = []
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
id: this.id,
|
||||
type: this.type,
|
||||
name: this.name,
|
||||
startedAt: this.startedAt,
|
||||
finishedAt: this.finishedAt,
|
||||
elapsed: this.elapsed
|
||||
}
|
||||
}
|
||||
|
||||
setData(type, name) {
|
||||
this.id = uuidv4()
|
||||
this.type = type
|
||||
this.name = name
|
||||
this.startedAt = Date.now()
|
||||
}
|
||||
|
||||
setComplete() {
|
||||
this.finishedAt = Date.now()
|
||||
this.elapsed = this.finishedAt - this.startedAt
|
||||
}
|
||||
|
||||
getLogLevelString(level) {
|
||||
for (const key in LogLevel) {
|
||||
if (LogLevel[key] === level) {
|
||||
return key
|
||||
}
|
||||
}
|
||||
return 'UNKNOWN'
|
||||
}
|
||||
|
||||
addLog(level, ...args) {
|
||||
const logObj = {
|
||||
timestamp: (new Date()).toISOString(),
|
||||
message: args.join(' '),
|
||||
levelName: this.getLogLevelString(level),
|
||||
level
|
||||
}
|
||||
|
||||
if (this.verbose) {
|
||||
Logger.debug(`[Scan] "${this.name}":`, ...args)
|
||||
}
|
||||
this.logs.push(logObj)
|
||||
}
|
||||
}
|
||||
module.exports = ScanLogger
|
||||
@ -390,11 +390,12 @@ module.exports = {
|
||||
|
||||
/**
|
||||
* Get filter data used in filter menus
|
||||
* @param {import('../../objects/Library')} oldLibrary
|
||||
* @param {string} mediaType
|
||||
* @param {string} libraryId
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
async getFilterData(oldLibrary) {
|
||||
const cachedFilterData = Database.libraryFilterData[oldLibrary.id]
|
||||
async getFilterData(mediaType, libraryId) {
|
||||
const cachedFilterData = Database.libraryFilterData[libraryId]
|
||||
if (cachedFilterData) {
|
||||
const cacheElapsed = Date.now() - cachedFilterData.loadedAt
|
||||
// Cache library filters for 30 mins
|
||||
@ -416,13 +417,13 @@ module.exports = {
|
||||
numIssues: 0
|
||||
}
|
||||
|
||||
if (oldLibrary.isPodcast) {
|
||||
if (mediaType === 'podcast') {
|
||||
const podcasts = await Database.podcastModel.findAll({
|
||||
include: {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: [],
|
||||
where: {
|
||||
libraryId: oldLibrary.id
|
||||
libraryId: libraryId
|
||||
}
|
||||
},
|
||||
attributes: ['tags', 'genres']
|
||||
@ -441,7 +442,7 @@ module.exports = {
|
||||
model: Database.libraryItemModel,
|
||||
attributes: ['isMissing', 'isInvalid'],
|
||||
where: {
|
||||
libraryId: oldLibrary.id
|
||||
libraryId: libraryId
|
||||
}
|
||||
},
|
||||
attributes: ['tags', 'genres', 'publisher', 'narrators', 'language']
|
||||
@ -463,7 +464,7 @@ module.exports = {
|
||||
|
||||
const series = await Database.seriesModel.findAll({
|
||||
where: {
|
||||
libraryId: oldLibrary.id
|
||||
libraryId: libraryId
|
||||
},
|
||||
attributes: ['id', 'name']
|
||||
})
|
||||
@ -471,7 +472,7 @@ module.exports = {
|
||||
|
||||
const authors = await Database.authorModel.findAll({
|
||||
where: {
|
||||
libraryId: oldLibrary.id
|
||||
libraryId: libraryId
|
||||
},
|
||||
attributes: ['id', 'name']
|
||||
})
|
||||
@ -486,7 +487,7 @@ module.exports = {
|
||||
data.publishers = naturalSort([...data.publishers]).asc()
|
||||
data.languages = naturalSort([...data.languages]).asc()
|
||||
data.loadedAt = Date.now()
|
||||
Database.libraryFilterData[oldLibrary.id] = data
|
||||
Database.libraryFilterData[libraryId] = data
|
||||
|
||||
Logger.debug(`Loaded filterdata in ${((Date.now() - start) / 1000).toFixed(2)}s`)
|
||||
return data
|
||||
|
||||
Loading…
Reference in New Issue
Block a user