Remove force re-scan and old scanner logic

This commit is contained in:
advplyr 2023-09-04 13:59:37 -05:00
parent b9da3fa30e
commit b1c07834be
14 changed files with 146 additions and 689 deletions

View File

@ -11,10 +11,6 @@
<ui-btn @click="clickAddLibrary">{{ $strings.ButtonAddYourFirstLibrary }}</ui-btn>
</div>
<p v-if="libraries.length" class="text-xs mt-4 text-gray-200">
*<strong>{{ $strings.ButtonForceReScan }}</strong> {{ $strings.MessageForceReScanDescription }}
</p>
<p v-if="libraries.length && libraries.some((li) => li.mediaType === 'book')" class="text-xs mt-4 text-gray-200">
**<strong>{{ $strings.ButtonMatchBooks }}</strong> {{ $strings.MessageMatchBooksDescription }}
</p>

View File

@ -71,11 +71,6 @@ export default {
text: this.$strings.ButtonScan,
action: 'scan',
value: 'scan'
},
{
text: this.$strings.ButtonForceReScan,
action: 'force-scan',
value: 'force-scan'
}
]
if (this.isBookLibrary) {
@ -137,26 +132,6 @@ export default {
this.$toast.error(this.$strings.ToastLibraryScanFailedToStart)
})
},
forceScan() {
const payload = {
message: this.$strings.MessageConfirmForceReScan,
callback: (confirmed) => {
if (confirmed) {
this.$store
.dispatch('libraries/requestLibraryScan', { libraryId: this.library.id, force: 1 })
.then(() => {
this.$toast.success(this.$strings.ToastLibraryScanStarted)
})
.catch((error) => {
console.error('Failed to start scan', error)
this.$toast.error(this.$strings.ToastLibraryScanFailedToStart)
})
}
},
type: 'yesNo'
}
this.$store.commit('globals/setConfirmPrompt', payload)
},
deleteClick() {
const payload = {
message: this.$getString('MessageConfirmDeleteLibrary', [this.library.name]),

View File

@ -974,12 +974,9 @@ class LibraryController {
Logger.error(`[LibraryController] Non-root user attempted to scan library`, req.user)
return res.sendStatus(403)
}
const options = {
forceRescan: req.query.force == 1
}
res.sendStatus(200)
await LibraryScanner.scan(req.library, options)
await LibraryScanner.scan(req.library)
await Database.resetLibraryIssuesFilterData(req.library.id)
Logger.info('[LibraryController] Scan complete')

View File

@ -9,6 +9,7 @@ const { reqSupportsWebp } = require('../utils/index')
const { ScanResult } = require('../utils/constants')
const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils')
const LibraryItemScanner = require('../scanner/LibraryItemScanner')
const AudioFileScanner = require('../scanner/AudioFileScanner')
class LibraryItemController {
constructor() { }
@ -555,7 +556,7 @@ class LibraryItemController {
return res.sendStatus(404)
}
const ffprobeData = await this.scanner.probeAudioFile(audioFile)
const ffprobeData = await AudioFileScanner.probeAudioFile(audioFile)
res.json(ffprobeData)
}

View File

@ -338,183 +338,6 @@ class LibraryItem {
return hasUpdated
}
// Data pulled from scandir during a scan, check it with current data
checkScanData(dataFound) {
let hasUpdated = false
if (this.isMissing) {
// Item no longer missing
this.isMissing = false
hasUpdated = true
}
if (dataFound.isFile !== this.isFile && dataFound.isFile !== undefined) {
Logger.info(`[LibraryItem] Check scan item isFile toggled from ${this.isFile} => ${dataFound.isFile}`)
this.isFile = dataFound.isFile
hasUpdated = true
}
if (dataFound.ino !== this.ino) {
Logger.warn(`[LibraryItem] Check scan item changed inode "${this.ino}" -> "${dataFound.ino}"`)
this.ino = dataFound.ino
hasUpdated = true
}
if (dataFound.folderId !== this.folderId) {
Logger.warn(`[LibraryItem] Check scan item changed folder ${this.folderId} -> ${dataFound.folderId}`)
this.folderId = dataFound.folderId
hasUpdated = true
}
if (dataFound.path !== this.path) {
Logger.warn(`[LibraryItem] Check scan item changed path "${this.path}" -> "${dataFound.path}" (inode ${this.ino})`)
this.path = dataFound.path
this.relPath = dataFound.relPath
hasUpdated = true
}
['mtimeMs', 'ctimeMs', 'birthtimeMs'].forEach((key) => {
if (dataFound[key] != this[key]) {
this[key] = dataFound[key] || 0
hasUpdated = true
}
})
const newLibraryFiles = []
const existingLibraryFiles = []
dataFound.libraryFiles.forEach((lf) => {
const fileFoundCheck = this.checkFileFound(lf, true)
if (fileFoundCheck === null) {
newLibraryFiles.push(lf)
} else if (fileFoundCheck && lf.metadata.format !== 'abs' && lf.metadata.filename !== 'metadata.json') { // Ignore abs file updates
hasUpdated = true
existingLibraryFiles.push(lf)
} else {
existingLibraryFiles.push(lf)
}
})
const filesRemoved = []
// Remove files not found (inodes will all be up to date at this point)
this.libraryFiles = this.libraryFiles.filter(lf => {
if (!dataFound.libraryFiles.find(_lf => _lf.ino === lf.ino)) {
// Check if removing cover path
if (lf.metadata.path === this.media.coverPath) {
Logger.debug(`[LibraryItem] "${this.media.metadata.title}" check scan cover removed`)
this.media.updateCover('')
}
filesRemoved.push(lf.toJSON())
this.media.removeFileWithInode(lf.ino)
return false
}
return true
})
if (filesRemoved.length) {
if (this.media.mediaType === 'book') {
this.media.checkUpdateMissingTracks()
}
hasUpdated = true
}
// Add library files to library item
if (newLibraryFiles.length) {
newLibraryFiles.forEach((lf) => this.libraryFiles.push(lf.clone()))
hasUpdated = true
}
// Check if invalid
this.isInvalid = !this.media.hasMediaEntities
// If cover path is in item folder, make sure libraryFile exists for it
if (this.media.coverPath && this.media.coverPath.startsWith(this.path)) {
const lf = this.libraryFiles.find(lf => lf.metadata.path === this.media.coverPath)
if (!lf) {
Logger.warn(`[LibraryItem] Invalid cover path - library file dne "${this.media.coverPath}"`)
this.media.updateCover('')
hasUpdated = true
}
}
if (hasUpdated) {
this.setLastScan()
}
return {
updated: hasUpdated,
newLibraryFiles,
filesRemoved,
existingLibraryFiles // Existing file data may get re-scanned if forceRescan is set
}
}
// Set metadata from files
async syncFiles(preferOpfMetadata, librarySettings) {
let hasUpdated = false
if (this.isBook) {
// Add/update ebook files (ebooks that were removed are removed in checkScanData)
if (librarySettings.audiobooksOnly) {
hasUpdated = this.media.ebookFile
if (hasUpdated) {
// If library was set to audiobooks only then set primary ebook as supplementary
Logger.info(`[LibraryItem] Library is audiobooks only so setting ebook "${this.media.ebookFile.metadata.filename}" as supplementary`)
}
this.setPrimaryEbook(null)
} else if (this.media.ebookFile) {
const matchingLibraryFile = this.libraryFiles.find(lf => lf.ino === this.media.ebookFile.ino)
if (matchingLibraryFile && this.media.ebookFile.updateFromLibraryFile(matchingLibraryFile)) {
hasUpdated = true
}
// Set any other ebook files as supplementary
const suppEbookLibraryFiles = this.libraryFiles.filter(lf => lf.isEBookFile && !lf.isSupplementary && this.media.ebookFile.ino !== lf.ino)
if (suppEbookLibraryFiles.length) {
for (const libraryFile of suppEbookLibraryFiles) {
libraryFile.isSupplementary = true
}
hasUpdated = true
}
} else {
const ebookLibraryFiles = this.libraryFiles.filter(lf => lf.isEBookFile && !lf.isSupplementary)
// Prefer epub ebook then fallback to first other ebook file
const ebookLibraryFile = ebookLibraryFiles.find(lf => lf.metadata.format === 'epub') || ebookLibraryFiles[0]
if (ebookLibraryFile) {
this.setPrimaryEbook(ebookLibraryFile)
hasUpdated = true
}
}
}
// Set cover image if not set
const imageFiles = this.libraryFiles.filter(lf => lf.fileType === 'image')
if (imageFiles.length && !this.media.coverPath) {
// attempt to find a file called cover.<ext> otherwise just fall back to the first image found
const coverMatch = imageFiles.find(iFile => /\/cover\.[^.\/]*$/.test(iFile.metadata.path))
if (coverMatch) {
this.media.coverPath = coverMatch.metadata.path
} else {
this.media.coverPath = imageFiles[0].metadata.path
}
Logger.info('[LibraryItem] Set media cover path', this.media.coverPath)
hasUpdated = true
}
// Parse metadata files
const textMetadataFiles = this.libraryFiles.filter(lf => lf.fileType === 'metadata' || lf.fileType === 'text')
if (textMetadataFiles.length) {
if (await this.media.syncMetadataFiles(textMetadataFiles, preferOpfMetadata)) {
hasUpdated = true
}
}
if (hasUpdated) {
this.updatedAt = Date.now()
}
return hasUpdated
}
searchQuery(query) {
query = cleanStringForSearch(query)
return this.media.searchQuery(query)
@ -556,19 +379,27 @@ class LibraryItem {
return fs.writeFile(metadataFilePath, JSON.stringify(this.media.toJSONForMetadataFile(), null, 2)).then(async () => {
// Add metadata.json to libraryFiles array if it is new
let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
this.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
this.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(this.path)
if (libraryItemDirTimestamps) {
this.mtimeMs = libraryItemDirTimestamps.mtimeMs
this.ctimeMs = libraryItemDirTimestamps.ctimeMs
}
}
Logger.debug(`[LibraryItem] Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
@ -593,17 +424,24 @@ class LibraryItem {
}
// Add metadata.abs to libraryFiles array if it is new
let metadataLibraryFile = this.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
this.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
metadataLibraryFile = new LibraryFile()
await metadataLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
this.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(this.path)
if (libraryItemDirTimestamps) {
this.mtimeMs = libraryItemDirTimestamps.mtimeMs
this.ctimeMs = libraryItemDirTimestamps.ctimeMs
}
}

View File

@ -116,7 +116,7 @@ class AudioFile {
return !this.invalid && !this.exclude
}
// New scanner creates AudioFile from MediaFileScanner
// New scanner creates AudioFile from AudioFileScanner
setDataFromProbe(libraryFile, probeData) {
this.ino = libraryFile.ino || null

View File

@ -153,12 +153,12 @@ class AudioFileScanner {
const probeData = await prober.probe(libraryFile.metadata.path)
if (probeData.error) {
Logger.error(`[MediaFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`)
Logger.error(`[AudioFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`)
return null
}
if (!probeData.audioStream) {
Logger.error('[MediaFileScanner] Invalid audio file no audio stream')
Logger.error('[AudioFileScanner] Invalid audio file no audio stream')
return null
}
@ -195,5 +195,15 @@ class AudioFileScanner {
return results
}
/**
*
* @param {AudioFile} audioFile
* @returns {object}
*/
probeAudioFile(audioFile) {
Logger.debug(`[AudioFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`)
return prober.rawProbe(audioFile.metadata.path)
}
}
module.exports = new AudioFileScanner()

View File

@ -899,20 +899,31 @@ class BookScanner {
return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
// Add metadata.json to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
if (libraryItemDirTimestamps) {
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
let size = 0
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.size = size
}
}
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
@ -935,18 +946,28 @@ class BookScanner {
}
// Add metadata.abs to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
if (libraryItemDirTimestamps) {
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
let size = 0
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.size = size
}
}

View File

@ -17,8 +17,6 @@ class LibraryScan {
this.library = null
this.verbose = false
this.scanOptions = null
this.startedAt = null
this.finishedAt = null
this.elapsed = null
@ -40,12 +38,6 @@ class LibraryScan {
get libraryMediaType() { return this.library.mediaType }
get folders() { return this.library.folders }
get _scanOptions() { return this.scanOptions || {} }
get forceRescan() { return !!this._scanOptions.forceRescan }
get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
get preferOverdriveMediaMarker() { return !!this._scanOptions.preferOverdriveMediaMarker }
get findCovers() { return !!this._scanOptions.findCovers }
get timestamp() {
return (new Date()).toISOString()
}
@ -80,7 +72,6 @@ class LibraryScan {
id: this.id,
type: this.type,
library: this.library.toJSON(),
scanOptions: this.scanOptions ? this.scanOptions.toJSON() : null,
startedAt: this.startedAt,
finishedAt: this.finishedAt,
elapsed: this.elapsed,
@ -90,13 +81,11 @@ class LibraryScan {
}
}
setData(library, scanOptions, type = 'scan') {
setData(library, type = 'scan') {
this.id = uuidv4()
this.type = type
this.library = new Library(library.toJSON()) // clone library
this.scanOptions = scanOptions
this.startedAt = Date.now()
}

View File

@ -10,7 +10,6 @@ const scanUtils = require('../utils/scandir')
const { LogLevel, ScanResult } = require('../utils/constants')
const libraryFilters = require('../utils/queries/libraryFilters')
const LibraryItemScanner = require('./LibraryItemScanner')
const ScanOptions = require('./ScanOptions')
const LibraryScan = require('./LibraryScan')
const LibraryItemScanData = require('./LibraryItemScanData')
@ -58,11 +57,8 @@ class LibraryScanner {
return
}
const scanOptions = new ScanOptions()
scanOptions.setData(options, Database.serverSettings)
const libraryScan = new LibraryScan()
libraryScan.setData(library, scanOptions)
libraryScan.setData(library)
libraryScan.verbose = true
this.librariesScanning.push(libraryScan.getScanEmitData)
@ -471,7 +467,7 @@ class LibraryScanner {
})
if (existingLibraryItem) {
Logger.debug(`[LibraryScanner] scanFolderUpdates: Library item found by inode value=${dirIno}. "${existingLibraryItem.relPath} => ${itemDir}"`)
// Update library item paths for scan and all library item paths will get updated in LibraryItem.checkScanData
// Update library item paths for scan
existingLibraryItem.path = fullPath
existingLibraryItem.relPath = itemDir
}

View File

@ -1,342 +0,0 @@
const Path = require('path')
const AudioFile = require('../objects/files/AudioFile')
const VideoFile = require('../objects/files/VideoFile')
const prober = require('../utils/prober')
const Logger = require('../Logger')
const { LogLevel } = require('../utils/constants')
class MediaFileScanner {
constructor() { }
/**
* Get track and disc number from audio filename
* @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan
* @param {import('../objects/files/LibraryFile')} audioLibraryFile
* @returns {{trackNumber:number, discNumber:number}}
*/
getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) {
const { title, author, series, publishedYear } = mediaMetadataFromScan
const { filename, path } = audioLibraryFile.metadata
let partbasename = Path.basename(filename, Path.extname(filename))
// Remove title, author, series, and publishedYear from filename if there
if (title) partbasename = partbasename.replace(title, '')
if (author) partbasename = partbasename.replace(author, '')
if (series) partbasename = partbasename.replace(series, '')
if (publishedYear) partbasename = partbasename.replace(publishedYear)
// Look for disc number
let discNumber = null
const discMatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i)
if (discMatch && discMatch.length > 2 && discMatch[2]) {
if (!isNaN(discMatch[2])) {
discNumber = Number(discMatch[2])
}
// Remove disc number from filename
partbasename = partbasename.replace(/\b(disc|cd) ?(\d\d?)\b/i, '')
}
// Look for disc number in folder path e.g. /Book Title/CD01/audiofile.mp3
const pathdir = Path.dirname(path).split('/').pop()
if (pathdir && /^cd\d{1,3}$/i.test(pathdir)) {
const discFromFolder = Number(pathdir.replace(/cd/i, ''))
if (!isNaN(discFromFolder) && discFromFolder !== null) discNumber = discFromFolder
}
const numbersinpath = partbasename.match(/\d{1,4}/g)
const trackNumber = numbersinpath && numbersinpath.length ? parseInt(numbersinpath[0]) : null
return {
trackNumber,
discNumber
}
}
getAverageScanDurationMs(results) {
if (!results.length) return 0
let total = 0
for (let i = 0; i < results.length; i++) total += results[i].elapsed
return Math.floor(total / results.length)
}
async scan(mediaType, libraryFile, mediaMetadataFromScan, verbose = false) {
const probeStart = Date.now()
const probeData = await prober.probe(libraryFile.metadata.path, verbose)
if (probeData.error) {
Logger.error(`[MediaFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`)
return null
}
if (mediaType === 'video') {
if (!probeData.videoStream) {
Logger.error('[MediaFileScanner] Invalid video file no video stream')
return null
}
const videoFile = new VideoFile()
videoFile.setDataFromProbe(libraryFile, probeData)
return {
videoFile,
elapsed: Date.now() - probeStart
}
} else {
if (!probeData.audioStream) {
Logger.error('[MediaFileScanner] Invalid audio file no audio stream')
return null
}
const audioFile = new AudioFile()
audioFile.trackNumFromMeta = probeData.audioMetaTags.trackNumber
audioFile.discNumFromMeta = probeData.audioMetaTags.discNumber
if (mediaType === 'book') {
const { trackNumber, discNumber } = this.getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, libraryFile)
audioFile.trackNumFromFilename = trackNumber
audioFile.discNumFromFilename = discNumber
}
audioFile.setDataFromProbe(libraryFile, probeData)
return {
audioFile,
elapsed: Date.now() - probeStart
}
}
}
/**
* Returns array of { MediaFile, elapsed, averageScanDuration } from audio file scan objects
* @param {import('../objects/LibraryItem')} libraryItem
* @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles
* @returns {Promise<object>}
*/
async executeMediaFileScans(libraryItem, mediaLibraryFiles) {
const mediaType = libraryItem.mediaType
const scanStart = Date.now()
const mediaMetadata = libraryItem.media.metadata || null
const batchSize = 32
const results = []
for (let batch = 0; batch < mediaLibraryFiles.length; batch += batchSize) {
const proms = []
for (let i = batch; i < Math.min(batch + batchSize, mediaLibraryFiles.length); i++) {
proms.push(this.scan(mediaType, mediaLibraryFiles[i], mediaMetadata))
}
results.push(...await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)))
}
return {
audioFiles: results.filter(r => r.audioFile).map(r => r.audioFile),
videoFiles: results.filter(r => r.videoFile).map(r => r.videoFile),
elapsed: Date.now() - scanStart,
averageScanDuration: this.getAverageScanDurationMs(results)
}
}
isSequential(nums) {
if (!nums || !nums.length) return false
if (nums.length === 1) return true
let prev = nums[0]
for (let i = 1; i < nums.length; i++) {
if (nums[i] - prev > 1) return false
prev = nums[i]
}
return true
}
removeDupes(nums) {
if (!nums || !nums.length) return []
if (nums.length === 1) return nums
var nodupes = [nums[0]]
nums.forEach((num) => {
if (num > nodupes[nodupes.length - 1]) nodupes.push(num)
})
return nodupes
}
runSmartTrackOrder(libraryItem, audioFiles) {
var discsFromFilename = []
var tracksFromFilename = []
var discsFromMeta = []
var tracksFromMeta = []
audioFiles.forEach((af) => {
if (af.discNumFromFilename !== null) discsFromFilename.push(af.discNumFromFilename)
if (af.discNumFromMeta !== null) discsFromMeta.push(af.discNumFromMeta)
if (af.trackNumFromFilename !== null) tracksFromFilename.push(af.trackNumFromFilename)
if (af.trackNumFromMeta !== null) tracksFromMeta.push(af.trackNumFromMeta)
})
discsFromFilename.sort((a, b) => a - b)
discsFromMeta.sort((a, b) => a - b)
tracksFromFilename.sort((a, b) => a - b)
tracksFromMeta.sort((a, b) => a - b)
var discKey = null
if (discsFromMeta.length === audioFiles.length && this.isSequential(discsFromMeta)) {
discKey = 'discNumFromMeta'
} else if (discsFromFilename.length === audioFiles.length && this.isSequential(discsFromFilename)) {
discKey = 'discNumFromFilename'
}
var trackKey = null
tracksFromFilename = this.removeDupes(tracksFromFilename)
tracksFromMeta = this.removeDupes(tracksFromMeta)
if (tracksFromFilename.length > tracksFromMeta.length) {
trackKey = 'trackNumFromFilename'
} else {
trackKey = 'trackNumFromMeta'
}
if (discKey !== null) {
Logger.debug(`[MediaFileScanner] Smart track order for "${libraryItem.media.metadata.title}" using disc key ${discKey} and track key ${trackKey}`)
audioFiles.sort((a, b) => {
let Dx = a[discKey] - b[discKey]
if (Dx === 0) Dx = a[trackKey] - b[trackKey]
return Dx
})
} else {
Logger.debug(`[MediaFileScanner] Smart track order for "${libraryItem.media.metadata.title}" using track key ${trackKey}`)
audioFiles.sort((a, b) => a[trackKey] - b[trackKey])
}
for (let i = 0; i < audioFiles.length; i++) {
audioFiles[i].index = i + 1
var existingAF = libraryItem.media.findFileWithInode(audioFiles[i].ino)
if (existingAF) {
if (existingAF.updateFromScan) existingAF.updateFromScan(audioFiles[i])
} else {
libraryItem.media.addAudioFile(audioFiles[i])
}
}
}
/**
* Scans media files for a library item and adds them as audio tracks and sets library item metadata
* @param {import('../objects/files/LibraryFile')[]} mediaLibraryFiles
* @param {import('../objects/LibraryItem')} libraryItem
* @param {import('./LibraryScan')} [libraryScan=null] - Optional when doing a library scan to use LibraryScan config/logs
* @return {Promise<boolean>} True if any updates were made
*/
async scanMediaFiles(mediaLibraryFiles, libraryItem, libraryScan = null) {
const preferAudioMetadata = libraryScan ? !!libraryScan.preferAudioMetadata : !!global.ServerSettings.scannerPreferAudioMetadata
let hasUpdated = false
const mediaScanResult = await this.executeMediaFileScans(libraryItem, mediaLibraryFiles)
if (libraryItem.mediaType === 'video') {
if (mediaScanResult.videoFiles.length) {
// TODO: Check for updates etc
hasUpdated = true
libraryItem.media.setVideoFile(mediaScanResult.videoFiles[0])
}
} else if (mediaScanResult.audioFiles.length) {
if (libraryScan) {
libraryScan.addLog(LogLevel.DEBUG, `Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms per MB`)
}
Logger.debug(`Library Item "${libraryItem.path}" Media file scan took ${mediaScanResult.elapsed}ms with ${mediaScanResult.audioFiles.length} audio files averaging ${mediaScanResult.averageScanDuration}ms per MB`)
const newAudioFiles = mediaScanResult.audioFiles.filter(af => {
return !libraryItem.media.findFileWithInode(af.ino)
})
// Book: Adding audio files to book media
if (libraryItem.mediaType === 'book') {
const mediaScanFileInodes = mediaScanResult.audioFiles.map(af => af.ino)
// Filter for existing valid track audio files not included in the audio files scanned
const existingAudioFiles = libraryItem.media.audioFiles.filter(af => af.isValidTrack && !mediaScanFileInodes.includes(af.ino))
if (newAudioFiles.length) {
// Single Track Audiobooks
if (mediaScanFileInodes.length + existingAudioFiles.length === 1) {
const af = mediaScanResult.audioFiles[0]
af.index = 1
libraryItem.media.addAudioFile(af)
hasUpdated = true
} else {
const allAudioFiles = existingAudioFiles.concat(mediaScanResult.audioFiles)
this.runSmartTrackOrder(libraryItem, allAudioFiles)
hasUpdated = true
}
} else {
// Only update metadata not index
mediaScanResult.audioFiles.forEach((af) => {
const existingAF = libraryItem.media.findFileWithInode(af.ino)
if (existingAF) {
af.index = existingAF.index
if (existingAF.updateFromScan && existingAF.updateFromScan(af)) {
hasUpdated = true
}
}
})
}
// Set book details from audio file ID3 tags, optional prefer
if (libraryItem.media.setMetadataFromAudioFile(preferAudioMetadata)) {
hasUpdated = true
}
if (hasUpdated) {
libraryItem.media.rebuildTracks()
}
} else if (libraryItem.mediaType === 'podcast') { // Podcast Media Type
const existingAudioFiles = mediaScanResult.audioFiles.filter(af => libraryItem.media.findFileWithInode(af.ino))
if (newAudioFiles.length) {
let newIndex = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1
newAudioFiles.forEach((newAudioFile) => {
libraryItem.media.addNewEpisodeFromAudioFile(newAudioFile, newIndex++)
})
hasUpdated = true
}
// Update audio file metadata for audio files already there
existingAudioFiles.forEach((af) => {
const podcastEpisode = libraryItem.media.findEpisodeWithInode(af.ino)
af.index = 1
if (podcastEpisode?.audioFile.updateFromScan(af)) {
hasUpdated = true
podcastEpisode.setDataFromAudioMetaTags(podcastEpisode.audioFile.metaTags, false)
}
})
if (libraryItem.media.setMetadataFromAudioFile(preferAudioMetadata)) {
hasUpdated = true
}
} else if (libraryItem.mediaType === 'music') { // Music
// Only one audio file in library item
if (newAudioFiles.length) { // New audio file
libraryItem.media.setAudioFile(newAudioFiles[0])
hasUpdated = true
} else if (libraryItem.media.audioFile && libraryItem.media.audioFile.updateFromScan(mediaScanResult.audioFiles[0])) {
hasUpdated = true
console.log('Updated from scan')
}
if (libraryItem.media.setMetadataFromAudioFile()) {
hasUpdated = true
}
// If the audio track has no title meta tag then use the audio file name
if (!libraryItem.media.metadata.title && libraryItem.media.audioFile) {
const audioFileName = libraryItem.media.audioFile.metadata.filename
libraryItem.media.metadata.title = Path.basename(audioFileName, Path.extname(audioFileName))
hasUpdated = true
}
}
}
return hasUpdated
}
probeAudioFile(audioFile) {
Logger.debug(`[MediaFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`)
return prober.rawProbe(audioFile.metadata.path)
}
}
module.exports = new MediaFileScanner()

View File

@ -490,20 +490,31 @@ class PodcastScanner {
return fsExtra.writeFile(metadataFilePath, JSON.stringify(jsonObject, null, 2)).then(async () => {
// Add metadata.json to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.json`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
if (libraryItemDirTimestamps) {
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
let size = 0
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.size = size
}
}
libraryScan.addLog(LogLevel.DEBUG, `Success saving abmetadata to "${metadataFilePath}"`)
return metadataLibraryFile
@ -526,18 +537,28 @@ class PodcastScanner {
}
// Add metadata.abs to libraryFiles array if it is new
let metadataLibraryFile = libraryItem.libraryFiles.find(lf => lf.metadata.path === filePathToPOSIX(metadataFilePath))
if (storeMetadataWithItem && !metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else if (storeMetadataWithItem) {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
if (storeMetadataWithItem) {
if (!metadataLibraryFile) {
const newLibraryFile = new LibraryFile()
await newLibraryFile.setDataFromPath(metadataFilePath, `metadata.abs`)
metadataLibraryFile = newLibraryFile.toJSON()
libraryItem.libraryFiles.push(metadataLibraryFile)
} else {
const fileTimestamps = await getFileTimestampsWithIno(metadataFilePath)
if (fileTimestamps) {
metadataLibraryFile.metadata.mtimeMs = fileTimestamps.mtimeMs
metadataLibraryFile.metadata.ctimeMs = fileTimestamps.ctimeMs
metadataLibraryFile.metadata.size = fileTimestamps.size
metadataLibraryFile.ino = fileTimestamps.ino
}
}
const libraryItemDirTimestamps = await getFileTimestampsWithIno(libraryItem.path)
if (libraryItemDirTimestamps) {
libraryItem.mtime = libraryItemDirTimestamps.mtimeMs
libraryItem.ctime = libraryItemDirTimestamps.ctimeMs
let size = 0
libraryItem.libraryFiles.forEach((lf) => size += (!isNaN(lf.metadata.size) ? Number(lf.metadata.size) : 0))
libraryItem.size = size
}
}

View File

@ -1,40 +0,0 @@
class ScanOptions {
constructor() {
this.forceRescan = false
// Server settings
this.parseSubtitles = false
this.findCovers = false
this.storeCoverWithItem = false
this.preferAudioMetadata = false
this.preferOpfMetadata = false
this.preferMatchedMetadata = false
this.preferOverdriveMediaMarker = false
}
toJSON() {
return {
forceRescan: this.forceRescan,
parseSubtitles: this.parseSubtitles,
findCovers: this.findCovers,
storeCoverWithItem: this.storeCoverWithItem,
preferAudioMetadata: this.preferAudioMetadata,
preferOpfMetadata: this.preferOpfMetadata,
preferMatchedMetadata: this.preferMatchedMetadata,
preferOverdriveMediaMarker: this.preferOverdriveMediaMarker
}
}
setData(options, serverSettings) {
this.forceRescan = !!options.forceRescan
this.parseSubtitles = !!serverSettings.scannerParseSubtitle
this.findCovers = !!serverSettings.scannerFindCovers
this.storeCoverWithItem = serverSettings.storeCoverWithItem
this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata
this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata
this.scannerPreferMatchedMetadata = serverSettings.scannerPreferMatchedMetadata
this.preferOverdriveMediaMarker = serverSettings.scannerPreferOverdriveMediaMarker
}
}
module.exports = ScanOptions

View File

@ -6,7 +6,6 @@ const Database = require('../Database')
const { LogLevel } = require('../utils/constants')
const { findMatchingEpisodesInFeed, getPodcastFeed } = require('../utils/podcastUtils')
const MediaFileScanner = require('./MediaFileScanner')
const BookFinder = require('../finders/BookFinder')
const PodcastFinder = require('../finders/PodcastFinder')
const LibraryScan = require('./LibraryScan')
@ -374,9 +373,5 @@ class Scanner {
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
SocketAuthority.emitter('scan_complete', libraryScan.getScanEmitData)
}
probeAudioFile(audioFile) {
return MediaFileScanner.probeAudioFile(audioFile)
}
}
module.exports = Scanner