Add:New scanner and scanner server settings

This commit is contained in:
advplyr 2021-11-25 18:39:02 -06:00
parent bf11d266dc
commit a5fc382cad
17 changed files with 681 additions and 176 deletions

View File

@ -104,11 +104,23 @@ export default {
if (payload.serverSettings) {
this.$store.commit('setServerSettings', payload.serverSettings)
}
// Start scans currently running
if (payload.librariesScanning) {
payload.librariesScanning.forEach((libraryScan) => {
this.scanStart(libraryScan)
})
}
// Remove any current scans that are no longer running
var currentScans = [...this.$store.state.scanners.libraryScans]
currentScans.forEach((ls) => {
if (!payload.librariesScanning || !payload.librariesScanning.find((_ls) => _ls.id === ls.id)) {
this.$toast.dismiss(ls.toastId)
this.$store.commit('scanners/remove', ls)
}
})
if (payload.backups && payload.backups.length) {
this.$store.commit('setBackups', payload.backups)
}
@ -152,6 +164,16 @@ export default {
}
this.$store.commit('audiobooks/remove', audiobook)
},
audiobooksAdded(audiobooks) {
audiobooks.forEach((ab) => {
this.$store.commit('audiobooks/addUpdate', ab)
})
},
audiobooksUpdated(audiobooks) {
audiobooks.forEach((ab) => {
this.$store.commit('audiobooks/addUpdate', ab)
})
},
libraryAdded(library) {
this.$store.commit('libraries/addUpdate', library)
},
@ -162,6 +184,8 @@ export default {
this.$store.commit('libraries/remove', library)
},
scanComplete(data) {
console.log('Scan complete received', data)
var message = `Scan "${data.name}" complete!`
if (data.results) {
var scanResultMsgs = []
@ -337,6 +361,8 @@ export default {
this.socket.on('audiobook_updated', this.audiobookUpdated)
this.socket.on('audiobook_added', this.audiobookAdded)
this.socket.on('audiobook_removed', this.audiobookRemoved)
this.socket.on('audiobooks_updated', this.audiobooksUpdated)
this.socket.on('audiobooks_added', this.audiobooksAdded)
// Library Listeners
this.socket.on('library_updated', this.libraryUpdated)

View File

@ -21,6 +21,20 @@
</ui-tooltip>
</div>
<div class="flex items-center py-2">
<ui-toggle-switch v-model="newServerSettings.scannerPreferAudioMetadata" :disabled="updatingServerSettings" @input="updateScannerPreferAudioMeta" />
<ui-tooltip :text="scannerPreferAudioMetaTooltip">
<p class="pl-4 text-lg">Scanner prefer audio metadata <span class="material-icons icon-text">info_outlined</span></p>
</ui-tooltip>
</div>
<div class="flex items-center py-2">
<ui-toggle-switch v-model="newServerSettings.scannerPreferOpfMetadata" :disabled="updatingServerSettings" @input="updateScannerPreferOpfMeta" />
<ui-tooltip :text="scannerPreferOpfMetaTooltip">
<p class="pl-4 text-lg">Scanner prefer OPF metadata <span class="material-icons icon-text">info_outlined</span></p>
</ui-tooltip>
</div>
<div class="flex items-center py-2">
<ui-toggle-switch v-model="storeCoversInAudiobookDir" :disabled="updatingServerSettings" @input="updateCoverStorageDestination" />
<ui-tooltip :text="coverDestinationTooltip">
@ -83,6 +97,12 @@ export default {
}
},
computed: {
scannerPreferAudioMetaTooltip() {
return 'Audio file ID3 meta tags will be used for book details over folder & filenames'
},
scannerPreferOpfMetaTooltip() {
return 'OPF file metadata will be used for book details over folder & filenames'
},
saveMetadataTooltip() {
return 'This will write a "metadata.nfo" file in all of your audiobook directories.'
},
@ -127,6 +147,16 @@ export default {
scannerParseSubtitle: !!val
})
},
updateScannerPreferAudioMeta(val) {
this.updateServerSettings({
scannerPreferAudioMetadata: !!val
})
},
updateScannerPreferOpfMeta(val) {
this.updateServerSettings({
scannerPreferOpfMetadata: !!val
})
},
updateServerSettings(payload) {
this.updatingServerSettings = true
this.$store
@ -144,7 +174,6 @@ export default {
this.newServerSettings = this.serverSettings ? { ...this.serverSettings } : {}
this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
},
resetAudiobooks() {
if (confirm('WARNING! This action will remove all audiobooks from the database including any updates or matches you have made. This does not do anything to your actual files. Shall we continue?')) {

View File

@ -196,7 +196,7 @@ class Scanner {
// Sync other files (all files that are not audio files) - Updates cover path
var hasOtherFileUpdates = false
var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, forceAudioFileScan)
var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, false, forceAudioFileScan)
if (otherFilesUpdated) {
hasOtherFileUpdates = true
}
@ -250,7 +250,7 @@ class Scanner {
var hasUpdates = hasOtherFileUpdates || hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles
// Check that audio tracks are in sequential order with no gaps
if (existingAudiobook.checkUpdateMissingParts()) {
if (existingAudiobook.checkUpdateMissingTracks()) {
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
hasUpdates = true
}
@ -299,7 +299,7 @@ class Scanner {
}
// Look for desc.txt and reader.txt and update
await audiobook.saveDataFromTextFiles()
await audiobook.saveDataFromTextFiles(false)
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
@ -314,7 +314,7 @@ class Scanner {
audiobook.setDetailsFromFileMetadata()
// Check for gaps in track numbers
audiobook.checkUpdateMissingParts()
audiobook.checkUpdateMissingTracks()
// Set chapters from audio files
audiobook.setChapters()
@ -671,11 +671,6 @@ class Scanner {
var folder = library.getFolderById(folderId)
if (!folder) {
Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
Logger.debug(`Looking at folders in library "${library.name}" for folderid ${folderId}`)
library.folders.forEach((fold) => {
Logger.debug(`Folder "${fold.id}" "${fold.fullPath}"`)
})
continue;
}

View File

@ -311,19 +311,18 @@ class Server {
async filesChanged(fileUpdates) {
Logger.info('[Server]', fileUpdates.length, 'Files Changed')
await this.scanner.filesChanged(fileUpdates)
// Logger.debug('[Server] Files changed result', result)
await this.scanner2.scanFilesChanged(fileUpdates)
}
async scan(libraryId, forceAudioFileScan = false) {
async scan(libraryId) {
Logger.info('[Server] Starting Scan')
// await this.scanner2.scan(libraryId)
await this.scanner(libraryId, forceAudioFileScan)
await this.scanner2.scan(libraryId)
// await this.scanner.scan(libraryId)
Logger.info('[Server] Scan complete')
}
async scanAudiobook(socket, audiobookId) {
var result = await this.scanner.scanAudiobookById(audiobookId)
var result = await this.scanner2.scanAudiobookById(audiobookId)
var scanResultName = ''
for (const key in ScanResult) {
if (ScanResult[key] === result) {
@ -335,7 +334,7 @@ class Server {
cancelScan(id) {
Logger.debug('[Server] Cancel scan', id)
this.scanner.cancelLibraryScan[id] = true
this.scanner2.cancelLibraryScan[id] = true
}
// Generates an NFO metadata file, if no audiobookId is passed then all audiobooks are done
@ -624,7 +623,7 @@ class Server {
configPath: this.ConfigPath,
user: client.user.toJSONForBrowser(),
stream: client.stream || null,
librariesScanning: this.scanner.librariesScanning,
librariesScanning: this.scanner2.librariesScanning,
backups: (this.backupManager.backups || []).map(b => b.toJSON())
}
if (user.type === 'root') {

View File

@ -24,8 +24,11 @@ class FolderWatcher extends EventEmitter {
Logger.warn('[Watcher] Already watching library', library.name)
return
}
Logger.info(`[Watcher] Initializing watcher for "${library.name}"..`)
Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
var folderPaths = library.folderPaths
folderPaths.forEach((fp) => {
Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
})
var watcher = new Watcher(folderPaths, {
ignored: /(^|[\/\\])\../, // ignore dotfiles
renameDetection: true,
@ -48,6 +51,8 @@ class FolderWatcher extends EventEmitter {
Logger.error(`[Watcher] ${error}`)
}).on('ready', () => {
Logger.info(`[Watcher] "${library.name}" Ready`)
}).on('close', () => {
Logger.debug(`[Watcher] "${library.name}" Closed`)
})
this.libraryWatchers.push({

View File

@ -165,9 +165,9 @@ class AudioFile {
this.fullPath = fileData.fullPath
this.addedAt = Date.now()
this.trackNumFromMeta = fileData.trackNumFromMeta || null
this.trackNumFromFilename = fileData.trackNumFromFilename || null
this.cdNumFromFilename = fileData.cdNumFromFilename || null
this.trackNumFromMeta = fileData.trackNumFromMeta
this.trackNumFromFilename = fileData.trackNumFromFilename
this.cdNumFromFilename = fileData.cdNumFromFilename
this.format = probeData.format
this.duration = probeData.duration
@ -180,15 +180,13 @@ class AudioFile {
this.channelLayout = probeData.channelLayout
this.chapters = probeData.chapters || []
this.metadata = probeData.audioFileMetadata
this.embeddedCoverArt = probeData.embeddedCoverArt
}
validateTrackIndex(isSingleTrack) {
validateTrackIndex() {
var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta)
var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename)
if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1
return numFromMeta ? numFromMeta : 1
}
if (numFromMeta !== null) return numFromMeta
if (numFromFilename !== null) return numFromFilename
@ -284,5 +282,33 @@ class AudioFile {
})
return hasUpdates
}
updateFromScan(scannedAudioFile) {
var hasUpdated = false
var newjson = scannedAudioFile.toJSON()
if (this.manuallyVerified) newjson.manuallyVerified = true
if (this.exclude) newjson.exclude = true
newjson.addedAt = this.addedAt
for (const key in newjson) {
if (key === 'metadata') {
if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) {
this.metadata = scannedAudioFile.metadata
hasUpdated = true
// console.log('metadata updated for audio file')
}
} else if (key === 'chapters') {
if (this.syncChapters(newjson.chapters || [])) {
hasUpdated = true
}
} else if (this[key] !== newjson[key]) {
this[key] = newjson[key]
hasUpdated = true
// console.log('key', key, 'updated', this[key], newjson[key])
}
}
return hasUpdated
}
}
module.exports = AudioFile

View File

@ -101,5 +101,13 @@ class AudioFileMetadata {
}
return hasUpdates
}
isEqual(audioFileMetadata) {
if (!audioFileMetadata || !audioFileMetadata.toJSON) return false
for (const key in audioFileMetadata.toJSON()) {
if (audioFileMetadata[key] !== this[key]) return false
}
return true
}
}
module.exports = AudioFileMetadata

View File

@ -353,6 +353,11 @@ class Audiobook {
this.lastUpdate = Date.now()
}
setInvalid() {
this.isInvalid = true
this.lastUpdate = Date.now()
}
setBook(data) {
// Use first image file as cover
if (this.otherFiles && this.otherFiles.length) {
@ -400,6 +405,11 @@ class Audiobook {
}
}
updateAudioFile(updatedAudioFile) {
var audioFile = this.audioFiles.find(af => af.ino === updatedAudioFile.ino)
return audioFile.updateFromScan(updatedAudioFile)
}
addOtherFile(fileData) {
var file = new AudiobookFile()
file.setData(fileData)
@ -437,8 +447,8 @@ class Audiobook {
return this.book.updateCover(cover, coverFullPath)
}
checkHasTrackNum(trackNum) {
return this.tracks.find(t => t.index === trackNum)
checkHasTrackNum(trackNum, excludeIno) {
return this._audioFiles.find(t => t.index === trackNum && t.ino !== excludeIno)
}
updateAudioTracks(orderedFileData) {
@ -473,6 +483,7 @@ class Audiobook {
}
})
this.setChapters()
this.checkUpdateMissingTracks()
this.lastUpdate = Date.now()
}
@ -486,7 +497,7 @@ class Audiobook {
this.audioFiles = this.audioFiles.filter(f => f.ino !== track.ino)
}
checkUpdateMissingParts() {
checkUpdateMissingTracks() {
var currMissingParts = (this.missingParts || []).join(',') || ''
var current_index = 1
@ -515,13 +526,14 @@ class Audiobook {
}
// On scan check other files found with other files saved
async syncOtherFiles(newOtherFiles, metadataPath, forceRescan = false) {
async syncOtherFiles(newOtherFiles, metadataPath, opfMetadataOverrideDetails, forceRescan = false) {
var hasUpdates = false
var currOtherFileNum = this.otherFiles.length
var alreadyHasDescTxt = this.otherFiles.find(of => of.filename === 'desc.txt')
var alreadyHasReaderTxt = this.otherFiles.find(of => of.filename === 'reader.txt')
var otherFilenamesAlreadyInBook = this.otherFiles.map(ofile => ofile.filename)
var alreadyHasDescTxt = otherFilenamesAlreadyInBook.includes('desc.txt')
var alreadyHasReaderTxt = otherFilenamesAlreadyInBook.includes('reader.txt')
var newOtherFilePaths = newOtherFiles.map(f => f.path)
this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
@ -553,21 +565,22 @@ class Audiobook {
}
}
// If OPF file and was not already there
var metadataOpf = newOtherFiles.find(file => file.ext === '.opf' || file.filename === 'metadata.xml')
if (metadataOpf) {
if (metadataOpf && (!otherFilenamesAlreadyInBook.includes(metadataOpf.filename) || opfMetadataOverrideDetails)) {
var xmlText = await readTextFile(metadataOpf.fullPath)
if (xmlText) {
var opfMetadata = await parseOpfMetadataXML(xmlText)
Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
// Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
if (opfMetadata) {
const bookUpdatePayload = {}
for (const key in opfMetadata) {
// Add genres only if genres are empty
if (key === 'genres') {
if (opfMetadata.genres.length && !this.book._genres.length) {
if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata.genres
}
} else if (opfMetadata[key] && !this.book[key]) {
} else if (opfMetadata[key] && (!this.book[key] || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata[key]
}
}
@ -789,7 +802,7 @@ class Audiobook {
}
// Look for desc.txt and reader.txt and update details if found
async saveDataFromTextFiles() {
async saveDataFromTextFiles(opfMetadataOverrideDetails) {
var bookUpdatePayload = {}
var descriptionText = await this.fetchTextFromTextFile('desc.txt')
if (descriptionText) {
@ -807,15 +820,15 @@ class Audiobook {
var xmlText = await readTextFile(metadataOpf.fullPath)
if (xmlText) {
var opfMetadata = await parseOpfMetadataXML(xmlText)
Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
// Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
if (opfMetadata) {
for (const key in opfMetadata) {
// Add genres only if genres are empty
if (key === 'genres') {
if (opfMetadata.genres.length && !this.book._genres.length) {
if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata.genres
}
} else if (opfMetadata[key] && !this.book[key] && !bookUpdatePayload[key]) {
} else if (opfMetadata[key] && ((!this.book[key] && !bookUpdatePayload[key]) || opfMetadataOverrideDetails)) {
bookUpdatePayload[key] = opfMetadata[key]
}
}
@ -836,10 +849,10 @@ class Audiobook {
}
// Audio file metadata tags map to book details (will not overwrite)
setDetailsFromFileMetadata() {
setDetailsFromFileMetadata(overrideExistingDetails = false) {
if (!this.audioFiles.length) return false
var audioFile = this.audioFiles[0]
return this.book.setDetailsFromFileMetadata(audioFile.metadata)
return this.book.setDetailsFromFileMetadata(audioFile.metadata, overrideExistingDetails)
}
// Returns null if file not found, true if file was updated, false if up to date
@ -884,9 +897,15 @@ class Audiobook {
return hasUpdated
}
checkScanData(dataFound) {
checkScanData(dataFound, version) {
var hasUpdated = false
if (this.isMissing) {
// Audiobook no longer missing
this.isMissing = false
hasUpdated = true
}
if (dataFound.ino !== this.ino) {
this.ino = dataFound.ino
hasUpdated = true
@ -916,7 +935,7 @@ class Audiobook {
var audioFileFoundCheck = this.checkFileFound(af, true)
if (audioFileFoundCheck === null) {
newAudioFileData.push(af)
} else if (audioFileFoundCheck === true) {
} else if (audioFileFoundCheck) {
hasUpdated = true
}
})
@ -925,7 +944,7 @@ class Audiobook {
var fileFoundCheck = this.checkFileFound(otherFileData, false)
if (fileFoundCheck === null) {
newOtherFileData.push(otherFileData)
} else if (fileFoundCheck === true) {
} else if (fileFoundCheck) {
hasUpdated = true
}
})
@ -933,7 +952,7 @@ class Audiobook {
const audioFilesRemoved = []
const otherFilesRemoved = []
// inodes will all be up to date at this point
// Remove audio files not found (inodes will all be up to date at this point)
this.audioFiles = this.audioFiles.filter(af => {
if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) {
audioFilesRemoved.push(af.toJSON())
@ -946,10 +965,11 @@ class Audiobook {
if (audioFilesRemoved.length) {
const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino)
this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino))
this.checkUpdateMissingParts()
this.checkUpdateMissingTracks()
hasUpdated = true
}
// Remove other files not found
this.otherFiles = this.otherFiles.filter(otherFile => {
if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) {
otherFilesRemoved.push(otherFile.toJSON())
@ -969,6 +989,15 @@ class Audiobook {
hasUpdated = true
}
// Check if invalid (has no audio files or ebooks)
if (!this.audioFilesToInclude.length && !this.ebooks.length && !newAudioFileData.length && !newOtherFileData.length) {
this.isInvalid = true
}
if (hasUpdated) {
this.setLastScan(version)
}
return {
updated: hasUpdated,
newAudioFileData,

View File

@ -43,6 +43,7 @@ class Book {
get _genres() { return this.genres || [] }
get shouldSearchForCover() {
if (this.cover) return false
if (this.authorFL !== this.lastCoverSearchAuthor || this.title !== this.lastCoverSearchTitle || !this.lastCoverSearch) return true
var timeSinceLastSearch = Date.now() - this.lastCoverSearch
return timeSinceLastSearch > 1000 * 60 * 60 * 24 * 7 // every 7 days do another lookup
@ -297,7 +298,7 @@ class Book {
return [genreTag]
}
setDetailsFromFileMetadata(audioFileMetadata) {
setDetailsFromFileMetadata(audioFileMetadata, overrideExistingDetails = false) {
const MetadataMapArray = [
{
tag: 'tagComposer',
@ -319,6 +320,10 @@ class Book {
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagAlbum',
key: 'title',
},
{
tag: 'tagArtist',
key: 'author'
@ -342,12 +347,12 @@ class Book {
MetadataMapArray.forEach((mapping) => {
if (audioFileMetadata[mapping.tag]) {
// Genres can contain multiple
if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key])) {
if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key] || overrideExistingDetails)) {
updatePayload[mapping.key] = this.parseGenresTag(audioFileMetadata[mapping.tag])
Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
} else if (!this[mapping.key]) {
// Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
} else if (!this[mapping.key] || overrideExistingDetails) {
updatePayload[mapping.key] = audioFileMetadata[mapping.tag]
Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
// Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
}
}
})

View File

@ -46,6 +46,9 @@ class ServerSettings {
this.newTagExpireDays = settings.newTagExpireDays
this.scannerFindCovers = !!settings.scannerFindCovers
this.scannerParseSubtitle = settings.scannerParseSubtitle
this.scannerPreferAudioMetadata = !!settings.scannerPreferAudioMetadata
this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata
this.coverDestination = settings.coverDestination || CoverDestination.METADATA
this.saveMetadataFile = !!settings.saveMetadataFile
this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10
@ -73,6 +76,8 @@ class ServerSettings {
newTagExpireDays: this.newTagExpireDays,
scannerFindCovers: this.scannerFindCovers,
scannerParseSubtitle: this.scannerParseSubtitle,
scannerPreferAudioMetadata: this.scannerPreferAudioMetadata,
scannerPreferOpfMetadata: this.scannerPreferOpfMetadata,
coverDestination: this.coverDestination,
saveMetadataFile: !!this.saveMetadataFile,
rateLimitLoginRequests: this.rateLimitLoginRequests,

View File

@ -4,7 +4,7 @@ const AudioFile = require('../objects/AudioFile')
const prober = require('../utils/prober')
const Logger = require('../Logger')
const { msToTimestamp } = require('../utils')
const { LogLevel } = require('../utils/constants')
class AudioFileScanner {
constructor() { }
@ -80,6 +80,9 @@ class AudioFileScanner {
audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
audioFile.setDataFromProbe(audioFileData, probeData)
if (audioFile.embeddedCoverArt) {
}
return {
audioFile,
elapsed: Date.now() - probeStart
@ -87,12 +90,11 @@ class AudioFileScanner {
}
// Returns array of { AudioFile, elapsed } from audio file scan objects
async scanAudioFiles(audioFileDataArray, bookScanData) {
// Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects
async executeAudioFileScans(audioFileDataArray, bookScanData) {
var proms = []
for (let i = 0; i < audioFileDataArray.length; i++) {
var prom = this.scan(audioFileDataArray[i], bookScanData)
proms.push(prom)
proms.push(this.scan(audioFileDataArray[i], bookScanData))
}
var scanStart = Date.now()
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
@ -102,5 +104,62 @@ class AudioFileScanner {
averageScanDuration: this.getAverageScanDurationMs(results)
}
}
async scanAudioFiles(audioFileDataArray, bookScanData, audiobook, preferAudioMetadata, libraryScan = null) {
var hasUpdated = false
var audioScanResult = await this.executeAudioFileScans(audioFileDataArray, bookScanData)
if (audioScanResult.audioFiles.length) {
if (libraryScan) {
libraryScan.addLog(LogLevel.DEBUG, `Book "${bookScanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`)
}
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.filter(af => !audioScanResult.audioFiles.find(_af => _af.ino === af.ino)).length + audioScanResult.audioFiles.length
// validate & add/update audio files to audiobook
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
var newAF = audioScanResult.audioFiles[i]
var existingAF = audiobook.getAudioFileByIno(newAF.ino)
var trackIndex = null
if (totalAudioFilesToInclude === 1) { // Single track audiobooks
trackIndex = 1
} else if (existingAF && existingAF.manuallyVerified) { // manually verified audio files use existing index
trackIndex = existingAF.index
} else {
trackIndex = newAF.validateTrackIndex()
}
if (trackIndex !== null) {
if (audiobook.checkHasTrackNum(trackIndex, newAF.ino)) {
newAF.setDuplicateTrackNumber(trackIndex)
} else {
newAF.index = trackIndex
}
}
if (existingAF) {
if (audiobook.updateAudioFile(newAF)) {
// console.log('update dauido file')
hasUpdated = true
}
} else {
audiobook.addAudioFile(newAF)
// console.log('added auido file')
hasUpdated = true
}
}
if (hasUpdated) {
audiobook.rebuildTracks()
}
// Set book details from audio file ID3 tags, optional prefer
if (audiobook.setDetailsFromFileMetadata(preferAudioMetadata)) {
hasUpdated = true
}
}
return hasUpdated
}
}
module.exports = new AudioFileScanner()

View File

@ -35,7 +35,6 @@ class AudioProbeData {
setData(data) {
var audioStream = this.getDefaultAudioStream(data.audio_streams)
this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
this.format = data.format
this.duration = data.duration

View File

@ -1,6 +1,10 @@
const Folder = require('../objects/Folder')
const Constants = require('../utils/constants')
const Path = require('path')
const fs = require('fs-extra')
const date = require('date-and-time')
const Logger = require('../Logger')
const Folder = require('../objects/Folder')
const { LogLevel } = require('../utils/constants')
const { getId, secondsToTimestamp } = require('../utils/index')
class LibraryScan {
@ -9,6 +13,7 @@ class LibraryScan {
this.libraryId = null
this.libraryName = null
this.folders = null
this.verbose = false
this.scanOptions = null
@ -16,14 +21,21 @@ class LibraryScan {
this.finishedAt = null
this.elapsed = null
this.status = Constants.ScanStatus.NOTHING
this.resultsMissing = 0
this.resultsAdded = 0
this.resultsUpdated = 0
this.logs = []
}
get _scanOptions() { return this.scanOptions || {} }
get forceRescan() { return !!this._scanOptions.forceRescan }
get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
get findCovers() { return !!this._scanOptions.findCovers }
get timestamp() {
return (new Date()).toISOString()
}
get resultStats() {
return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
@ -42,6 +54,28 @@ class LibraryScan {
}
}
}
get totalResults() {
return this.resultsAdded + this.resultsUpdated + this.resultsMissing
}
get getLogFilename() {
return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
}
toJSON() {
return {
id: this.id,
libraryId: this.libraryId,
libraryName: this.libraryName,
folders: this.folders.map(f => f.toJSON()),
scanOptions: this.scanOptions.toJSON(),
startedAt: this.startedAt,
finishedAt: this.finishedAt,
elapsed: this.elapsed,
resultsAdded: this.resultsAdded,
resultsUpdated: this.resultsUpdated,
resultsMissing: this.resultsMissing
}
}
setData(library, scanOptions) {
this.id = getId('lscan')
@ -58,5 +92,39 @@ class LibraryScan {
this.finishedAt = Date.now()
this.elapsed = this.finishedAt - this.startedAt
}
getLogLevelString(level) {
for (const key in LogLevel) {
if (LogLevel[key] === level) {
return key
}
}
return 'UNKNOWN'
}
addLog(level, ...args) {
const logObj = {
timestamp: this.timestamp,
message: args.join(' '),
levelName: this.getLogLevelString(level),
level
}
if (this.verbose) {
Logger.debug(`[LibraryScan] "${this.libraryName}":`, args)
}
this.logs.push(logObj)
}
async saveLog(logDir) {
await fs.ensureDir(logDir)
var outputPath = Path.join(logDir, this.getLogFilename)
var logLines = [JSON.stringify(this.toJSON())]
this.logs.forEach(l => {
logLines.push(JSON.stringify(l))
})
await fs.writeFile(outputPath, logLines.join('\n') + '\n')
Logger.info(`[LibraryScan] Scan log saved "${outputPath}"`)
}
}
module.exports = LibraryScan

View File

@ -4,29 +4,6 @@ class ScanOptions {
constructor(options) {
this.forceRescan = false
// this.metadataPrecedence = [
// {
// id: 'directory',
// include: true
// },
// {
// id: 'reader-desc-txt',
// include: true
// },
// {
// id: 'audio-file-metadata',
// include: true
// },
// {
// id: 'metadata-opf',
// include: true
// },
// {
// id: 'external-source',
// include: false
// }
// ]
// Server settings
this.parseSubtitles = false
this.findCovers = false

View File

@ -4,10 +4,9 @@ const Path = require('path')
// Utils
const Logger = require('../Logger')
const { version } = require('../../package.json')
const audioFileScanner = require('../utils/audioFileScanner')
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
const { ScanResult, CoverDestination } = require('../utils/constants')
const { ScanResult, CoverDestination, LogLevel } = require('../utils/constants')
const AudioFileScanner = require('./AudioFileScanner')
const BookFinder = require('../BookFinder')
@ -20,6 +19,8 @@ class Scanner {
this.AudiobookPath = AUDIOBOOK_PATH
this.MetadataPath = METADATA_PATH
this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
var LogDirPath = Path.join(this.MetadataPath, 'logs')
this.ScanLogPath = Path.join(LogDirPath, 'scans')
this.db = db
this.coverController = coverController
@ -46,8 +47,82 @@ class Scanner {
}
}
isLibraryScanning(libraryId) {
return this.librariesScanning.find(ls => ls.id === libraryId)
}
async scanAudiobookById(audiobookId) {
var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId)
if (!audiobook) {
Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`)
return ScanResult.NOTHING
}
const library = this.db.libraries.find(lib => lib.id === audiobook.libraryId)
if (!library) {
Logger.error(`[Scanner] Scan audiobook by id library not found "${audiobook.libraryId}"`)
return ScanResult.NOTHING
}
const folder = library.folders.find(f => f.id === audiobook.folderId)
if (!folder) {
Logger.error(`[Scanner] Scan audiobook by id folder not found "${audiobook.folderId}" in library "${library.name}"`)
return ScanResult.NOTHING
}
Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`)
return this.scanAudiobook(folder, audiobook)
}
async scanAudiobook(folder, audiobook) {
var audiobookData = await getAudiobookFileData(folder, audiobook.fullPath, this.db.serverSettings)
if (!audiobookData) {
return ScanResult.NOTHING
}
var hasUpdated = false
var checkRes = audiobook.checkScanData(audiobookData, version)
if (checkRes.updated) hasUpdated = true
// Sync other files first so that local images are used as cover art
// TODO: Cleanup other file sync
var allOtherFiles = checkRes.newOtherFileData.concat(audiobook._otherFiles)
if (await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, this.db.serverSettings.scannerPreferOpfMetadata)) {
hasUpdated = true
}
// Scan all audio files
if (audiobookData.audioFiles.length) {
if (await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, this.db.serverSettings.scannerPreferAudioMetadata)) {
hasUpdated = true
}
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
var outputCoverDirs = this.getCoverDirectory(audiobook)
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
if (relativeDir) {
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
hasUpdated = true
}
}
}
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
audiobook.setInvalid()
hasUpdated = true
} else if (audiobook.isInvalid) {
audiobook.isInvalid = false
hasUpdated = true
}
if (hasUpdated) {
this.emitter('audiobook_updated', audiobook.toJSONExpanded())
await this.db.updateEntity('audiobook', audiobook)
return ScanResult.UPDATED
}
return ScanResult.UPTODATE
}
async scan(libraryId, options = {}) {
if (this.librariesScanning.includes(libraryId)) {
if (this.isLibraryScanning(libraryId)) {
Logger.error(`[Scanner] Already scanning ${libraryId}`)
return
}
@ -66,172 +141,380 @@ class Scanner {
var libraryScan = new LibraryScan()
libraryScan.setData(library, scanOptions)
this.librariesScanning.push(libraryScan)
libraryScan.verbose = false
this.librariesScanning.push(libraryScan.getScanEmitData)
this.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
await this.scanLibrary(libraryScan)
var canceled = await this.scanLibrary(libraryScan)
if (canceled) {
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
delete this.cancelLibraryScan[libraryScan.libraryId]
}
libraryScan.setComplete()
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
if (canceled && !libraryScan.totalResults) {
var emitData = libraryScan.getScanEmitData
emitData.results = null
this.emitter('scan_complete', emitData)
return
}
this.emitter('scan_complete', libraryScan.getScanEmitData)
if (libraryScan.totalResults) {
libraryScan.saveLog(this.ScanLogPath)
}
}
async scanLibrary(libraryScan) {
var audiobookDataFound = []
// Scan each library
for (let i = 0; i < libraryScan.folders.length; i++) {
var folder = libraryScan.folders[i]
var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
libraryScan.addLog(LogLevel.INFO, `${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
}
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
// Remove audiobooks with no inode
audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
const NumScansPerChunk = 25
const audiobooksToUpdateChunks = []
const audiobookDataToRescanChunks = []
const newAudiobookDataToScanChunks = []
var audiobooksToUpdate = []
var audiobookRescans = []
var newAudiobookScans = []
var audiobookDataToRescan = []
var newAudiobookDataToScan = []
var audiobooksToFindCovers = []
// Check for existing & removed audiobooks
for (let i = 0; i < audiobooksInLibrary.length; i++) {
var audiobook = audiobooksInLibrary[i]
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
if (!dataFound) {
Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
libraryScan.addLog(LogLevel.WARN, `Audiobook "${audiobook.title}" is missing`)
libraryScan.resultsMissing++
audiobook.setMissing()
audiobooksToUpdate.push(audiobook)
if (audiobooksToUpdate.length === NumScansPerChunk) {
audiobooksToUpdateChunks.push(audiobooksToUpdate)
audiobooksToUpdate = []
}
} else {
var checkRes = audiobook.checkScanData(dataFound)
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
// existing audiobook has new files
var checkRes = audiobook.checkScanData(dataFound, version)
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // Audiobook has new files
checkRes.audiobook = audiobook
checkRes.bookScanData = dataFound
audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
libraryScan.resultsMissing++
} else if (checkRes.updated) {
audiobooksToUpdate.push(audiobook)
audiobookDataToRescan.push(checkRes)
if (audiobookDataToRescan.length === NumScansPerChunk) {
audiobookDataToRescanChunks.push(audiobookDataToRescan)
audiobookDataToRescan = []
}
} else if (libraryScan.findCovers && audiobook.book.shouldSearchForCover) {
libraryScan.resultsUpdated++
audiobooksToFindCovers.push(audiobook)
audiobooksToUpdate.push(audiobook)
if (audiobooksToUpdate.length === NumScansPerChunk) {
audiobooksToUpdateChunks.push(audiobooksToUpdate)
audiobooksToUpdate = []
}
} else if (checkRes.updated) { // Updated but no scan required
libraryScan.resultsUpdated++
audiobooksToUpdate.push(audiobook)
if (audiobooksToUpdate.length === NumScansPerChunk) {
audiobooksToUpdateChunks.push(audiobooksToUpdate)
audiobooksToUpdate = []
}
}
audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
}
}
if (audiobooksToUpdate.length) audiobooksToUpdateChunks.push(audiobooksToUpdate)
if (audiobookDataToRescan.length) audiobookDataToRescanChunks.push(audiobookDataToRescan)
// Potential NEW Audiobooks
for (let i = 0; i < audiobookDataFound.length; i++) {
var dataFound = audiobookDataFound[i]
var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
if (!hasEbook && !dataFound.audioFiles.length) {
Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
libraryScan.addLog(LogLevel.WARN, `Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
} else {
newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
newAudiobookDataToScan.push(dataFound)
if (newAudiobookDataToScan.length === NumScansPerChunk) {
newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
newAudiobookDataToScan = []
}
}
}
if (newAudiobookDataToScan.length) newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
// console.log('Num chunks to update', audiobooksToUpdateChunks.length)
// console.log('Num chunks to rescan', audiobookDataToRescanChunks.length)
// console.log('Num chunks to new scan', newAudiobookDataToScanChunks.length)
// Audiobooks not requiring a scan but require a search for cover
for (let i = 0; i < audiobooksToFindCovers.length; i++) {
var audiobook = audiobooksToFindCovers[i]
var updatedCover = await this.searchForCover(audiobook, libraryScan)
audiobook.book.updateLastCoverSearch(updatedCover)
}
for (let i = 0; i < audiobooksToUpdateChunks.length; i++) {
await this.updateAudiobooksChunk(audiobooksToUpdateChunks[i])
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
// console.log('Update chunk done', i, 'of', audiobooksToUpdateChunks.length)
}
for (let i = 0; i < audiobookDataToRescanChunks.length; i++) {
await this.rescanAudiobookDataChunk(audiobookDataToRescanChunks[i], libraryScan)
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
// console.log('Rescan chunk done', i, 'of', audiobookDataToRescanChunks.length)
}
for (let i = 0; i < newAudiobookDataToScanChunks.length; i++) {
await this.scanNewAudiobookDataChunk(newAudiobookDataToScanChunks[i], libraryScan)
// console.log('New scan chunk done', i, 'of', newAudiobookDataToScanChunks.length)
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
}
}
if (audiobookRescans.length) {
var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
if (updatedAudiobooks.length) {
audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
libraryScan.resultsUpdated += updatedAudiobooks.length
}
}
if (audiobooksToUpdate.length) {
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
async updateAudiobooksChunk(audiobooksToUpdate) {
await this.db.updateEntities('audiobook', audiobooksToUpdate)
this.emitter('audiobooks_updated', audiobooksToUpdate.map(ab => ab.toJSONExpanded()))
}
if (newAudiobookScans.length) {
var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
if (newAudiobooks.length) {
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
async rescanAudiobookDataChunk(audiobookDataToRescan, libraryScan) {
var audiobooksUpdated = await Promise.all(audiobookDataToRescan.map((abd) => {
return this.rescanAudiobook(abd, libraryScan)
}))
audiobooksUpdated = audiobooksUpdated.filter(ab => ab) // Filter out nulls
libraryScan.resultsUpdated += audiobooksUpdated.length
await this.db.updateEntities('audiobook', audiobooksUpdated)
this.emitter('audiobooks_updated', audiobooksUpdated.map(ab => ab.toJSONExpanded()))
}
async scanNewAudiobookDataChunk(newAudiobookDataToScan, libraryScan) {
var newAudiobooks = await Promise.all(newAudiobookDataToScan.map((abd) => {
return this.scanNewAudiobook(abd, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan)
}))
newAudiobooks = newAudiobooks.filter(ab => ab) // Filter out nulls
libraryScan.resultsAdded += newAudiobooks.length
await this.db.insertEntities('audiobook', newAudiobooks)
libraryScan.resultsAdded = newAudiobooks.length
}
}
this.emitter('audiobooks_added', newAudiobooks.map(ab => ab.toJSONExpanded()))
}
async rescanAudiobook(audiobookCheckData, libraryScan) {
const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
// Sync other files first to use local images as cover before extracting audio file cover
if (newOtherFileData.length) {
// TODO: Cleanup other file sync
var allOtherFiles = newOtherFileData.concat(audiobook._otherFiles)
await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, libraryScan.preferOpfMetadata)
}
if (newAudioFileData.length) {
var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
if (audioScanResult.audioFiles.length) {
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
// validate & add audio files to audiobook
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
var newAF = audioScanResult.audioFiles[i]
var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
if (trackIndex !== null) {
if (audiobook.checkHasTrackNum(trackIndex)) {
newAF.setDuplicateTrackNumber(trackIndex)
} else {
newAF.index = trackIndex
}
}
audiobook.addAudioFile(newAF)
}
audiobook.rebuildTracks()
}
}
if (newOtherFileData.length) {
await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
}
return audiobook
}
async scanNewAudiobook(audiobookData, libraryScan) {
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
if (audiobookData.audioFiles.length) {
var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
if (audioScanResult.audioFiles.length) {
// validate & add audio files to audiobook
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
var newAF = audioScanResult.audioFiles[i]
var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
if (trackIndex !== null) {
if (audiobook.checkHasTrackNum(trackIndex)) {
newAF.setDuplicateTrackNumber(trackIndex)
} else {
newAF.index = trackIndex
}
}
audiobook.addAudioFile(newAF)
}
audiobook.rebuildTracks()
} else if (!audiobook.ebooks.length) {
// Audiobook has no ebooks and no valid audio tracks do not continue
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
return null
}
}
// Look for desc.txt and reader.txt and update
await audiobook.saveDataFromTextFiles()
await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData, audiobook, libraryScan.preferAudioMetadata, libraryScan)
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
var outputCoverDirs = this.getCoverDirectory(audiobook)
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
if (relativeDir) {
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
}
}
}
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
audiobook.setInvalid()
} else if (audiobook.isInvalid) {
audiobook.isInvalid = false
}
// Scan for cover if enabled and has no cover
if (audiobook && libraryScan.findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
var updatedCover = await this.searchForCover(audiobook, libraryScan)
audiobook.book.updateLastCoverSearch(updatedCover)
}
return audiobook
}
async scanNewAudiobook(audiobookData, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new book "${audiobookData.path}"`)
else Logger.debug(`[Scanner] Scanning new book "${audiobookData.path}"`)
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
if (audiobookData.audioFiles.length) {
await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, preferAudioMetadata, libraryScan)
}
if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) {
// Audiobook has no ebooks and no valid audio tracks do not continue
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
return null
}
// Look for desc.txt and reader.txt and update
await audiobook.saveDataFromTextFiles(preferOpfMetadata)
// Extract embedded cover art if cover is not already in directory
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
var outputCoverDirs = this.getCoverDirectory(audiobook)
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
if (relativeDir) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
else Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
}
}
// Scan for cover if enabled and has no cover
if (audiobook && findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
var updatedCover = await this.searchForCover(audiobook, libraryScan)
audiobook.book.updateLastCoverSearch(updatedCover)
}
return audiobook
}
getFileUpdatesGrouped(fileUpdates) {
var folderGroups = {}
fileUpdates.forEach((file) => {
if (folderGroups[file.folderId]) {
folderGroups[file.folderId].fileUpdates.push(file)
} else {
folderGroups[file.folderId] = {
libraryId: file.libraryId,
folderId: file.folderId,
fileUpdates: [file]
}
}
})
return folderGroups
}
async scanFilesChanged(fileUpdates) {
if (!fileUpdates.length) return
// files grouped by folder
var folderGroups = this.getFileUpdatesGrouped(fileUpdates)
for (const folderId in folderGroups) {
var libraryId = folderGroups[folderId].libraryId
var library = this.db.libraries.find(lib => lib.id === libraryId)
if (!library) {
Logger.error(`[Scanner] Library not found in files changed ${libraryId}`)
continue;
}
var folder = library.getFolderById(folderId)
if (!folder) {
Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
continue;
}
var relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
var fileUpdateBookGroup = groupFilesIntoAudiobookPaths(relFilePaths, true)
var folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateBookGroup)
Logger.debug(`[Scanner] Folder scan results`, folderScanResults)
}
}
async scanFolderUpdates(library, folder, fileUpdateBookGroup) {
Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
var bookGroupingResults = {}
for (const bookDir in fileUpdateBookGroup) {
var fullPath = Path.posix.join(folder.fullPath.replace(/\\/g, '/'), bookDir)
// Check if book dir group is already an audiobook or in a subdir of an audiobook
var existingAudiobook = this.db.audiobooks.find(ab => fullPath.startsWith(ab.fullPath))
if (existingAudiobook) {
// Is the audiobook exactly - check if was deleted
if (existingAudiobook.fullPath === fullPath) {
var exists = await fs.pathExists(fullPath)
if (!exists) {
Logger.info(`[Scanner] Scanning file update group and audiobook was deleted "${existingAudiobook.title}" - marking as missing`)
existingAudiobook.setMissing()
await this.db.updateAudiobook(existingAudiobook)
this.emitter('audiobook_updated', existingAudiobook.toJSONExpanded())
bookGroupingResults[bookDir] = ScanResult.REMOVED
continue;
}
}
// Scan audiobook for updates
Logger.debug(`[Scanner] Folder update for relative path "${bookDir}" is in audiobook "${existingAudiobook.title}" - scan for updates`)
bookGroupingResults[bookDir] = await this.scanAudiobook(folder, existingAudiobook)
continue;
}
// Check if an audiobook is a subdirectory of this dir
var childAudiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(fullPath))
if (childAudiobook) {
Logger.warn(`[Scanner] Files were modified in a parent directory of an audiobook "${childAudiobook.title}" - ignoring`)
bookGroupingResults[bookDir] = ScanResult.NOTHING
continue;
}
Logger.debug(`[Scanner] Folder update group must be a new book "${bookDir}" in library "${library.name}"`)
var newAudiobook = await this.scanPotentialNewAudiobook(folder, fullPath)
if (newAudiobook) {
await this.db.insertEntity('audiobook', newAudiobook)
this.emitter('audiobook_added', newAudiobook.toJSONExpanded())
}
bookGroupingResults[bookDir] = newAudiobook ? ScanResult.ADDED : ScanResult.NOTHING
}
return bookGroupingResults
}
async scanPotentialNewAudiobook(folder, fullPath) {
var audiobookData = await getAudiobookFileData(folder, fullPath, this.db.serverSettings)
if (!audiobookData) return null
var serverSettings = this.db.serverSettings
return this.scanNewAudiobook(audiobookData, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers)
}
async searchForCover(audiobook, libraryScan = null) {
var options = {
titleDistance: 2,
authorDistance: 2
}
var results = await this.bookFinder.findCovers('google', audiobook.title, audiobook.authorFL, options)
if (results.length) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${audiobook.title}"`)
else Logger.debug(`[Scanner] Found best cover for "${audiobook.title}"`)
// If the first cover result fails, attempt to download the second
for (let i = 0; i < results.length && i < 2; i++) {
// Downloads and updates the book cover
var result = await this.coverController.downloadCoverFromUrl(audiobook, results[i])
if (result.error) {
Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
} else {
return true
}
}
}
return false
}
}
module.exports = Scanner

View File

@ -6,14 +6,6 @@ module.exports.ScanResult = {
UPTODATE: 4
}
module.exports.ScanStatus = {
NOTHING: 0,
ADDED: 1,
UPDATED: 2,
REMOVED: 3,
UPTODATE: 4
}
module.exports.CoverDestination = {
METADATA: 0,
AUDIOBOOK: 1

View File

@ -16,7 +16,7 @@ function isBookFile(path) {
// TODO: Function needs to be re-done
// Input: array of relative file paths
// Output: map of files grouped into potential audiobook dirs
function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) {
function groupFilesIntoAudiobookPaths(paths) {
// Step 1: Clean path, Remove leading "/", Filter out files in root dir
var pathsFiltered = paths.map(path => {
return path.startsWith('/') ? path.slice(1) : path