From c59cc52667c6d79af119d8ad8bee16a76be70e27 Mon Sep 17 00:00:00 2001 From: advplyr Date: Tue, 24 Aug 2021 07:15:56 -0500 Subject: [PATCH] Scanner update - remove and update audiobooks on scans --- client/layouts/default.vue | 10 ++- client/package.json | 2 +- package.json | 2 +- server/Audiobook.js | 84 ++++++++++++++++++++++ server/Db.js | 2 + server/Scanner.js | 118 ++++++++++++++++++++++++++----- server/Server.js | 21 ++---- server/Stream.js | 1 - server/Watcher.js | 6 +- server/utils/audioFileScanner.js | 32 +++------ server/utils/index.js | 24 +++++++ server/utils/scandir.js | 2 +- 12 files changed, 239 insertions(+), 65 deletions(-) diff --git a/client/layouts/default.vue b/client/layouts/default.vue index a33d65f2..7672eccb 100644 --- a/client/layouts/default.vue +++ b/client/layouts/default.vue @@ -83,9 +83,15 @@ export default { } this.$store.commit('audiobooks/remove', audiobook) }, - scanComplete() { + scanComplete(results) { + if (!results) results = {} this.$store.commit('setIsScanning', false) - this.$toast.success('Scan Finished') + var scanResultMsgs = [] + if (results.added) scanResultMsgs.push(`${results.added} added`) + if (results.updated) scanResultMsgs.push(`${results.updated} updated`) + if (results.removed) scanResultMsgs.push(`${results.removed} removed`) + if (!scanResultMsgs.length) this.$toast.success('Scan Finished\nEverything was up to date') + else this.$toast.success('Scan Finished\n' + scanResultMsgs.join('\n')) }, scanStart() { this.$store.commit('setIsScanning', true) diff --git a/client/package.json b/client/package.json index 6e9a2e48..27de580f 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf-client", - "version": "0.9.71-beta", + "version": "0.9.72-beta", "description": "Audiobook manager and player", "main": "index.js", "scripts": { diff --git a/package.json b/package.json index 286bc205..090790d8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "audiobookshelf", - "version": "0.9.71-beta", + "version": "0.9.72-beta", "description": "Self-hosted audiobook server for managing and playing audiobooks.", "main": "index.js", "scripts": { diff --git a/server/Audiobook.js b/server/Audiobook.js index 11da5272..ea2ff7ee 100644 --- a/server/Audiobook.js +++ b/server/Audiobook.js @@ -1,4 +1,7 @@ +const Path = require('path') const { bytesPretty, elapsedPretty } = require('./utils/fileUtils') +const { comparePaths } = require('./utils/index') +const Logger = require('./Logger') const Book = require('./Book') const AudioTrack = require('./AudioTrack') @@ -8,6 +11,7 @@ class Audiobook { this.path = null this.fullPath = null this.addedAt = null + this.lastUpdate = null this.tracks = [] this.missingParts = [] @@ -29,6 +33,7 @@ class Audiobook { this.path = audiobook.path this.fullPath = audiobook.fullPath this.addedAt = audiobook.addedAt + this.lastUpdate = audiobook.lastUpdate || this.addedAt this.tracks = audiobook.tracks.map(track => { return new AudioTrack(track) @@ -99,6 +104,7 @@ class Audiobook { path: this.path, fullPath: this.fullPath, addedAt: this.addedAt, + lastUpdate: this.lastUpdate, missingParts: this.missingParts, invalidParts: this.invalidParts, tags: this.tags, @@ -117,6 +123,7 @@ class Audiobook { path: this.path, fullPath: this.fullPath, addedAt: this.addedAt, + lastUpdate: this.lastUpdate, duration: this.totalDuration, size: this.totalSize, hasBookMatch: !!this.book, @@ -135,6 +142,7 @@ class Audiobook { path: this.path, fullPath: this.fullPath, addedAt: this.addedAt, + lastUpdate: this.lastUpdate, duration: this.totalDuration, durationPretty: this.durationPretty, size: this.totalSize, @@ -154,6 +162,7 @@ class Audiobook { this.path = data.path this.fullPath = data.fullPath this.addedAt = Date.now() + this.lastUpdate = this.addedAt this.otherFiles = data.otherFiles || [] this.setBook(data) @@ -188,6 +197,10 @@ class Audiobook { } } + if (hasUpdates) { + this.lastUpdate = Date.now() + } + return hasUpdates } @@ -206,6 +219,77 @@ class Audiobook { this.audioFiles.forEach((file) => { this.addTrack(file) }) + this.lastUpdate = Date.now() + } + + removeAudioFile(audioFile) { + this.tracks = this.tracks.filter(t => t.path !== audioFile.path) + this.audioFiles = this.audioFiles.filter(f => f.path !== audioFile.path) + } + + audioPartExists(part) { + var path = Path.join(this.path, part) + return this.audioFiles.find(file => file.path === path) + } + + checkUpdateMissingParts() { + var currMissingParts = this.missingParts.join(',') + + var current_index = 1 + var missingParts = [] + for (let i = 0; i < this.tracks.length; i++) { + var _track = this.tracks[i] + if (_track.index > current_index) { + var num_parts_missing = _track.index - current_index + for (let x = 0; x < num_parts_missing; x++) { + missingParts.push(current_index + x) + } + } + current_index = _track.index + 1 + } + + this.missingParts = missingParts + + var wasUpdated = this.missingParts.join(',') !== currMissingParts + if (wasUpdated && this.missingParts.length) { + Logger.info(`[Audiobook] "${this.title}" has ${missingParts.length} missing parts`) + } + + return wasUpdated + } + + // On scan check other files found with other files saved + syncOtherFiles(newOtherFiles) { + var currOtherFileNum = this.otherFiles.length + + var newOtherFilePaths = newOtherFiles.map(f => f.path) + this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path)) + newOtherFiles.forEach((file) => { + var existingOtherFile = this.otherFiles.find(f => f.path === file.path) + if (!existingOtherFile) { + Logger.info(`[Audiobook] New other file found on sync ${file.filename}/${file.filetype} | "${this.title}"`) + this.otherFiles.push(file) + } + }) + + var hasUpdates = currOtherFileNum !== this.otherFiles.length + + var imageFiles = this.otherFiles.filter(f => f.filetype === 'image') + if (this.book.cover && this.book.cover.substr(1).startsWith('local')) { + var coverStillExists = imageFiles.find(f => comparePaths(f.path, this.book.cover.substr('/local/'.length))) + if (!coverStillExists) { + Logger.info(`[Audiobook] Local cover was removed | "${this.title}"`) + this.book.cover = null + hasUpdates = true + } + } + + if (!this.book.cover && imageFiles.length) { + this.book.cover = Path.join('/local', imageFiles[0].path) + Logger.info(`[Audiobook] Local cover was set | "${this.title}"`) + hasUpdates = true + } + return hasUpdates } isSearchMatch(search) { diff --git a/server/Db.js b/server/Db.js index c60025f1..6b02716c 100644 --- a/server/Db.js +++ b/server/Db.js @@ -104,8 +104,10 @@ class Db { updateAudiobook(audiobook) { return this.audiobooksDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => { Logger.debug(`[DB] Audiobook updated ${results.updated}`) + return true }).catch((error) => { Logger.error(`[DB] Audiobook update failed ${error}`) + return false }) } diff --git a/server/Scanner.js b/server/Scanner.js index 73ce85ba..f65142f2 100644 --- a/server/Scanner.js +++ b/server/Scanner.js @@ -20,41 +20,121 @@ class Scanner { } async scan() { - // console.log('Start scan audiobooks', this.audiobooks.map(a => a.fullPath).join(', ')) const scanStart = Date.now() var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath) + + var scanResults = { + removed: 0, + updated: 0, + added: 0 + } + + // Check for removed audiobooks + for (let i = 0; i < this.audiobooks.length; i++) { + var dataFound = audiobookDataFound.find(abd => abd.path === this.audiobooks[i].path) + if (!dataFound) { + Logger.info(`[Scanner] Removing audiobook "${this.audiobooks[i].title}" - no longer in dir`) + + await this.db.removeEntity('audiobook', this.audiobooks[i].id) + if (!this.audiobooks[i]) { + Logger.error('[Scanner] Oops... audiobook is now invalid...') + continue; + } + scanResults.removed++ + this.emitter('audiobook_removed', this.audiobooks[i].toJSONMinified()) + } + } + for (let i = 0; i < audiobookDataFound.length; i++) { var audiobookData = audiobookDataFound[i] - if (!audiobookData.parts.length) { - Logger.error('No Valid Parts for Audiobook', audiobookData) - } else { - var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath) - if (existingAudiobook) { - Logger.info('Audiobook already added', audiobookData.title) - // Todo: Update Audiobook here + var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath) + if (existingAudiobook) { + Logger.debug(`[Scanner] Audiobook already added, check updates for "${existingAudiobook.title}"`) + + if (!audiobookData.parts.length) { + Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`) + + await this.db.removeEntity('audiobook', existingAudiobook.id) + this.emitter('audiobook_removed', existingAudiobook.toJSONMinified()) + scanResults.removed++ + } else { + + // Check for audio files that were removed + var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !audiobookData.parts.includes(file.filename)) + if (removedAudioFiles.length) { + Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`) + removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af)) + } + + // Check for audio files that were added + var newParts = audiobookData.parts.filter(part => !existingAudiobook.audioPartExists(part)) + if (newParts.length) { + Logger.info(`[Scanner] ${newParts.length} new audio parts were found for audiobook "${existingAudiobook.title}"`) + + // If previously invalid part, remove from invalid list because it will be re-scanned + newParts.forEach((part) => { + if (existingAudiobook.invalidParts.includes(part)) { + existingAudiobook.invalidParts = existingAudiobook.invalidParts.filter(p => p !== part) + } + }) + // Scan new audio parts found + await audioFileScanner.scanParts(existingAudiobook, newParts) + } + + if (!existingAudiobook.tracks.length) { + Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`) + + await this.db.removeEntity('audiobook', existingAudiobook.id) + this.emitter('audiobook_removed', existingAudiobook.toJSONMinified()) + } else { + var hasUpdates = removedAudioFiles.length || newParts.length + + if (existingAudiobook.checkUpdateMissingParts()) { + Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`) + hasUpdates = true + } + + if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) { + hasUpdates = true + } + + if (hasUpdates) { + Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`) + existingAudiobook.lastUpdate = Date.now() + await this.db.updateAudiobook(existingAudiobook) + this.emitter('audiobook_updated', existingAudiobook.toJSONMinified()) + scanResults.updated++ + } + } + } // end if update existing + } else { + if (!audiobookData.parts.length) { + Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData) } else { - // console.log('Audiobook not already there... add new audiobook', audiobookData.fullPath) var audiobook = new Audiobook() audiobook.setData(audiobookData) await audioFileScanner.scanParts(audiobook, audiobookData.parts) if (!audiobook.tracks.length) { - Logger.warn('Invalid audiobook, no valid tracks', audiobook.title) + Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title) } else { - Logger.info('Audiobook Scanned', audiobook.title, `(${audiobook.sizePretty}) [${audiobook.durationPretty}]`) + audiobook.checkUpdateMissingParts() + Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`) await this.db.insertAudiobook(audiobook) this.emitter('audiobook_added', audiobook.toJSONMinified()) + scanResults.added++ } - } - var progress = Math.round(100 * (i + 1) / audiobookDataFound.length) - this.emitter('scan_progress', { - total: audiobookDataFound.length, - done: i + 1, - progress - }) + } // end if add new } + var progress = Math.round(100 * (i + 1) / audiobookDataFound.length) + this.emitter('scan_progress', { + total: audiobookDataFound.length, + done: i + 1, + progress + }) } const scanElapsed = Math.floor((Date.now() - scanStart) / 1000) - Logger.info(`[SCANNER] Finished ${secondsToTimestamp(scanElapsed)}`) + Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | elapsed: ${secondsToTimestamp(scanElapsed)}`) + return scanResults } async fetchMetadata(id, trackIndex = 0) { diff --git a/server/Server.js b/server/Server.js index 74532a8d..628220ec 100644 --- a/server/Server.js +++ b/server/Server.js @@ -53,33 +53,26 @@ class Server { } emitter(ev, data) { - Logger.debug('EMITTER', ev) - if (!this.io) { - Logger.error('Invalid IO') - return - } + // Logger.debug('EMITTER', ev) this.io.emit(ev, data) } - async fileAddedUpdated({ path, fullPath }) { - Logger.info('[SERVER] FileAddedUpdated', path, fullPath) - } - + async fileAddedUpdated({ path, fullPath }) { } async fileRemoved({ path, fullPath }) { } async scan() { - Logger.info('[SERVER] Starting Scan') + Logger.info('[Server] Starting Scan') this.isScanning = true this.isInitialized = true this.emitter('scan_start') - await this.scanner.scan() + var results = await this.scanner.scan() this.isScanning = false - this.emitter('scan_complete') - Logger.info('[SERVER] Scan complete') + this.emitter('scan_complete', results) + Logger.info('[Server] Scan complete') } async init() { - Logger.info('[SERVER] Init') + Logger.info('[Server] Init') await this.streamManager.removeOrphanStreams() await this.db.init() this.auth.init() diff --git a/server/Stream.js b/server/Stream.js index bfbe323c..ba8a419f 100644 --- a/server/Stream.js +++ b/server/Stream.js @@ -129,7 +129,6 @@ class Stream extends EventEmitter { async generatePlaylist() { fs.ensureDirSync(this.streamPath) await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength) - console.log('Playlist generated') return this.clientPlaylistUri } diff --git a/server/Watcher.js b/server/Watcher.js index d324879f..740f04a2 100644 --- a/server/Watcher.js +++ b/server/Watcher.js @@ -45,7 +45,7 @@ class FolderWatcher extends EventEmitter { } onNewFile(path) { - Logger.info('FolderWatcher: New File', path) + Logger.debug('FolderWatcher: New File', path) this.emit('file_added', { path: path.replace(this.AudiobookPath, ''), fullPath: path @@ -53,7 +53,7 @@ class FolderWatcher extends EventEmitter { } onFileRemoved(path) { - Logger.info('FolderWatcher: File Removed', path) + Logger.debug('FolderWatcher: File Removed', path) this.emit('file_removed', { path: path.replace(this.AudiobookPath, ''), fullPath: path @@ -61,7 +61,7 @@ class FolderWatcher extends EventEmitter { } onFileUpdated(path) { - Logger.info('FolderWatcher: Updated File', path) + Logger.debug('FolderWatcher: Updated File', path) this.emit('file_updated', { path: path.replace(this.AudiobookPath, ''), fullPath: path diff --git a/server/utils/audioFileScanner.js b/server/utils/audioFileScanner.js index 6086e1d9..674182b3 100644 --- a/server/utils/audioFileScanner.js +++ b/server/utils/audioFileScanner.js @@ -78,7 +78,7 @@ function getTrackNumberFromFilename(filename) { async function scanParts(audiobook, parts) { if (!parts || !parts.length) { - Logger.error('Scan Parts', audiobook.title, 'No Parts', parts) + Logger.error('[AudioFileScanner] Scan Parts', audiobook.title, 'No Parts', parts) return } var tracks = [] @@ -87,7 +87,7 @@ async function scanParts(audiobook, parts) { var scanData = await scan(fullPath) if (!scanData || scanData.error) { - Logger.error('Scan failed for', parts[i]) + Logger.error('[AudioFileScanner] Scan failed for', parts[i]) audiobook.invalidParts.push(parts[i]) continue; } @@ -110,7 +110,7 @@ async function scanParts(audiobook, parts) { if (parts.length > 1) { trackNumber = isNumber(trackNumFromMeta) ? trackNumFromMeta : trackNumFromFilename if (trackNumber === null) { - Logger.error('Invalid track number for', parts[i]) + Logger.error('[AudioFileScanner] Invalid track number for', parts[i]) audioFileObj.invalid = true audioFileObj.error = 'Failed to get track number' continue; @@ -118,7 +118,7 @@ async function scanParts(audiobook, parts) { } if (tracks.find(t => t.index === trackNumber)) { - Logger.error('Duplicate track number for', parts[i]) + Logger.error('[AudioFileScanner] Duplicate track number for', parts[i]) audioFileObj.invalid = true audioFileObj.error = 'Duplicate track number' continue; @@ -129,7 +129,7 @@ async function scanParts(audiobook, parts) { } if (!tracks.length) { - Logger.warn('No Tracks for audiobook', audiobook.id) + Logger.warn('[AudioFileScanner] No Tracks for audiobook', audiobook.id) return } @@ -148,26 +148,12 @@ async function scanParts(audiobook, parts) { }) } - var parts_copy = tracks.map(p => ({ ...p })) - var current_index = 1 - - for (let i = 0; i < parts_copy.length; i++) { - var cleaned_part = parts_copy[i] - if (cleaned_part.index > current_index) { - var num_parts_missing = cleaned_part.index - current_index - for (let x = 0; x < num_parts_missing; x++) { - audiobook.missingParts.push(current_index + x) - } - } - current_index = cleaned_part.index + 1 - } - - if (audiobook.missingParts.length) { - Logger.info('Audiobook', audiobook.title, 'Has missing parts', audiobook.missingParts) - } - + var hasTracksAlready = audiobook.tracks.length tracks.forEach((track) => { audiobook.addTrack(track) }) + if (hasTracksAlready) { + audiobook.tracks.sort((a, b) => a.index - b.index) + } } module.exports.scanParts = scanParts \ No newline at end of file diff --git a/server/utils/index.js b/server/utils/index.js index 181c3f50..c41ee0ac 100644 --- a/server/utils/index.js +++ b/server/utils/index.js @@ -1,3 +1,5 @@ +const Path = require('path') + const levenshteinDistance = (str1, str2, caseSensitive = false) => { if (!caseSensitive) { str1 = str1.toLowerCase() @@ -44,4 +46,26 @@ module.exports.cleanString = cleanString module.exports.isObject = (val) => { return val !== null && typeof val === 'object' +} + +function normalizePath(path) { + const replace = [ + [/\\/g, '/'], + [/(\w):/, '/$1'], + [/(\w+)\/\.\.\/?/g, ''], + [/^\.\//, ''], + [/\/\.\//, '/'], + [/\/\.$/, ''], + [/\/$/, ''], + ] + replace.forEach(array => { + while (array[0].test(path)) { + path = path.replace(array[0], array[1]) + } + }) + return path +} + +module.exports.comparePaths = (path1, path2) => { + return (path1 === path2) || (normalizePath(path1) === normalizePath(path2)) } \ No newline at end of file diff --git a/server/utils/scandir.js b/server/utils/scandir.js index 63592652..4cb1537b 100644 --- a/server/utils/scandir.js +++ b/server/utils/scandir.js @@ -69,7 +69,7 @@ async function getAllAudiobookFiles(abRootPath) { title: title, series: cleanString(series), publishYear: publishYear, - path: relpath, + path: path, fullPath: Path.join(abRootPath, path), parts: [], otherFiles: []