Scanner update - remove and update audiobooks on scans

This commit is contained in:
advplyr 2021-08-24 07:15:56 -05:00
parent db2f2d6660
commit c59cc52667
12 changed files with 239 additions and 65 deletions

View File

@ -83,9 +83,15 @@ export default {
}
this.$store.commit('audiobooks/remove', audiobook)
},
scanComplete() {
scanComplete(results) {
if (!results) results = {}
this.$store.commit('setIsScanning', false)
this.$toast.success('Scan Finished')
var scanResultMsgs = []
if (results.added) scanResultMsgs.push(`${results.added} added`)
if (results.updated) scanResultMsgs.push(`${results.updated} updated`)
if (results.removed) scanResultMsgs.push(`${results.removed} removed`)
if (!scanResultMsgs.length) this.$toast.success('Scan Finished\nEverything was up to date')
else this.$toast.success('Scan Finished\n' + scanResultMsgs.join('\n'))
},
scanStart() {
this.$store.commit('setIsScanning', true)

View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf-client",
"version": "0.9.71-beta",
"version": "0.9.72-beta",
"description": "Audiobook manager and player",
"main": "index.js",
"scripts": {

View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf",
"version": "0.9.71-beta",
"version": "0.9.72-beta",
"description": "Self-hosted audiobook server for managing and playing audiobooks.",
"main": "index.js",
"scripts": {

View File

@ -1,4 +1,7 @@
const Path = require('path')
const { bytesPretty, elapsedPretty } = require('./utils/fileUtils')
const { comparePaths } = require('./utils/index')
const Logger = require('./Logger')
const Book = require('./Book')
const AudioTrack = require('./AudioTrack')
@ -8,6 +11,7 @@ class Audiobook {
this.path = null
this.fullPath = null
this.addedAt = null
this.lastUpdate = null
this.tracks = []
this.missingParts = []
@ -29,6 +33,7 @@ class Audiobook {
this.path = audiobook.path
this.fullPath = audiobook.fullPath
this.addedAt = audiobook.addedAt
this.lastUpdate = audiobook.lastUpdate || this.addedAt
this.tracks = audiobook.tracks.map(track => {
return new AudioTrack(track)
@ -99,6 +104,7 @@ class Audiobook {
path: this.path,
fullPath: this.fullPath,
addedAt: this.addedAt,
lastUpdate: this.lastUpdate,
missingParts: this.missingParts,
invalidParts: this.invalidParts,
tags: this.tags,
@ -117,6 +123,7 @@ class Audiobook {
path: this.path,
fullPath: this.fullPath,
addedAt: this.addedAt,
lastUpdate: this.lastUpdate,
duration: this.totalDuration,
size: this.totalSize,
hasBookMatch: !!this.book,
@ -135,6 +142,7 @@ class Audiobook {
path: this.path,
fullPath: this.fullPath,
addedAt: this.addedAt,
lastUpdate: this.lastUpdate,
duration: this.totalDuration,
durationPretty: this.durationPretty,
size: this.totalSize,
@ -154,6 +162,7 @@ class Audiobook {
this.path = data.path
this.fullPath = data.fullPath
this.addedAt = Date.now()
this.lastUpdate = this.addedAt
this.otherFiles = data.otherFiles || []
this.setBook(data)
@ -188,6 +197,10 @@ class Audiobook {
}
}
if (hasUpdates) {
this.lastUpdate = Date.now()
}
return hasUpdates
}
@ -206,6 +219,77 @@ class Audiobook {
this.audioFiles.forEach((file) => {
this.addTrack(file)
})
this.lastUpdate = Date.now()
}
removeAudioFile(audioFile) {
this.tracks = this.tracks.filter(t => t.path !== audioFile.path)
this.audioFiles = this.audioFiles.filter(f => f.path !== audioFile.path)
}
audioPartExists(part) {
var path = Path.join(this.path, part)
return this.audioFiles.find(file => file.path === path)
}
checkUpdateMissingParts() {
var currMissingParts = this.missingParts.join(',')
var current_index = 1
var missingParts = []
for (let i = 0; i < this.tracks.length; i++) {
var _track = this.tracks[i]
if (_track.index > current_index) {
var num_parts_missing = _track.index - current_index
for (let x = 0; x < num_parts_missing; x++) {
missingParts.push(current_index + x)
}
}
current_index = _track.index + 1
}
this.missingParts = missingParts
var wasUpdated = this.missingParts.join(',') !== currMissingParts
if (wasUpdated && this.missingParts.length) {
Logger.info(`[Audiobook] "${this.title}" has ${missingParts.length} missing parts`)
}
return wasUpdated
}
// On scan check other files found with other files saved
syncOtherFiles(newOtherFiles) {
var currOtherFileNum = this.otherFiles.length
var newOtherFilePaths = newOtherFiles.map(f => f.path)
this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
newOtherFiles.forEach((file) => {
var existingOtherFile = this.otherFiles.find(f => f.path === file.path)
if (!existingOtherFile) {
Logger.info(`[Audiobook] New other file found on sync ${file.filename}/${file.filetype} | "${this.title}"`)
this.otherFiles.push(file)
}
})
var hasUpdates = currOtherFileNum !== this.otherFiles.length
var imageFiles = this.otherFiles.filter(f => f.filetype === 'image')
if (this.book.cover && this.book.cover.substr(1).startsWith('local')) {
var coverStillExists = imageFiles.find(f => comparePaths(f.path, this.book.cover.substr('/local/'.length)))
if (!coverStillExists) {
Logger.info(`[Audiobook] Local cover was removed | "${this.title}"`)
this.book.cover = null
hasUpdates = true
}
}
if (!this.book.cover && imageFiles.length) {
this.book.cover = Path.join('/local', imageFiles[0].path)
Logger.info(`[Audiobook] Local cover was set | "${this.title}"`)
hasUpdates = true
}
return hasUpdates
}
isSearchMatch(search) {

View File

@ -104,8 +104,10 @@ class Db {
updateAudiobook(audiobook) {
return this.audiobooksDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => {
Logger.debug(`[DB] Audiobook updated ${results.updated}`)
return true
}).catch((error) => {
Logger.error(`[DB] Audiobook update failed ${error}`)
return false
})
}

View File

@ -20,41 +20,121 @@ class Scanner {
}
async scan() {
// console.log('Start scan audiobooks', this.audiobooks.map(a => a.fullPath).join(', '))
const scanStart = Date.now()
var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath)
var scanResults = {
removed: 0,
updated: 0,
added: 0
}
// Check for removed audiobooks
for (let i = 0; i < this.audiobooks.length; i++) {
var dataFound = audiobookDataFound.find(abd => abd.path === this.audiobooks[i].path)
if (!dataFound) {
Logger.info(`[Scanner] Removing audiobook "${this.audiobooks[i].title}" - no longer in dir`)
await this.db.removeEntity('audiobook', this.audiobooks[i].id)
if (!this.audiobooks[i]) {
Logger.error('[Scanner] Oops... audiobook is now invalid...')
continue;
}
scanResults.removed++
this.emitter('audiobook_removed', this.audiobooks[i].toJSONMinified())
}
}
for (let i = 0; i < audiobookDataFound.length; i++) {
var audiobookData = audiobookDataFound[i]
if (!audiobookData.parts.length) {
Logger.error('No Valid Parts for Audiobook', audiobookData)
} else {
var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
if (existingAudiobook) {
Logger.info('Audiobook already added', audiobookData.title)
// Todo: Update Audiobook here
var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
if (existingAudiobook) {
Logger.debug(`[Scanner] Audiobook already added, check updates for "${existingAudiobook.title}"`)
if (!audiobookData.parts.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
scanResults.removed++
} else {
// Check for audio files that were removed
var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !audiobookData.parts.includes(file.filename))
if (removedAudioFiles.length) {
Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
}
// Check for audio files that were added
var newParts = audiobookData.parts.filter(part => !existingAudiobook.audioPartExists(part))
if (newParts.length) {
Logger.info(`[Scanner] ${newParts.length} new audio parts were found for audiobook "${existingAudiobook.title}"`)
// If previously invalid part, remove from invalid list because it will be re-scanned
newParts.forEach((part) => {
if (existingAudiobook.invalidParts.includes(part)) {
existingAudiobook.invalidParts = existingAudiobook.invalidParts.filter(p => p !== part)
}
})
// Scan new audio parts found
await audioFileScanner.scanParts(existingAudiobook, newParts)
}
if (!existingAudiobook.tracks.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
} else {
var hasUpdates = removedAudioFiles.length || newParts.length
if (existingAudiobook.checkUpdateMissingParts()) {
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
hasUpdates = true
}
if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
hasUpdates = true
}
if (hasUpdates) {
Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
existingAudiobook.lastUpdate = Date.now()
await this.db.updateAudiobook(existingAudiobook)
this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
scanResults.updated++
}
}
} // end if update existing
} else {
if (!audiobookData.parts.length) {
Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData)
} else {
// console.log('Audiobook not already there... add new audiobook', audiobookData.fullPath)
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
await audioFileScanner.scanParts(audiobook, audiobookData.parts)
if (!audiobook.tracks.length) {
Logger.warn('Invalid audiobook, no valid tracks', audiobook.title)
Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
} else {
Logger.info('Audiobook Scanned', audiobook.title, `(${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
audiobook.checkUpdateMissingParts()
Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
await this.db.insertAudiobook(audiobook)
this.emitter('audiobook_added', audiobook.toJSONMinified())
scanResults.added++
}
}
var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
this.emitter('scan_progress', {
total: audiobookDataFound.length,
done: i + 1,
progress
})
} // end if add new
}
var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
this.emitter('scan_progress', {
total: audiobookDataFound.length,
done: i + 1,
progress
})
}
const scanElapsed = Math.floor((Date.now() - scanStart) / 1000)
Logger.info(`[SCANNER] Finished ${secondsToTimestamp(scanElapsed)}`)
Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | elapsed: ${secondsToTimestamp(scanElapsed)}`)
return scanResults
}
async fetchMetadata(id, trackIndex = 0) {

View File

@ -53,33 +53,26 @@ class Server {
}
emitter(ev, data) {
Logger.debug('EMITTER', ev)
if (!this.io) {
Logger.error('Invalid IO')
return
}
// Logger.debug('EMITTER', ev)
this.io.emit(ev, data)
}
async fileAddedUpdated({ path, fullPath }) {
Logger.info('[SERVER] FileAddedUpdated', path, fullPath)
}
async fileAddedUpdated({ path, fullPath }) { }
async fileRemoved({ path, fullPath }) { }
async scan() {
Logger.info('[SERVER] Starting Scan')
Logger.info('[Server] Starting Scan')
this.isScanning = true
this.isInitialized = true
this.emitter('scan_start')
await this.scanner.scan()
var results = await this.scanner.scan()
this.isScanning = false
this.emitter('scan_complete')
Logger.info('[SERVER] Scan complete')
this.emitter('scan_complete', results)
Logger.info('[Server] Scan complete')
}
async init() {
Logger.info('[SERVER] Init')
Logger.info('[Server] Init')
await this.streamManager.removeOrphanStreams()
await this.db.init()
this.auth.init()

View File

@ -129,7 +129,6 @@ class Stream extends EventEmitter {
async generatePlaylist() {
fs.ensureDirSync(this.streamPath)
await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength)
console.log('Playlist generated')
return this.clientPlaylistUri
}

View File

@ -45,7 +45,7 @@ class FolderWatcher extends EventEmitter {
}
onNewFile(path) {
Logger.info('FolderWatcher: New File', path)
Logger.debug('FolderWatcher: New File', path)
this.emit('file_added', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path
@ -53,7 +53,7 @@ class FolderWatcher extends EventEmitter {
}
onFileRemoved(path) {
Logger.info('FolderWatcher: File Removed', path)
Logger.debug('FolderWatcher: File Removed', path)
this.emit('file_removed', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path
@ -61,7 +61,7 @@ class FolderWatcher extends EventEmitter {
}
onFileUpdated(path) {
Logger.info('FolderWatcher: Updated File', path)
Logger.debug('FolderWatcher: Updated File', path)
this.emit('file_updated', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path

View File

@ -78,7 +78,7 @@ function getTrackNumberFromFilename(filename) {
async function scanParts(audiobook, parts) {
if (!parts || !parts.length) {
Logger.error('Scan Parts', audiobook.title, 'No Parts', parts)
Logger.error('[AudioFileScanner] Scan Parts', audiobook.title, 'No Parts', parts)
return
}
var tracks = []
@ -87,7 +87,7 @@ async function scanParts(audiobook, parts) {
var scanData = await scan(fullPath)
if (!scanData || scanData.error) {
Logger.error('Scan failed for', parts[i])
Logger.error('[AudioFileScanner] Scan failed for', parts[i])
audiobook.invalidParts.push(parts[i])
continue;
}
@ -110,7 +110,7 @@ async function scanParts(audiobook, parts) {
if (parts.length > 1) {
trackNumber = isNumber(trackNumFromMeta) ? trackNumFromMeta : trackNumFromFilename
if (trackNumber === null) {
Logger.error('Invalid track number for', parts[i])
Logger.error('[AudioFileScanner] Invalid track number for', parts[i])
audioFileObj.invalid = true
audioFileObj.error = 'Failed to get track number'
continue;
@ -118,7 +118,7 @@ async function scanParts(audiobook, parts) {
}
if (tracks.find(t => t.index === trackNumber)) {
Logger.error('Duplicate track number for', parts[i])
Logger.error('[AudioFileScanner] Duplicate track number for', parts[i])
audioFileObj.invalid = true
audioFileObj.error = 'Duplicate track number'
continue;
@ -129,7 +129,7 @@ async function scanParts(audiobook, parts) {
}
if (!tracks.length) {
Logger.warn('No Tracks for audiobook', audiobook.id)
Logger.warn('[AudioFileScanner] No Tracks for audiobook', audiobook.id)
return
}
@ -148,26 +148,12 @@ async function scanParts(audiobook, parts) {
})
}
var parts_copy = tracks.map(p => ({ ...p }))
var current_index = 1
for (let i = 0; i < parts_copy.length; i++) {
var cleaned_part = parts_copy[i]
if (cleaned_part.index > current_index) {
var num_parts_missing = cleaned_part.index - current_index
for (let x = 0; x < num_parts_missing; x++) {
audiobook.missingParts.push(current_index + x)
}
}
current_index = cleaned_part.index + 1
}
if (audiobook.missingParts.length) {
Logger.info('Audiobook', audiobook.title, 'Has missing parts', audiobook.missingParts)
}
var hasTracksAlready = audiobook.tracks.length
tracks.forEach((track) => {
audiobook.addTrack(track)
})
if (hasTracksAlready) {
audiobook.tracks.sort((a, b) => a.index - b.index)
}
}
module.exports.scanParts = scanParts

View File

@ -1,3 +1,5 @@
const Path = require('path')
const levenshteinDistance = (str1, str2, caseSensitive = false) => {
if (!caseSensitive) {
str1 = str1.toLowerCase()
@ -44,4 +46,26 @@ module.exports.cleanString = cleanString
module.exports.isObject = (val) => {
return val !== null && typeof val === 'object'
}
function normalizePath(path) {
const replace = [
[/\\/g, '/'],
[/(\w):/, '/$1'],
[/(\w+)\/\.\.\/?/g, ''],
[/^\.\//, ''],
[/\/\.\//, '/'],
[/\/\.$/, ''],
[/\/$/, ''],
]
replace.forEach(array => {
while (array[0].test(path)) {
path = path.replace(array[0], array[1])
}
})
return path
}
module.exports.comparePaths = (path1, path2) => {
return (path1 === path2) || (normalizePath(path1) === normalizePath(path2))
}

View File

@ -69,7 +69,7 @@ async function getAllAudiobookFiles(abRootPath) {
title: title,
series: cleanString(series),
publishYear: publishYear,
path: relpath,
path: path,
fullPath: Path.join(abRootPath, path),
parts: [],
otherFiles: []