mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-12-29 01:58:49 +01:00
Scanner update - remove and update audiobooks on scans
This commit is contained in:
parent
db2f2d6660
commit
c59cc52667
@ -83,9 +83,15 @@ export default {
|
||||
}
|
||||
this.$store.commit('audiobooks/remove', audiobook)
|
||||
},
|
||||
scanComplete() {
|
||||
scanComplete(results) {
|
||||
if (!results) results = {}
|
||||
this.$store.commit('setIsScanning', false)
|
||||
this.$toast.success('Scan Finished')
|
||||
var scanResultMsgs = []
|
||||
if (results.added) scanResultMsgs.push(`${results.added} added`)
|
||||
if (results.updated) scanResultMsgs.push(`${results.updated} updated`)
|
||||
if (results.removed) scanResultMsgs.push(`${results.removed} removed`)
|
||||
if (!scanResultMsgs.length) this.$toast.success('Scan Finished\nEverything was up to date')
|
||||
else this.$toast.success('Scan Finished\n' + scanResultMsgs.join('\n'))
|
||||
},
|
||||
scanStart() {
|
||||
this.$store.commit('setIsScanning', true)
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "audiobookshelf-client",
|
||||
"version": "0.9.71-beta",
|
||||
"version": "0.9.72-beta",
|
||||
"description": "Audiobook manager and player",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "audiobookshelf",
|
||||
"version": "0.9.71-beta",
|
||||
"version": "0.9.72-beta",
|
||||
"description": "Self-hosted audiobook server for managing and playing audiobooks.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
@ -1,4 +1,7 @@
|
||||
const Path = require('path')
|
||||
const { bytesPretty, elapsedPretty } = require('./utils/fileUtils')
|
||||
const { comparePaths } = require('./utils/index')
|
||||
const Logger = require('./Logger')
|
||||
const Book = require('./Book')
|
||||
const AudioTrack = require('./AudioTrack')
|
||||
|
||||
@ -8,6 +11,7 @@ class Audiobook {
|
||||
this.path = null
|
||||
this.fullPath = null
|
||||
this.addedAt = null
|
||||
this.lastUpdate = null
|
||||
|
||||
this.tracks = []
|
||||
this.missingParts = []
|
||||
@ -29,6 +33,7 @@ class Audiobook {
|
||||
this.path = audiobook.path
|
||||
this.fullPath = audiobook.fullPath
|
||||
this.addedAt = audiobook.addedAt
|
||||
this.lastUpdate = audiobook.lastUpdate || this.addedAt
|
||||
|
||||
this.tracks = audiobook.tracks.map(track => {
|
||||
return new AudioTrack(track)
|
||||
@ -99,6 +104,7 @@ class Audiobook {
|
||||
path: this.path,
|
||||
fullPath: this.fullPath,
|
||||
addedAt: this.addedAt,
|
||||
lastUpdate: this.lastUpdate,
|
||||
missingParts: this.missingParts,
|
||||
invalidParts: this.invalidParts,
|
||||
tags: this.tags,
|
||||
@ -117,6 +123,7 @@ class Audiobook {
|
||||
path: this.path,
|
||||
fullPath: this.fullPath,
|
||||
addedAt: this.addedAt,
|
||||
lastUpdate: this.lastUpdate,
|
||||
duration: this.totalDuration,
|
||||
size: this.totalSize,
|
||||
hasBookMatch: !!this.book,
|
||||
@ -135,6 +142,7 @@ class Audiobook {
|
||||
path: this.path,
|
||||
fullPath: this.fullPath,
|
||||
addedAt: this.addedAt,
|
||||
lastUpdate: this.lastUpdate,
|
||||
duration: this.totalDuration,
|
||||
durationPretty: this.durationPretty,
|
||||
size: this.totalSize,
|
||||
@ -154,6 +162,7 @@ class Audiobook {
|
||||
this.path = data.path
|
||||
this.fullPath = data.fullPath
|
||||
this.addedAt = Date.now()
|
||||
this.lastUpdate = this.addedAt
|
||||
|
||||
this.otherFiles = data.otherFiles || []
|
||||
this.setBook(data)
|
||||
@ -188,6 +197,10 @@ class Audiobook {
|
||||
}
|
||||
}
|
||||
|
||||
if (hasUpdates) {
|
||||
this.lastUpdate = Date.now()
|
||||
}
|
||||
|
||||
return hasUpdates
|
||||
}
|
||||
|
||||
@ -206,6 +219,77 @@ class Audiobook {
|
||||
this.audioFiles.forEach((file) => {
|
||||
this.addTrack(file)
|
||||
})
|
||||
this.lastUpdate = Date.now()
|
||||
}
|
||||
|
||||
removeAudioFile(audioFile) {
|
||||
this.tracks = this.tracks.filter(t => t.path !== audioFile.path)
|
||||
this.audioFiles = this.audioFiles.filter(f => f.path !== audioFile.path)
|
||||
}
|
||||
|
||||
audioPartExists(part) {
|
||||
var path = Path.join(this.path, part)
|
||||
return this.audioFiles.find(file => file.path === path)
|
||||
}
|
||||
|
||||
checkUpdateMissingParts() {
|
||||
var currMissingParts = this.missingParts.join(',')
|
||||
|
||||
var current_index = 1
|
||||
var missingParts = []
|
||||
for (let i = 0; i < this.tracks.length; i++) {
|
||||
var _track = this.tracks[i]
|
||||
if (_track.index > current_index) {
|
||||
var num_parts_missing = _track.index - current_index
|
||||
for (let x = 0; x < num_parts_missing; x++) {
|
||||
missingParts.push(current_index + x)
|
||||
}
|
||||
}
|
||||
current_index = _track.index + 1
|
||||
}
|
||||
|
||||
this.missingParts = missingParts
|
||||
|
||||
var wasUpdated = this.missingParts.join(',') !== currMissingParts
|
||||
if (wasUpdated && this.missingParts.length) {
|
||||
Logger.info(`[Audiobook] "${this.title}" has ${missingParts.length} missing parts`)
|
||||
}
|
||||
|
||||
return wasUpdated
|
||||
}
|
||||
|
||||
// On scan check other files found with other files saved
|
||||
syncOtherFiles(newOtherFiles) {
|
||||
var currOtherFileNum = this.otherFiles.length
|
||||
|
||||
var newOtherFilePaths = newOtherFiles.map(f => f.path)
|
||||
this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
|
||||
newOtherFiles.forEach((file) => {
|
||||
var existingOtherFile = this.otherFiles.find(f => f.path === file.path)
|
||||
if (!existingOtherFile) {
|
||||
Logger.info(`[Audiobook] New other file found on sync ${file.filename}/${file.filetype} | "${this.title}"`)
|
||||
this.otherFiles.push(file)
|
||||
}
|
||||
})
|
||||
|
||||
var hasUpdates = currOtherFileNum !== this.otherFiles.length
|
||||
|
||||
var imageFiles = this.otherFiles.filter(f => f.filetype === 'image')
|
||||
if (this.book.cover && this.book.cover.substr(1).startsWith('local')) {
|
||||
var coverStillExists = imageFiles.find(f => comparePaths(f.path, this.book.cover.substr('/local/'.length)))
|
||||
if (!coverStillExists) {
|
||||
Logger.info(`[Audiobook] Local cover was removed | "${this.title}"`)
|
||||
this.book.cover = null
|
||||
hasUpdates = true
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.book.cover && imageFiles.length) {
|
||||
this.book.cover = Path.join('/local', imageFiles[0].path)
|
||||
Logger.info(`[Audiobook] Local cover was set | "${this.title}"`)
|
||||
hasUpdates = true
|
||||
}
|
||||
return hasUpdates
|
||||
}
|
||||
|
||||
isSearchMatch(search) {
|
||||
|
@ -104,8 +104,10 @@ class Db {
|
||||
updateAudiobook(audiobook) {
|
||||
return this.audiobooksDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => {
|
||||
Logger.debug(`[DB] Audiobook updated ${results.updated}`)
|
||||
return true
|
||||
}).catch((error) => {
|
||||
Logger.error(`[DB] Audiobook update failed ${error}`)
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -20,41 +20,121 @@ class Scanner {
|
||||
}
|
||||
|
||||
async scan() {
|
||||
// console.log('Start scan audiobooks', this.audiobooks.map(a => a.fullPath).join(', '))
|
||||
const scanStart = Date.now()
|
||||
var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath)
|
||||
|
||||
var scanResults = {
|
||||
removed: 0,
|
||||
updated: 0,
|
||||
added: 0
|
||||
}
|
||||
|
||||
// Check for removed audiobooks
|
||||
for (let i = 0; i < this.audiobooks.length; i++) {
|
||||
var dataFound = audiobookDataFound.find(abd => abd.path === this.audiobooks[i].path)
|
||||
if (!dataFound) {
|
||||
Logger.info(`[Scanner] Removing audiobook "${this.audiobooks[i].title}" - no longer in dir`)
|
||||
|
||||
await this.db.removeEntity('audiobook', this.audiobooks[i].id)
|
||||
if (!this.audiobooks[i]) {
|
||||
Logger.error('[Scanner] Oops... audiobook is now invalid...')
|
||||
continue;
|
||||
}
|
||||
scanResults.removed++
|
||||
this.emitter('audiobook_removed', this.audiobooks[i].toJSONMinified())
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < audiobookDataFound.length; i++) {
|
||||
var audiobookData = audiobookDataFound[i]
|
||||
if (!audiobookData.parts.length) {
|
||||
Logger.error('No Valid Parts for Audiobook', audiobookData)
|
||||
} else {
|
||||
var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
|
||||
if (existingAudiobook) {
|
||||
Logger.info('Audiobook already added', audiobookData.title)
|
||||
// Todo: Update Audiobook here
|
||||
var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
|
||||
if (existingAudiobook) {
|
||||
Logger.debug(`[Scanner] Audiobook already added, check updates for "${existingAudiobook.title}"`)
|
||||
|
||||
if (!audiobookData.parts.length) {
|
||||
Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
|
||||
|
||||
await this.db.removeEntity('audiobook', existingAudiobook.id)
|
||||
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
|
||||
scanResults.removed++
|
||||
} else {
|
||||
|
||||
// Check for audio files that were removed
|
||||
var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !audiobookData.parts.includes(file.filename))
|
||||
if (removedAudioFiles.length) {
|
||||
Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
|
||||
removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
|
||||
}
|
||||
|
||||
// Check for audio files that were added
|
||||
var newParts = audiobookData.parts.filter(part => !existingAudiobook.audioPartExists(part))
|
||||
if (newParts.length) {
|
||||
Logger.info(`[Scanner] ${newParts.length} new audio parts were found for audiobook "${existingAudiobook.title}"`)
|
||||
|
||||
// If previously invalid part, remove from invalid list because it will be re-scanned
|
||||
newParts.forEach((part) => {
|
||||
if (existingAudiobook.invalidParts.includes(part)) {
|
||||
existingAudiobook.invalidParts = existingAudiobook.invalidParts.filter(p => p !== part)
|
||||
}
|
||||
})
|
||||
// Scan new audio parts found
|
||||
await audioFileScanner.scanParts(existingAudiobook, newParts)
|
||||
}
|
||||
|
||||
if (!existingAudiobook.tracks.length) {
|
||||
Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
|
||||
|
||||
await this.db.removeEntity('audiobook', existingAudiobook.id)
|
||||
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
|
||||
} else {
|
||||
var hasUpdates = removedAudioFiles.length || newParts.length
|
||||
|
||||
if (existingAudiobook.checkUpdateMissingParts()) {
|
||||
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
|
||||
hasUpdates = true
|
||||
}
|
||||
|
||||
if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
|
||||
hasUpdates = true
|
||||
}
|
||||
|
||||
if (hasUpdates) {
|
||||
Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
|
||||
existingAudiobook.lastUpdate = Date.now()
|
||||
await this.db.updateAudiobook(existingAudiobook)
|
||||
this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
|
||||
scanResults.updated++
|
||||
}
|
||||
}
|
||||
} // end if update existing
|
||||
} else {
|
||||
if (!audiobookData.parts.length) {
|
||||
Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData)
|
||||
} else {
|
||||
// console.log('Audiobook not already there... add new audiobook', audiobookData.fullPath)
|
||||
var audiobook = new Audiobook()
|
||||
audiobook.setData(audiobookData)
|
||||
await audioFileScanner.scanParts(audiobook, audiobookData.parts)
|
||||
if (!audiobook.tracks.length) {
|
||||
Logger.warn('Invalid audiobook, no valid tracks', audiobook.title)
|
||||
Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
|
||||
} else {
|
||||
Logger.info('Audiobook Scanned', audiobook.title, `(${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
|
||||
audiobook.checkUpdateMissingParts()
|
||||
Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
|
||||
await this.db.insertAudiobook(audiobook)
|
||||
this.emitter('audiobook_added', audiobook.toJSONMinified())
|
||||
scanResults.added++
|
||||
}
|
||||
}
|
||||
var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
|
||||
this.emitter('scan_progress', {
|
||||
total: audiobookDataFound.length,
|
||||
done: i + 1,
|
||||
progress
|
||||
})
|
||||
} // end if add new
|
||||
}
|
||||
var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
|
||||
this.emitter('scan_progress', {
|
||||
total: audiobookDataFound.length,
|
||||
done: i + 1,
|
||||
progress
|
||||
})
|
||||
}
|
||||
const scanElapsed = Math.floor((Date.now() - scanStart) / 1000)
|
||||
Logger.info(`[SCANNER] Finished ${secondsToTimestamp(scanElapsed)}`)
|
||||
Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | elapsed: ${secondsToTimestamp(scanElapsed)}`)
|
||||
return scanResults
|
||||
}
|
||||
|
||||
async fetchMetadata(id, trackIndex = 0) {
|
||||
|
@ -53,33 +53,26 @@ class Server {
|
||||
}
|
||||
|
||||
emitter(ev, data) {
|
||||
Logger.debug('EMITTER', ev)
|
||||
if (!this.io) {
|
||||
Logger.error('Invalid IO')
|
||||
return
|
||||
}
|
||||
// Logger.debug('EMITTER', ev)
|
||||
this.io.emit(ev, data)
|
||||
}
|
||||
|
||||
async fileAddedUpdated({ path, fullPath }) {
|
||||
Logger.info('[SERVER] FileAddedUpdated', path, fullPath)
|
||||
}
|
||||
|
||||
async fileAddedUpdated({ path, fullPath }) { }
|
||||
async fileRemoved({ path, fullPath }) { }
|
||||
|
||||
async scan() {
|
||||
Logger.info('[SERVER] Starting Scan')
|
||||
Logger.info('[Server] Starting Scan')
|
||||
this.isScanning = true
|
||||
this.isInitialized = true
|
||||
this.emitter('scan_start')
|
||||
await this.scanner.scan()
|
||||
var results = await this.scanner.scan()
|
||||
this.isScanning = false
|
||||
this.emitter('scan_complete')
|
||||
Logger.info('[SERVER] Scan complete')
|
||||
this.emitter('scan_complete', results)
|
||||
Logger.info('[Server] Scan complete')
|
||||
}
|
||||
|
||||
async init() {
|
||||
Logger.info('[SERVER] Init')
|
||||
Logger.info('[Server] Init')
|
||||
await this.streamManager.removeOrphanStreams()
|
||||
await this.db.init()
|
||||
this.auth.init()
|
||||
|
@ -129,7 +129,6 @@ class Stream extends EventEmitter {
|
||||
async generatePlaylist() {
|
||||
fs.ensureDirSync(this.streamPath)
|
||||
await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength)
|
||||
console.log('Playlist generated')
|
||||
return this.clientPlaylistUri
|
||||
}
|
||||
|
||||
|
@ -45,7 +45,7 @@ class FolderWatcher extends EventEmitter {
|
||||
}
|
||||
|
||||
onNewFile(path) {
|
||||
Logger.info('FolderWatcher: New File', path)
|
||||
Logger.debug('FolderWatcher: New File', path)
|
||||
this.emit('file_added', {
|
||||
path: path.replace(this.AudiobookPath, ''),
|
||||
fullPath: path
|
||||
@ -53,7 +53,7 @@ class FolderWatcher extends EventEmitter {
|
||||
}
|
||||
|
||||
onFileRemoved(path) {
|
||||
Logger.info('FolderWatcher: File Removed', path)
|
||||
Logger.debug('FolderWatcher: File Removed', path)
|
||||
this.emit('file_removed', {
|
||||
path: path.replace(this.AudiobookPath, ''),
|
||||
fullPath: path
|
||||
@ -61,7 +61,7 @@ class FolderWatcher extends EventEmitter {
|
||||
}
|
||||
|
||||
onFileUpdated(path) {
|
||||
Logger.info('FolderWatcher: Updated File', path)
|
||||
Logger.debug('FolderWatcher: Updated File', path)
|
||||
this.emit('file_updated', {
|
||||
path: path.replace(this.AudiobookPath, ''),
|
||||
fullPath: path
|
||||
|
@ -78,7 +78,7 @@ function getTrackNumberFromFilename(filename) {
|
||||
|
||||
async function scanParts(audiobook, parts) {
|
||||
if (!parts || !parts.length) {
|
||||
Logger.error('Scan Parts', audiobook.title, 'No Parts', parts)
|
||||
Logger.error('[AudioFileScanner] Scan Parts', audiobook.title, 'No Parts', parts)
|
||||
return
|
||||
}
|
||||
var tracks = []
|
||||
@ -87,7 +87,7 @@ async function scanParts(audiobook, parts) {
|
||||
|
||||
var scanData = await scan(fullPath)
|
||||
if (!scanData || scanData.error) {
|
||||
Logger.error('Scan failed for', parts[i])
|
||||
Logger.error('[AudioFileScanner] Scan failed for', parts[i])
|
||||
audiobook.invalidParts.push(parts[i])
|
||||
continue;
|
||||
}
|
||||
@ -110,7 +110,7 @@ async function scanParts(audiobook, parts) {
|
||||
if (parts.length > 1) {
|
||||
trackNumber = isNumber(trackNumFromMeta) ? trackNumFromMeta : trackNumFromFilename
|
||||
if (trackNumber === null) {
|
||||
Logger.error('Invalid track number for', parts[i])
|
||||
Logger.error('[AudioFileScanner] Invalid track number for', parts[i])
|
||||
audioFileObj.invalid = true
|
||||
audioFileObj.error = 'Failed to get track number'
|
||||
continue;
|
||||
@ -118,7 +118,7 @@ async function scanParts(audiobook, parts) {
|
||||
}
|
||||
|
||||
if (tracks.find(t => t.index === trackNumber)) {
|
||||
Logger.error('Duplicate track number for', parts[i])
|
||||
Logger.error('[AudioFileScanner] Duplicate track number for', parts[i])
|
||||
audioFileObj.invalid = true
|
||||
audioFileObj.error = 'Duplicate track number'
|
||||
continue;
|
||||
@ -129,7 +129,7 @@ async function scanParts(audiobook, parts) {
|
||||
}
|
||||
|
||||
if (!tracks.length) {
|
||||
Logger.warn('No Tracks for audiobook', audiobook.id)
|
||||
Logger.warn('[AudioFileScanner] No Tracks for audiobook', audiobook.id)
|
||||
return
|
||||
}
|
||||
|
||||
@ -148,26 +148,12 @@ async function scanParts(audiobook, parts) {
|
||||
})
|
||||
}
|
||||
|
||||
var parts_copy = tracks.map(p => ({ ...p }))
|
||||
var current_index = 1
|
||||
|
||||
for (let i = 0; i < parts_copy.length; i++) {
|
||||
var cleaned_part = parts_copy[i]
|
||||
if (cleaned_part.index > current_index) {
|
||||
var num_parts_missing = cleaned_part.index - current_index
|
||||
for (let x = 0; x < num_parts_missing; x++) {
|
||||
audiobook.missingParts.push(current_index + x)
|
||||
}
|
||||
}
|
||||
current_index = cleaned_part.index + 1
|
||||
}
|
||||
|
||||
if (audiobook.missingParts.length) {
|
||||
Logger.info('Audiobook', audiobook.title, 'Has missing parts', audiobook.missingParts)
|
||||
}
|
||||
|
||||
var hasTracksAlready = audiobook.tracks.length
|
||||
tracks.forEach((track) => {
|
||||
audiobook.addTrack(track)
|
||||
})
|
||||
if (hasTracksAlready) {
|
||||
audiobook.tracks.sort((a, b) => a.index - b.index)
|
||||
}
|
||||
}
|
||||
module.exports.scanParts = scanParts
|
@ -1,3 +1,5 @@
|
||||
const Path = require('path')
|
||||
|
||||
const levenshteinDistance = (str1, str2, caseSensitive = false) => {
|
||||
if (!caseSensitive) {
|
||||
str1 = str1.toLowerCase()
|
||||
@ -44,4 +46,26 @@ module.exports.cleanString = cleanString
|
||||
|
||||
module.exports.isObject = (val) => {
|
||||
return val !== null && typeof val === 'object'
|
||||
}
|
||||
|
||||
function normalizePath(path) {
|
||||
const replace = [
|
||||
[/\\/g, '/'],
|
||||
[/(\w):/, '/$1'],
|
||||
[/(\w+)\/\.\.\/?/g, ''],
|
||||
[/^\.\//, ''],
|
||||
[/\/\.\//, '/'],
|
||||
[/\/\.$/, ''],
|
||||
[/\/$/, ''],
|
||||
]
|
||||
replace.forEach(array => {
|
||||
while (array[0].test(path)) {
|
||||
path = path.replace(array[0], array[1])
|
||||
}
|
||||
})
|
||||
return path
|
||||
}
|
||||
|
||||
module.exports.comparePaths = (path1, path2) => {
|
||||
return (path1 === path2) || (normalizePath(path1) === normalizePath(path2))
|
||||
}
|
@ -69,7 +69,7 @@ async function getAllAudiobookFiles(abRootPath) {
|
||||
title: title,
|
||||
series: cleanString(series),
|
||||
publishYear: publishYear,
|
||||
path: relpath,
|
||||
path: path,
|
||||
fullPath: Path.join(abRootPath, path),
|
||||
parts: [],
|
||||
otherFiles: []
|
||||
|
Loading…
Reference in New Issue
Block a user