Scanner update - remove and update audiobooks on scans

client/layouts/default.vue

@@ -83,9 +83,15 @@ export default {
       }
       this.$store.commit('audiobooks/remove', audiobook)
     },
-    scanComplete() {
+    scanComplete(results) {
+      if (!results) results = {}
       this.$store.commit('setIsScanning', false)
-      this.$toast.success('Scan Finished')
+      var scanResultMsgs = []
+      if (results.added) scanResultMsgs.push(`${results.added} added`)
+      if (results.updated) scanResultMsgs.push(`${results.updated} updated`)
+      if (results.removed) scanResultMsgs.push(`${results.removed} removed`)
+      if (!scanResultMsgs.length) this.$toast.success('Scan Finished\nEverything was up to date')
+      else this.$toast.success('Scan Finished\n' + scanResultMsgs.join('\n'))
     },
     scanStart() {
       this.$store.commit('setIsScanning', true)

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf-client",
-  "version": "0.9.71-beta",
+  "version": "0.9.72-beta",
   "description": "Audiobook manager and player",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "audiobookshelf",
-  "version": "0.9.71-beta",
+  "version": "0.9.72-beta",
   "description": "Self-hosted audiobook server for managing and playing audiobooks.",
   "main": "index.js",
   "scripts": {

server/Audiobook.js

@@ -1,4 +1,7 @@
+const Path = require('path')
 const { bytesPretty, elapsedPretty } = require('./utils/fileUtils')
+const { comparePaths } = require('./utils/index')
+const Logger = require('./Logger')
 const Book = require('./Book')
 const AudioTrack = require('./AudioTrack')
 
@@ -8,6 +11,7 @@ class Audiobook {
     this.path = null
     this.fullPath = null
     this.addedAt = null
+    this.lastUpdate = null
 
     this.tracks = []
     this.missingParts = []
@@ -29,6 +33,7 @@ class Audiobook {
     this.path = audiobook.path
     this.fullPath = audiobook.fullPath
     this.addedAt = audiobook.addedAt
+    this.lastUpdate = audiobook.lastUpdate || this.addedAt
 
     this.tracks = audiobook.tracks.map(track => {
       return new AudioTrack(track)
@@ -99,6 +104,7 @@ class Audiobook {
       path: this.path,
       fullPath: this.fullPath,
       addedAt: this.addedAt,
+      lastUpdate: this.lastUpdate,
       missingParts: this.missingParts,
       invalidParts: this.invalidParts,
       tags: this.tags,
@@ -117,6 +123,7 @@ class Audiobook {
       path: this.path,
       fullPath: this.fullPath,
       addedAt: this.addedAt,
+      lastUpdate: this.lastUpdate,
       duration: this.totalDuration,
       size: this.totalSize,
       hasBookMatch: !!this.book,
@@ -135,6 +142,7 @@ class Audiobook {
       path: this.path,
       fullPath: this.fullPath,
       addedAt: this.addedAt,
+      lastUpdate: this.lastUpdate,
       duration: this.totalDuration,
       durationPretty: this.durationPretty,
       size: this.totalSize,
@@ -154,6 +162,7 @@ class Audiobook {
     this.path = data.path
     this.fullPath = data.fullPath
     this.addedAt = Date.now()
+    this.lastUpdate = this.addedAt
 
     this.otherFiles = data.otherFiles || []
     this.setBook(data)
@@ -188,6 +197,10 @@ class Audiobook {
       }
     }
 
+    if (hasUpdates) {
+      this.lastUpdate = Date.now()
+    }
+
     return hasUpdates
   }
 
@@ -206,6 +219,77 @@ class Audiobook {
     this.audioFiles.forEach((file) => {
       this.addTrack(file)
     })
+    this.lastUpdate = Date.now()
+  }
+
+  removeAudioFile(audioFile) {
+    this.tracks = this.tracks.filter(t => t.path !== audioFile.path)
+    this.audioFiles = this.audioFiles.filter(f => f.path !== audioFile.path)
+  }
+
+  audioPartExists(part) {
+    var path = Path.join(this.path, part)
+    return this.audioFiles.find(file => file.path === path)
+  }
+
+  checkUpdateMissingParts() {
+    var currMissingParts = this.missingParts.join(',')
+
+    var current_index = 1
+    var missingParts = []
+    for (let i = 0; i < this.tracks.length; i++) {
+      var _track = this.tracks[i]
+      if (_track.index > current_index) {
+        var num_parts_missing = _track.index - current_index
+        for (let x = 0; x < num_parts_missing; x++) {
+          missingParts.push(current_index + x)
+        }
+      }
+      current_index = _track.index + 1
+    }
+
+    this.missingParts = missingParts
+
+    var wasUpdated = this.missingParts.join(',') !== currMissingParts
+    if (wasUpdated && this.missingParts.length) {
+      Logger.info(`[Audiobook] "${this.title}" has ${missingParts.length} missing parts`)
+    }
+
+    return wasUpdated
+  }
+
+  // On scan check other files found with other files saved
+  syncOtherFiles(newOtherFiles) {
+    var currOtherFileNum = this.otherFiles.length
+
+    var newOtherFilePaths = newOtherFiles.map(f => f.path)
+    this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
+    newOtherFiles.forEach((file) => {
+      var existingOtherFile = this.otherFiles.find(f => f.path === file.path)
+      if (!existingOtherFile) {
+        Logger.info(`[Audiobook] New other file found on sync ${file.filename}/${file.filetype} | "${this.title}"`)
+        this.otherFiles.push(file)
+      }
+    })
+
+    var hasUpdates = currOtherFileNum !== this.otherFiles.length
+
+    var imageFiles = this.otherFiles.filter(f => f.filetype === 'image')
+    if (this.book.cover && this.book.cover.substr(1).startsWith('local')) {
+      var coverStillExists = imageFiles.find(f => comparePaths(f.path, this.book.cover.substr('/local/'.length)))
+      if (!coverStillExists) {
+        Logger.info(`[Audiobook] Local cover was removed | "${this.title}"`)
+        this.book.cover = null
+        hasUpdates = true
+      }
+    }
+
+    if (!this.book.cover && imageFiles.length) {
+      this.book.cover = Path.join('/local', imageFiles[0].path)
+      Logger.info(`[Audiobook] Local cover was set | "${this.title}"`)
+      hasUpdates = true
+    }
+    return hasUpdates
   }
 
   isSearchMatch(search) {

server/Db.js

@@ -104,8 +104,10 @@ class Db {
   updateAudiobook(audiobook) {
     return this.audiobooksDb.update((record) => record.id === audiobook.id, () => audiobook).then((results) => {
       Logger.debug(`[DB] Audiobook updated ${results.updated}`)
+      return true
     }).catch((error) => {
       Logger.error(`[DB] Audiobook update failed ${error}`)
+      return false
     })
   }
 

server/Scanner.js

@@ -20,41 +20,121 @@ class Scanner {
   }
 
   async scan() {
-    // console.log('Start scan audiobooks', this.audiobooks.map(a => a.fullPath).join(', '))
     const scanStart = Date.now()
     var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath)
+
+    var scanResults = {
+      removed: 0,
+      updated: 0,
+      added: 0
+    }
+
+    // Check for removed audiobooks
+    for (let i = 0; i < this.audiobooks.length; i++) {
+      var dataFound = audiobookDataFound.find(abd => abd.path === this.audiobooks[i].path)
+      if (!dataFound) {
+        Logger.info(`[Scanner] Removing audiobook "${this.audiobooks[i].title}" - no longer in dir`)
+
+        await this.db.removeEntity('audiobook', this.audiobooks[i].id)
+        if (!this.audiobooks[i]) {
+          Logger.error('[Scanner] Oops... audiobook is now invalid...')
+          continue;
+        }
+        scanResults.removed++
+        this.emitter('audiobook_removed', this.audiobooks[i].toJSONMinified())
+      }
+    }
+
     for (let i = 0; i < audiobookDataFound.length; i++) {
       var audiobookData = audiobookDataFound[i]
-      if (!audiobookData.parts.length) {
-        Logger.error('No Valid Parts for Audiobook', audiobookData)
-      } else {
-        var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
-        if (existingAudiobook) {
-          Logger.info('Audiobook already added', audiobookData.title)
-          // Todo: Update Audiobook here
+      var existingAudiobook = this.audiobooks.find(a => a.fullPath === audiobookData.fullPath)
+      if (existingAudiobook) {
+        Logger.debug(`[Scanner] Audiobook already added, check updates for "${existingAudiobook.title}"`)
+
+        if (!audiobookData.parts.length) {
+          Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
+
+          await this.db.removeEntity('audiobook', existingAudiobook.id)
+          this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
+          scanResults.removed++
+        } else {
+
+          // Check for audio files that were removed
+          var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !audiobookData.parts.includes(file.filename))
+          if (removedAudioFiles.length) {
+            Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
+            removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
+          }
+
+          // Check for audio files that were added
+          var newParts = audiobookData.parts.filter(part => !existingAudiobook.audioPartExists(part))
+          if (newParts.length) {
+            Logger.info(`[Scanner] ${newParts.length} new audio parts were found for audiobook "${existingAudiobook.title}"`)
+
+            // If previously invalid part, remove from invalid list because it will be re-scanned
+            newParts.forEach((part) => {
+              if (existingAudiobook.invalidParts.includes(part)) {
+                existingAudiobook.invalidParts = existingAudiobook.invalidParts.filter(p => p !== part)
+              }
+            })
+            // Scan new audio parts found
+            await audioFileScanner.scanParts(existingAudiobook, newParts)
+          }
+
+          if (!existingAudiobook.tracks.length) {
+            Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
+
+            await this.db.removeEntity('audiobook', existingAudiobook.id)
+            this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
+          } else {
+            var hasUpdates = removedAudioFiles.length || newParts.length
+
+            if (existingAudiobook.checkUpdateMissingParts()) {
+              Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
+              hasUpdates = true
+            }
+
+            if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
+              hasUpdates = true
+            }
+
+            if (hasUpdates) {
+              Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
+              existingAudiobook.lastUpdate = Date.now()
+              await this.db.updateAudiobook(existingAudiobook)
+              this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
+              scanResults.updated++
+            }
+          }
+        } // end if update existing
+      } else {
+        if (!audiobookData.parts.length) {
+          Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData)
         } else {
-          // console.log('Audiobook not already there... add new audiobook', audiobookData.fullPath)
           var audiobook = new Audiobook()
           audiobook.setData(audiobookData)
           await audioFileScanner.scanParts(audiobook, audiobookData.parts)
           if (!audiobook.tracks.length) {
-            Logger.warn('Invalid audiobook, no valid tracks', audiobook.title)
+            Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
           } else {
-            Logger.info('Audiobook Scanned', audiobook.title, `(${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
+            audiobook.checkUpdateMissingParts()
+            Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
             await this.db.insertAudiobook(audiobook)
             this.emitter('audiobook_added', audiobook.toJSONMinified())
+            scanResults.added++
           }
-        }
-        var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
-        this.emitter('scan_progress', {
-          total: audiobookDataFound.length,
-          done: i + 1,
-          progress
-        })
+        } // end if add new
       }
+      var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
+      this.emitter('scan_progress', {
+        total: audiobookDataFound.length,
+        done: i + 1,
+        progress
+      })
     }
     const scanElapsed = Math.floor((Date.now() - scanStart) / 1000)
-    Logger.info(`[SCANNER] Finished ${secondsToTimestamp(scanElapsed)}`)
+    Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | elapsed: ${secondsToTimestamp(scanElapsed)}`)
+    return scanResults
   }
 
   async fetchMetadata(id, trackIndex = 0) {

server/Server.js

@@ -53,33 +53,26 @@ class Server {
   }
 
   emitter(ev, data) {
-    Logger.debug('EMITTER', ev)
-    if (!this.io) {
-      Logger.error('Invalid IO')
-      return
-    }
+    // Logger.debug('EMITTER', ev)
     this.io.emit(ev, data)
   }
 
-  async fileAddedUpdated({ path, fullPath }) {
-    Logger.info('[SERVER] FileAddedUpdated', path, fullPath)
-  }
-
+  async fileAddedUpdated({ path, fullPath }) { }
   async fileRemoved({ path, fullPath }) { }
 
   async scan() {
-    Logger.info('[SERVER] Starting Scan')
+    Logger.info('[Server] Starting Scan')
     this.isScanning = true
     this.isInitialized = true
     this.emitter('scan_start')
-    await this.scanner.scan()
+    var results = await this.scanner.scan()
     this.isScanning = false
-    this.emitter('scan_complete')
-    Logger.info('[SERVER] Scan complete')
+    this.emitter('scan_complete', results)
+    Logger.info('[Server] Scan complete')
   }
 
   async init() {
-    Logger.info('[SERVER] Init')
+    Logger.info('[Server] Init')
     await this.streamManager.removeOrphanStreams()
     await this.db.init()
     this.auth.init()

server/Stream.js

@@ -129,7 +129,6 @@ class Stream extends EventEmitter {
   async generatePlaylist() {
     fs.ensureDirSync(this.streamPath)
     await hlsPlaylistGenerator(this.playlistPath, 'output', this.totalDuration, this.segmentLength)
-    console.log('Playlist generated')
     return this.clientPlaylistUri
   }
 

server/Watcher.js

@@ -45,7 +45,7 @@ class FolderWatcher extends EventEmitter {
   }
 
   onNewFile(path) {
-    Logger.info('FolderWatcher: New File', path)
+    Logger.debug('FolderWatcher: New File', path)
     this.emit('file_added', {
       path: path.replace(this.AudiobookPath, ''),
       fullPath: path
@@ -53,7 +53,7 @@ class FolderWatcher extends EventEmitter {
   }
 
   onFileRemoved(path) {
-    Logger.info('FolderWatcher: File Removed', path)
+    Logger.debug('FolderWatcher: File Removed', path)
     this.emit('file_removed', {
       path: path.replace(this.AudiobookPath, ''),
       fullPath: path
@@ -61,7 +61,7 @@ class FolderWatcher extends EventEmitter {
   }
 
   onFileUpdated(path) {
-    Logger.info('FolderWatcher: Updated File', path)
+    Logger.debug('FolderWatcher: Updated File', path)
     this.emit('file_updated', {
       path: path.replace(this.AudiobookPath, ''),
       fullPath: path

server/utils/audioFileScanner.js

@@ -78,7 +78,7 @@ function getTrackNumberFromFilename(filename) {
 
 async function scanParts(audiobook, parts) {
   if (!parts || !parts.length) {
-    Logger.error('Scan Parts', audiobook.title, 'No Parts', parts)
+    Logger.error('[AudioFileScanner] Scan Parts', audiobook.title, 'No Parts', parts)
     return
   }
   var tracks = []
@@ -87,7 +87,7 @@ async function scanParts(audiobook, parts) {
 
     var scanData = await scan(fullPath)
     if (!scanData || scanData.error) {
-      Logger.error('Scan failed for', parts[i])
+      Logger.error('[AudioFileScanner] Scan failed for', parts[i])
       audiobook.invalidParts.push(parts[i])
       continue;
     }
@@ -110,7 +110,7 @@ async function scanParts(audiobook, parts) {
     if (parts.length > 1) {
       trackNumber = isNumber(trackNumFromMeta) ? trackNumFromMeta : trackNumFromFilename
       if (trackNumber === null) {
-        Logger.error('Invalid track number for', parts[i])
+        Logger.error('[AudioFileScanner] Invalid track number for', parts[i])
         audioFileObj.invalid = true
         audioFileObj.error = 'Failed to get track number'
         continue;
@@ -118,7 +118,7 @@ async function scanParts(audiobook, parts) {
     }
 
     if (tracks.find(t => t.index === trackNumber)) {
-      Logger.error('Duplicate track number for', parts[i])
+      Logger.error('[AudioFileScanner] Duplicate track number for', parts[i])
       audioFileObj.invalid = true
       audioFileObj.error = 'Duplicate track number'
       continue;
@@ -129,7 +129,7 @@ async function scanParts(audiobook, parts) {
   }
 
   if (!tracks.length) {
-    Logger.warn('No Tracks for audiobook', audiobook.id)
+    Logger.warn('[AudioFileScanner] No Tracks for audiobook', audiobook.id)
     return
   }
 
@@ -148,26 +148,12 @@ async function scanParts(audiobook, parts) {
     })
   }
 
-  var parts_copy = tracks.map(p => ({ ...p }))
-  var current_index = 1
-
-  for (let i = 0; i < parts_copy.length; i++) {
-    var cleaned_part = parts_copy[i]
-    if (cleaned_part.index > current_index) {
-      var num_parts_missing = cleaned_part.index - current_index
-      for (let x = 0; x < num_parts_missing; x++) {
-        audiobook.missingParts.push(current_index + x)
-      }
-    }
-    current_index = cleaned_part.index + 1
-  }
-
-  if (audiobook.missingParts.length) {
-    Logger.info('Audiobook', audiobook.title, 'Has missing parts', audiobook.missingParts)
-  }
-
+  var hasTracksAlready = audiobook.tracks.length
   tracks.forEach((track) => {
     audiobook.addTrack(track)
   })
+  if (hasTracksAlready) {
+    audiobook.tracks.sort((a, b) => a.index - b.index)
+  }
 }
 module.exports.scanParts = scanParts

server/utils/index.js

@@ -1,3 +1,5 @@
+const Path = require('path')
+
 const levenshteinDistance = (str1, str2, caseSensitive = false) => {
   if (!caseSensitive) {
     str1 = str1.toLowerCase()
@@ -45,3 +47,25 @@ module.exports.cleanString = cleanString
 module.exports.isObject = (val) => {
   return val !== null && typeof val === 'object'
 }
+
+function normalizePath(path) {
+  const replace = [
+    [/\\/g, '/'],
+    [/(\w):/, '/$1'],
+    [/(\w+)\/\.\.\/?/g, ''],
+    [/^\.\//, ''],
+    [/\/\.\//, '/'],
+    [/\/\.$/, ''],
+    [/\/$/, ''],
+  ]
+  replace.forEach(array => {
+    while (array[0].test(path)) {
+      path = path.replace(array[0], array[1])
+    }
+  })
+  return path
+}
+
+module.exports.comparePaths = (path1, path2) => {
+  return (path1 === path2) || (normalizePath(path1) === normalizePath(path2))
+}

server/utils/scandir.js

@@ -69,7 +69,7 @@ async function getAllAudiobookFiles(abRootPath) {
         title: title,
         series: cleanString(series),
         publishYear: publishYear,
-        path: relpath,
+        path: path,
         fullPath: Path.join(abRootPath, path),
         parts: [],
         otherFiles: []