Add:New scanner and scanner server settings

client/layouts/default.vue

@@ -104,11 +104,23 @@ export default {
       if (payload.serverSettings) {
         this.$store.commit('setServerSettings', payload.serverSettings)
       }
+
+      // Start scans currently running
       if (payload.librariesScanning) {
         payload.librariesScanning.forEach((libraryScan) => {
           this.scanStart(libraryScan)
         })
       }
+
+      // Remove any current scans that are no longer running
+      var currentScans = [...this.$store.state.scanners.libraryScans]
+      currentScans.forEach((ls) => {
+        if (!payload.librariesScanning || !payload.librariesScanning.find((_ls) => _ls.id === ls.id)) {
+          this.$toast.dismiss(ls.toastId)
+          this.$store.commit('scanners/remove', ls)
+        }
+      })
+
       if (payload.backups && payload.backups.length) {
         this.$store.commit('setBackups', payload.backups)
       }
@@ -152,6 +164,16 @@ export default {
       }
       this.$store.commit('audiobooks/remove', audiobook)
     },
+    audiobooksAdded(audiobooks) {
+      audiobooks.forEach((ab) => {
+        this.$store.commit('audiobooks/addUpdate', ab)
+      })
+    },
+    audiobooksUpdated(audiobooks) {
+      audiobooks.forEach((ab) => {
+        this.$store.commit('audiobooks/addUpdate', ab)
+      })
+    },
     libraryAdded(library) {
       this.$store.commit('libraries/addUpdate', library)
     },
@@ -162,6 +184,8 @@ export default {
       this.$store.commit('libraries/remove', library)
     },
     scanComplete(data) {
+      console.log('Scan complete received', data)
+
       var message = `Scan "${data.name}" complete!`
       if (data.results) {
         var scanResultMsgs = []
@@ -337,6 +361,8 @@ export default {
       this.socket.on('audiobook_updated', this.audiobookUpdated)
       this.socket.on('audiobook_added', this.audiobookAdded)
       this.socket.on('audiobook_removed', this.audiobookRemoved)
+      this.socket.on('audiobooks_updated', this.audiobooksUpdated)
+      this.socket.on('audiobooks_added', this.audiobooksAdded)
 
       // Library Listeners
       this.socket.on('library_updated', this.libraryUpdated)

client/pages/config/index.vue

@@ -21,6 +21,20 @@
         </ui-tooltip>
       </div>
 
+      <div class="flex items-center py-2">
+        <ui-toggle-switch v-model="newServerSettings.scannerPreferAudioMetadata" :disabled="updatingServerSettings" @input="updateScannerPreferAudioMeta" />
+        <ui-tooltip :text="scannerPreferAudioMetaTooltip">
+          <p class="pl-4 text-lg">Scanner prefer audio metadata <span class="material-icons icon-text">info_outlined</span></p>
+        </ui-tooltip>
+      </div>
+
+      <div class="flex items-center py-2">
+        <ui-toggle-switch v-model="newServerSettings.scannerPreferOpfMetadata" :disabled="updatingServerSettings" @input="updateScannerPreferOpfMeta" />
+        <ui-tooltip :text="scannerPreferOpfMetaTooltip">
+          <p class="pl-4 text-lg">Scanner prefer OPF metadata <span class="material-icons icon-text">info_outlined</span></p>
+        </ui-tooltip>
+      </div>
+
       <div class="flex items-center py-2">
         <ui-toggle-switch v-model="storeCoversInAudiobookDir" :disabled="updatingServerSettings" @input="updateCoverStorageDestination" />
         <ui-tooltip :text="coverDestinationTooltip">
@@ -83,6 +97,12 @@ export default {
     }
   },
   computed: {
+    scannerPreferAudioMetaTooltip() {
+      return 'Audio file ID3 meta tags will be used for book details over folder & filenames'
+    },
+    scannerPreferOpfMetaTooltip() {
+      return 'OPF file metadata will be used for book details over folder & filenames'
+    },
     saveMetadataTooltip() {
       return 'This will write a "metadata.nfo" file in all of your audiobook directories.'
     },
@@ -127,6 +147,16 @@ export default {
         scannerParseSubtitle: !!val
       })
     },
+    updateScannerPreferAudioMeta(val) {
+      this.updateServerSettings({
+        scannerPreferAudioMetadata: !!val
+      })
+    },
+    updateScannerPreferOpfMeta(val) {
+      this.updateServerSettings({
+        scannerPreferOpfMetadata: !!val
+      })
+    },
     updateServerSettings(payload) {
       this.updatingServerSettings = true
       this.$store
@@ -144,7 +174,6 @@ export default {
       this.newServerSettings = this.serverSettings ? { ...this.serverSettings } : {}
 
       this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
-      this.storeCoversInAudiobookDir = this.newServerSettings.coverDestination === this.$constants.CoverDestination.AUDIOBOOK
     },
     resetAudiobooks() {
       if (confirm('WARNING! This action will remove all audiobooks from the database including any updates or matches you have made. This does not do anything to your actual files. Shall we continue?')) {

server/Scanner.js

@@ -196,7 +196,7 @@ class Scanner {
 
     // Sync other files (all files that are not audio files) - Updates cover path
     var hasOtherFileUpdates = false
-    var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, forceAudioFileScan)
+    var otherFilesUpdated = await existingAudiobook.syncOtherFiles(audiobookData.otherFiles, this.MetadataPath, false, forceAudioFileScan)
     if (otherFilesUpdated) {
       hasOtherFileUpdates = true
     }
@@ -250,7 +250,7 @@ class Scanner {
     var hasUpdates = hasOtherFileUpdates || hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles
 
     // Check that audio tracks are in sequential order with no gaps
-    if (existingAudiobook.checkUpdateMissingParts()) {
+    if (existingAudiobook.checkUpdateMissingTracks()) {
       Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
       hasUpdates = true
     }
@@ -299,7 +299,7 @@ class Scanner {
     }
 
     // Look for desc.txt and reader.txt and update
-    await audiobook.saveDataFromTextFiles()
+    await audiobook.saveDataFromTextFiles(false)
 
     // Extract embedded cover art if cover is not already in directory
     if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
@@ -314,7 +314,7 @@ class Scanner {
     audiobook.setDetailsFromFileMetadata()
 
     // Check for gaps in track numbers
-    audiobook.checkUpdateMissingParts()
+    audiobook.checkUpdateMissingTracks()
 
     // Set chapters from audio files
     audiobook.setChapters()
@@ -671,11 +671,6 @@ class Scanner {
       var folder = library.getFolderById(folderId)
       if (!folder) {
         Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
-
-        Logger.debug(`Looking at folders in library "${library.name}" for folderid ${folderId}`)
-        library.folders.forEach((fold) => {
-          Logger.debug(`Folder "${fold.id}" "${fold.fullPath}"`)
-        })
         continue;
       }
 

server/Server.js

@@ -311,19 +311,18 @@ class Server {
 
   async filesChanged(fileUpdates) {
     Logger.info('[Server]', fileUpdates.length, 'Files Changed')
-    await this.scanner.filesChanged(fileUpdates)
+    await this.scanner2.scanFilesChanged(fileUpdates)
-    // Logger.debug('[Server] Files changed result', result)
   }
 
-  async scan(libraryId, forceAudioFileScan = false) {
+  async scan(libraryId) {
     Logger.info('[Server] Starting Scan')
-    // await this.scanner2.scan(libraryId)
+    await this.scanner2.scan(libraryId)
-    await this.scanner(libraryId, forceAudioFileScan)
+    // await this.scanner.scan(libraryId)
     Logger.info('[Server] Scan complete')
   }
 
   async scanAudiobook(socket, audiobookId) {
-    var result = await this.scanner.scanAudiobookById(audiobookId)
+    var result = await this.scanner2.scanAudiobookById(audiobookId)
     var scanResultName = ''
     for (const key in ScanResult) {
       if (ScanResult[key] === result) {
@@ -335,7 +334,7 @@ class Server {
 
   cancelScan(id) {
     Logger.debug('[Server] Cancel scan', id)
-    this.scanner.cancelLibraryScan[id] = true
+    this.scanner2.cancelLibraryScan[id] = true
   }
 
   // Generates an NFO metadata file, if no audiobookId is passed then all audiobooks are done
@@ -624,7 +623,7 @@ class Server {
       configPath: this.ConfigPath,
       user: client.user.toJSONForBrowser(),
       stream: client.stream || null,
-      librariesScanning: this.scanner.librariesScanning,
+      librariesScanning: this.scanner2.librariesScanning,
       backups: (this.backupManager.backups || []).map(b => b.toJSON())
     }
     if (user.type === 'root') {

server/Watcher.js

@@ -24,8 +24,11 @@ class FolderWatcher extends EventEmitter {
       Logger.warn('[Watcher] Already watching library', library.name)
       return
     }
-    Logger.info(`[Watcher] Initializing watcher for "${library.name}"..`)
+    Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
     var folderPaths = library.folderPaths
+    folderPaths.forEach((fp) => {
+      Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
+    })
     var watcher = new Watcher(folderPaths, {
       ignored: /(^|[\/\\])\../, // ignore dotfiles
       renameDetection: true,
@@ -48,6 +51,8 @@ class FolderWatcher extends EventEmitter {
         Logger.error(`[Watcher] ${error}`)
       }).on('ready', () => {
         Logger.info(`[Watcher] "${library.name}" Ready`)
+      }).on('close', () => {
+        Logger.debug(`[Watcher] "${library.name}" Closed`)
       })
 
     this.libraryWatchers.push({

server/objects/AudioFile.js

@@ -165,9 +165,9 @@ class AudioFile {
     this.fullPath = fileData.fullPath
     this.addedAt = Date.now()
 
-    this.trackNumFromMeta = fileData.trackNumFromMeta || null
+    this.trackNumFromMeta = fileData.trackNumFromMeta
-    this.trackNumFromFilename = fileData.trackNumFromFilename || null
+    this.trackNumFromFilename = fileData.trackNumFromFilename
-    this.cdNumFromFilename = fileData.cdNumFromFilename || null
+    this.cdNumFromFilename = fileData.cdNumFromFilename
 
     this.format = probeData.format
     this.duration = probeData.duration
@@ -180,15 +180,13 @@ class AudioFile {
     this.channelLayout = probeData.channelLayout
     this.chapters = probeData.chapters || []
     this.metadata = probeData.audioFileMetadata
+    this.embeddedCoverArt = probeData.embeddedCoverArt
   }
 
-  validateTrackIndex(isSingleTrack) {
+  validateTrackIndex() {
     var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta)
     var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename)
 
-    if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1
-      return numFromMeta ? numFromMeta : 1
-    }
     if (numFromMeta !== null) return numFromMeta
     if (numFromFilename !== null) return numFromFilename
 
@@ -284,5 +282,33 @@ class AudioFile {
     })
     return hasUpdates
   }
+
+  updateFromScan(scannedAudioFile) {
+    var hasUpdated = false
+
+    var newjson = scannedAudioFile.toJSON()
+    if (this.manuallyVerified) newjson.manuallyVerified = true
+    if (this.exclude) newjson.exclude = true
+    newjson.addedAt = this.addedAt
+
+    for (const key in newjson) {
+      if (key === 'metadata') {
+        if (!this.metadata || !this.metadata.isEqual(scannedAudioFile.metadata)) {
+          this.metadata = scannedAudioFile.metadata
+          hasUpdated = true
+          // console.log('metadata updated for audio file')
+        }
+      } else if (key === 'chapters') {
+        if (this.syncChapters(newjson.chapters || [])) {
+          hasUpdated = true
+        }
+      } else if (this[key] !== newjson[key]) {
+        this[key] = newjson[key]
+        hasUpdated = true
+        // console.log('key', key, 'updated', this[key], newjson[key])
+      }
+    }
+    return hasUpdated
+  }
 }
 module.exports = AudioFile

server/objects/AudioFileMetadata.js

@@ -101,5 +101,13 @@ class AudioFileMetadata {
     }
     return hasUpdates
   }
+
+  isEqual(audioFileMetadata) {
+    if (!audioFileMetadata || !audioFileMetadata.toJSON) return false
+    for (const key in audioFileMetadata.toJSON()) {
+      if (audioFileMetadata[key] !== this[key]) return false
+    }
+    return true
+  }
 }
 module.exports = AudioFileMetadata

server/objects/Audiobook.js

@@ -353,6 +353,11 @@ class Audiobook {
     this.lastUpdate = Date.now()
   }
 
+  setInvalid() {
+    this.isInvalid = true
+    this.lastUpdate = Date.now()
+  }
+
   setBook(data) {
     // Use first image file as cover
     if (this.otherFiles && this.otherFiles.length) {
@@ -400,6 +405,11 @@ class Audiobook {
     }
   }
 
+  updateAudioFile(updatedAudioFile) {
+    var audioFile = this.audioFiles.find(af => af.ino === updatedAudioFile.ino)
+    return audioFile.updateFromScan(updatedAudioFile)
+  }
+
   addOtherFile(fileData) {
     var file = new AudiobookFile()
     file.setData(fileData)
@@ -437,8 +447,8 @@ class Audiobook {
     return this.book.updateCover(cover, coverFullPath)
   }
 
-  checkHasTrackNum(trackNum) {
+  checkHasTrackNum(trackNum, excludeIno) {
-    return this.tracks.find(t => t.index === trackNum)
+    return this._audioFiles.find(t => t.index === trackNum && t.ino !== excludeIno)
   }
 
   updateAudioTracks(orderedFileData) {
@@ -473,6 +483,7 @@ class Audiobook {
       }
     })
     this.setChapters()
+    this.checkUpdateMissingTracks()
     this.lastUpdate = Date.now()
   }
 
@@ -486,7 +497,7 @@ class Audiobook {
     this.audioFiles = this.audioFiles.filter(f => f.ino !== track.ino)
   }
 
-  checkUpdateMissingParts() {
+  checkUpdateMissingTracks() {
     var currMissingParts = (this.missingParts || []).join(',') || ''
 
     var current_index = 1
@@ -515,13 +526,14 @@ class Audiobook {
   }
 
   // On scan check other files found with other files saved
-  async syncOtherFiles(newOtherFiles, metadataPath, forceRescan = false) {
+  async syncOtherFiles(newOtherFiles, metadataPath, opfMetadataOverrideDetails, forceRescan = false) {
     var hasUpdates = false
 
     var currOtherFileNum = this.otherFiles.length
 
-    var alreadyHasDescTxt = this.otherFiles.find(of => of.filename === 'desc.txt')
+    var otherFilenamesAlreadyInBook = this.otherFiles.map(ofile => ofile.filename)
-    var alreadyHasReaderTxt = this.otherFiles.find(of => of.filename === 'reader.txt')
+    var alreadyHasDescTxt = otherFilenamesAlreadyInBook.includes('desc.txt')
+    var alreadyHasReaderTxt = otherFilenamesAlreadyInBook.includes('reader.txt')
 
     var newOtherFilePaths = newOtherFiles.map(f => f.path)
     this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
@@ -553,21 +565,22 @@ class Audiobook {
       }
     }
 
+    // If OPF file and was not already there
     var metadataOpf = newOtherFiles.find(file => file.ext === '.opf' || file.filename === 'metadata.xml')
-    if (metadataOpf) {
+    if (metadataOpf && (!otherFilenamesAlreadyInBook.includes(metadataOpf.filename) || opfMetadataOverrideDetails)) {
       var xmlText = await readTextFile(metadataOpf.fullPath)
       if (xmlText) {
         var opfMetadata = await parseOpfMetadataXML(xmlText)
-        Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
+        // Logger.debug(`[Audiobook] Sync Other File "${metadataOpf.filename}" parsed:`, opfMetadata)
         if (opfMetadata) {
           const bookUpdatePayload = {}
           for (const key in opfMetadata) {
             // Add genres only if genres are empty
             if (key === 'genres') {
-              if (opfMetadata.genres.length && !this.book._genres.length) {
+              if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
                 bookUpdatePayload[key] = opfMetadata.genres
               }
-            } else if (opfMetadata[key] && !this.book[key]) {
+            } else if (opfMetadata[key] && (!this.book[key] || opfMetadataOverrideDetails)) {
               bookUpdatePayload[key] = opfMetadata[key]
             }
           }
@@ -789,7 +802,7 @@ class Audiobook {
   }
 
   // Look for desc.txt and reader.txt and update details if found
-  async saveDataFromTextFiles() {
+  async saveDataFromTextFiles(opfMetadataOverrideDetails) {
     var bookUpdatePayload = {}
     var descriptionText = await this.fetchTextFromTextFile('desc.txt')
     if (descriptionText) {
@@ -807,15 +820,15 @@ class Audiobook {
       var xmlText = await readTextFile(metadataOpf.fullPath)
       if (xmlText) {
         var opfMetadata = await parseOpfMetadataXML(xmlText)
-        Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
+        // Logger.debug(`[Audiobook] "${this.title}" found "${metadataOpf.filename}" parsed:`, opfMetadata)
         if (opfMetadata) {
           for (const key in opfMetadata) {
             // Add genres only if genres are empty
             if (key === 'genres') {
-              if (opfMetadata.genres.length && !this.book._genres.length) {
+              if (opfMetadata.genres.length && (!this.book._genres.length || opfMetadataOverrideDetails)) {
                 bookUpdatePayload[key] = opfMetadata.genres
               }
-            } else if (opfMetadata[key] && !this.book[key] && !bookUpdatePayload[key]) {
+            } else if (opfMetadata[key] && ((!this.book[key] && !bookUpdatePayload[key]) || opfMetadataOverrideDetails)) {
               bookUpdatePayload[key] = opfMetadata[key]
             }
           }
@@ -836,10 +849,10 @@ class Audiobook {
   }
 
   // Audio file metadata tags map to book details (will not overwrite)
-  setDetailsFromFileMetadata() {
+  setDetailsFromFileMetadata(overrideExistingDetails = false) {
     if (!this.audioFiles.length) return false
     var audioFile = this.audioFiles[0]
-    return this.book.setDetailsFromFileMetadata(audioFile.metadata)
+    return this.book.setDetailsFromFileMetadata(audioFile.metadata, overrideExistingDetails)
   }
 
   // Returns null if file not found, true if file was updated, false if up to date
@@ -884,9 +897,15 @@ class Audiobook {
     return hasUpdated
   }
 
-  checkScanData(dataFound) {
+  checkScanData(dataFound, version) {
     var hasUpdated = false
 
+    if (this.isMissing) {
+      // Audiobook no longer missing
+      this.isMissing = false
+      hasUpdated = true
+    }
+
     if (dataFound.ino !== this.ino) {
       this.ino = dataFound.ino
       hasUpdated = true
@@ -916,7 +935,7 @@ class Audiobook {
       var audioFileFoundCheck = this.checkFileFound(af, true)
       if (audioFileFoundCheck === null) {
         newAudioFileData.push(af)
-      } else if (audioFileFoundCheck === true) {
+      } else if (audioFileFoundCheck) {
         hasUpdated = true
       }
     })
@@ -925,7 +944,7 @@ class Audiobook {
       var fileFoundCheck = this.checkFileFound(otherFileData, false)
       if (fileFoundCheck === null) {
         newOtherFileData.push(otherFileData)
-      } else if (fileFoundCheck === true) {
+      } else if (fileFoundCheck) {
         hasUpdated = true
       }
     })
@@ -933,7 +952,7 @@ class Audiobook {
     const audioFilesRemoved = []
     const otherFilesRemoved = []
 
-    // inodes will all be up to date at this point
+    // Remove audio files not found (inodes will all be up to date at this point)
     this.audioFiles = this.audioFiles.filter(af => {
       if (!dataFound.audioFiles.find(_af => _af.ino === af.ino)) {
         audioFilesRemoved.push(af.toJSON())
@@ -946,10 +965,11 @@ class Audiobook {
     if (audioFilesRemoved.length) {
       const audioFilesRemovedInodes = audioFilesRemoved.map(afr => afr.ino)
       this.tracks = this.tracks.filter(t => !audioFilesRemovedInodes.includes(t.ino))
-      this.checkUpdateMissingParts()
+      this.checkUpdateMissingTracks()
       hasUpdated = true
     }
 
+    // Remove other files not found
     this.otherFiles = this.otherFiles.filter(otherFile => {
       if (!dataFound.otherFiles.find(_otherFile => _otherFile.ino === otherFile.ino)) {
         otherFilesRemoved.push(otherFile.toJSON())
@@ -969,6 +989,15 @@ class Audiobook {
       hasUpdated = true
     }
 
+    // Check if invalid (has no audio files or ebooks)
+    if (!this.audioFilesToInclude.length && !this.ebooks.length && !newAudioFileData.length && !newOtherFileData.length) {
+      this.isInvalid = true
+    }
+
+    if (hasUpdated) {
+      this.setLastScan(version)
+    }
+
     return {
       updated: hasUpdated,
       newAudioFileData,

server/objects/Book.js

@@ -43,6 +43,7 @@ class Book {
   get _genres() { return this.genres || [] }
 
   get shouldSearchForCover() {
+    if (this.cover) return false
     if (this.authorFL !== this.lastCoverSearchAuthor || this.title !== this.lastCoverSearchTitle || !this.lastCoverSearch) return true
     var timeSinceLastSearch = Date.now() - this.lastCoverSearch
     return timeSinceLastSearch > 1000 * 60 * 60 * 24 * 7 // every 7 days do another lookup
@@ -297,7 +298,7 @@ class Book {
     return [genreTag]
   }
 
-  setDetailsFromFileMetadata(audioFileMetadata) {
+  setDetailsFromFileMetadata(audioFileMetadata, overrideExistingDetails = false) {
     const MetadataMapArray = [
       {
         tag: 'tagComposer',
@@ -319,6 +320,10 @@ class Book {
         tag: 'tagSubtitle',
         key: 'subtitle'
       },
+      {
+        tag: 'tagAlbum',
+        key: 'title',
+      },
       {
         tag: 'tagArtist',
         key: 'author'
@@ -342,12 +347,12 @@ class Book {
     MetadataMapArray.forEach((mapping) => {
       if (audioFileMetadata[mapping.tag]) {
         // Genres can contain multiple
-        if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key])) {
+        if (mapping.key === 'genres' && (!this[mapping.key].length || !this[mapping.key] || overrideExistingDetails)) {
           updatePayload[mapping.key] = this.parseGenresTag(audioFileMetadata[mapping.tag])
-          Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
+          // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key].join(',')}`)
-        } else if (!this[mapping.key]) {
+        } else if (!this[mapping.key] || overrideExistingDetails) {
           updatePayload[mapping.key] = audioFileMetadata[mapping.tag]
-          Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
+          // Logger.debug(`[Book] Mapping metadata to key ${mapping.tag} => ${mapping.key}: ${updatePayload[mapping.key]}`)
         }
       }
     })

server/objects/ServerSettings.js

@@ -46,6 +46,9 @@ class ServerSettings {
     this.newTagExpireDays = settings.newTagExpireDays
     this.scannerFindCovers = !!settings.scannerFindCovers
     this.scannerParseSubtitle = settings.scannerParseSubtitle
+    this.scannerPreferAudioMetadata = !!settings.scannerPreferAudioMetadata
+    this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata
+
     this.coverDestination = settings.coverDestination || CoverDestination.METADATA
     this.saveMetadataFile = !!settings.saveMetadataFile
     this.rateLimitLoginRequests = !isNaN(settings.rateLimitLoginRequests) ? Number(settings.rateLimitLoginRequests) : 10
@@ -73,6 +76,8 @@ class ServerSettings {
       newTagExpireDays: this.newTagExpireDays,
       scannerFindCovers: this.scannerFindCovers,
       scannerParseSubtitle: this.scannerParseSubtitle,
+      scannerPreferAudioMetadata: this.scannerPreferAudioMetadata,
+      scannerPreferOpfMetadata: this.scannerPreferOpfMetadata,
       coverDestination: this.coverDestination,
       saveMetadataFile: !!this.saveMetadataFile,
       rateLimitLoginRequests: this.rateLimitLoginRequests,

server/scanner/AudioFileScanner.js

@@ -4,7 +4,7 @@ const AudioFile = require('../objects/AudioFile')
 
 const prober = require('../utils/prober')
 const Logger = require('../Logger')
-const { msToTimestamp } = require('../utils')
+const { LogLevel } = require('../utils/constants')
 
 class AudioFileScanner {
   constructor() { }
@@ -80,6 +80,9 @@ class AudioFileScanner {
     audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
     audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
     audioFile.setDataFromProbe(audioFileData, probeData)
+    if (audioFile.embeddedCoverArt) {
+
+    }
     return {
       audioFile,
       elapsed: Date.now() - probeStart
@@ -87,12 +90,11 @@ class AudioFileScanner {
   }
 
 
-  // Returns array of { AudioFile, elapsed } from audio file scan objects
+  // Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects
-  async scanAudioFiles(audioFileDataArray, bookScanData) {
+  async executeAudioFileScans(audioFileDataArray, bookScanData) {
     var proms = []
     for (let i = 0; i < audioFileDataArray.length; i++) {
-      var prom = this.scan(audioFileDataArray[i], bookScanData)
+      proms.push(this.scan(audioFileDataArray[i], bookScanData))
-      proms.push(prom)
     }
     var scanStart = Date.now()
     var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
@@ -102,5 +104,62 @@ class AudioFileScanner {
       averageScanDuration: this.getAverageScanDurationMs(results)
     }
   }
+
+  async scanAudioFiles(audioFileDataArray, bookScanData, audiobook, preferAudioMetadata, libraryScan = null) {
+    var hasUpdated = false
+
+    var audioScanResult = await this.executeAudioFileScans(audioFileDataArray, bookScanData)
+    if (audioScanResult.audioFiles.length) {
+      if (libraryScan) {
+        libraryScan.addLog(LogLevel.DEBUG, `Book "${bookScanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`)
+      }
+
+      var totalAudioFilesToInclude = audiobook.audioFilesToInclude.filter(af => !audioScanResult.audioFiles.find(_af => _af.ino === af.ino)).length + audioScanResult.audioFiles.length
+
+      // validate & add/update audio files to audiobook
+      for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
+        var newAF = audioScanResult.audioFiles[i]
+        var existingAF = audiobook.getAudioFileByIno(newAF.ino)
+
+        var trackIndex = null
+        if (totalAudioFilesToInclude === 1) { // Single track audiobooks
+          trackIndex = 1
+        } else if (existingAF && existingAF.manuallyVerified) { // manually verified audio files use existing index
+          trackIndex = existingAF.index
+        } else {
+          trackIndex = newAF.validateTrackIndex()
+        }
+
+        if (trackIndex !== null) {
+          if (audiobook.checkHasTrackNum(trackIndex, newAF.ino)) {
+            newAF.setDuplicateTrackNumber(trackIndex)
+          } else {
+            newAF.index = trackIndex
+          }
+        }
+        if (existingAF) {
+          if (audiobook.updateAudioFile(newAF)) {
+            // console.log('update dauido file')
+            hasUpdated = true
+          }
+        } else {
+          audiobook.addAudioFile(newAF)
+          // console.log('added auido file')
+          hasUpdated = true
+        }
+      }
+
+      if (hasUpdated) {
+        audiobook.rebuildTracks()
+      }
+
+      // Set book details from audio file ID3 tags, optional prefer
+      if (audiobook.setDetailsFromFileMetadata(preferAudioMetadata)) {
+        hasUpdated = true
+      }
+
+    }
+    return hasUpdated
+  }
 }
 module.exports = new AudioFileScanner()

server/scanner/AudioProbeData.js

@@ -35,7 +35,6 @@ class AudioProbeData {
 
   setData(data) {
     var audioStream = this.getDefaultAudioStream(data.audio_streams)
-
     this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
     this.format = data.format
     this.duration = data.duration

server/scanner/LibraryScan.js

@@ -1,6 +1,10 @@
-const Folder = require('../objects/Folder')
+const Path = require('path')
-const Constants = require('../utils/constants')
+const fs = require('fs-extra')
+const date = require('date-and-time')
 
+const Logger = require('../Logger')
+const Folder = require('../objects/Folder')
+const { LogLevel } = require('../utils/constants')
 const { getId, secondsToTimestamp } = require('../utils/index')
 
 class LibraryScan {
@@ -9,6 +13,7 @@ class LibraryScan {
     this.libraryId = null
     this.libraryName = null
     this.folders = null
+    this.verbose = false
 
     this.scanOptions = null
 
@@ -16,14 +21,21 @@ class LibraryScan {
     this.finishedAt = null
     this.elapsed = null
 
-    this.status = Constants.ScanStatus.NOTHING
     this.resultsMissing = 0
     this.resultsAdded = 0
     this.resultsUpdated = 0
+
+    this.logs = []
   }
 
   get _scanOptions() { return this.scanOptions || {} }
   get forceRescan() { return !!this._scanOptions.forceRescan }
+  get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
+  get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
+  get findCovers() { return !!this._scanOptions.findCovers }
+  get timestamp() {
+    return (new Date()).toISOString()
+  }
 
   get resultStats() {
     return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
@@ -42,6 +54,28 @@ class LibraryScan {
       }
     }
   }
+  get totalResults() {
+    return this.resultsAdded + this.resultsUpdated + this.resultsMissing
+  }
+  get getLogFilename() {
+    return date.format(new Date(), 'YYYY-MM-DD') + '_' + this.id + '.txt'
+  }
+
+  toJSON() {
+    return {
+      id: this.id,
+      libraryId: this.libraryId,
+      libraryName: this.libraryName,
+      folders: this.folders.map(f => f.toJSON()),
+      scanOptions: this.scanOptions.toJSON(),
+      startedAt: this.startedAt,
+      finishedAt: this.finishedAt,
+      elapsed: this.elapsed,
+      resultsAdded: this.resultsAdded,
+      resultsUpdated: this.resultsUpdated,
+      resultsMissing: this.resultsMissing
+    }
+  }
 
   setData(library, scanOptions) {
     this.id = getId('lscan')
@@ -58,5 +92,39 @@ class LibraryScan {
     this.finishedAt = Date.now()
     this.elapsed = this.finishedAt - this.startedAt
   }
+
+  getLogLevelString(level) {
+    for (const key in LogLevel) {
+      if (LogLevel[key] === level) {
+        return key
+      }
+    }
+    return 'UNKNOWN'
+  }
+
+  addLog(level, ...args) {
+    const logObj = {
+      timestamp: this.timestamp,
+      message: args.join(' '),
+      levelName: this.getLogLevelString(level),
+      level
+    }
+
+    if (this.verbose) {
+      Logger.debug(`[LibraryScan] "${this.libraryName}":`, args)
+    }
+    this.logs.push(logObj)
+  }
+
+  async saveLog(logDir) {
+    await fs.ensureDir(logDir)
+    var outputPath = Path.join(logDir, this.getLogFilename)
+    var logLines = [JSON.stringify(this.toJSON())]
+    this.logs.forEach(l => {
+      logLines.push(JSON.stringify(l))
+    })
+    await fs.writeFile(outputPath, logLines.join('\n') + '\n')
+    Logger.info(`[LibraryScan] Scan log saved "${outputPath}"`)
+  }
 }
 module.exports = LibraryScan

server/scanner/ScanOptions.js

@@ -4,29 +4,6 @@ class ScanOptions {
   constructor(options) {
     this.forceRescan = false
 
-    // this.metadataPrecedence = [
-    //   {
-    //     id: 'directory',
-    //     include: true
-    //   },
-    //   {
-    //     id: 'reader-desc-txt',
-    //     include: true
-    //   },
-    //   {
-    //     id: 'audio-file-metadata',
-    //     include: true
-    //   },
-    //   {
-    //     id: 'metadata-opf',
-    //     include: true
-    //   },
-    //   {
-    //     id: 'external-source',
-    //     include: false
-    //   }
-    // ]
-
     // Server settings
     this.parseSubtitles = false
     this.findCovers = false

server/scanner/Scanner.js

@@ -4,10 +4,9 @@ const Path = require('path')
 // Utils
 const Logger = require('../Logger')
 const { version } = require('../../package.json')
-const audioFileScanner = require('../utils/audioFileScanner')
 const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
 const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
-const { ScanResult, CoverDestination } = require('../utils/constants')
+const { ScanResult, CoverDestination, LogLevel } = require('../utils/constants')
 
 const AudioFileScanner = require('./AudioFileScanner')
 const BookFinder = require('../BookFinder')
@@ -20,6 +19,8 @@ class Scanner {
     this.AudiobookPath = AUDIOBOOK_PATH
     this.MetadataPath = METADATA_PATH
     this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
+    var LogDirPath = Path.join(this.MetadataPath, 'logs')
+    this.ScanLogPath = Path.join(LogDirPath, 'scans')
 
     this.db = db
     this.coverController = coverController
@@ -46,8 +47,82 @@ class Scanner {
     }
   }
 
+  isLibraryScanning(libraryId) {
+    return this.librariesScanning.find(ls => ls.id === libraryId)
+  }
+
+  async scanAudiobookById(audiobookId) {
+    var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId)
+    if (!audiobook) {
+      Logger.error(`[Scanner] Scan audiobook by id not found ${audiobookId}`)
+      return ScanResult.NOTHING
+    }
+    const library = this.db.libraries.find(lib => lib.id === audiobook.libraryId)
+    if (!library) {
+      Logger.error(`[Scanner] Scan audiobook by id library not found "${audiobook.libraryId}"`)
+      return ScanResult.NOTHING
+    }
+    const folder = library.folders.find(f => f.id === audiobook.folderId)
+    if (!folder) {
+      Logger.error(`[Scanner] Scan audiobook by id folder not found "${audiobook.folderId}" in library "${library.name}"`)
+      return ScanResult.NOTHING
+    }
+    Logger.info(`[Scanner] Scanning Audiobook "${audiobook.title}"`)
+    return this.scanAudiobook(folder, audiobook)
+  }
+
+  async scanAudiobook(folder, audiobook) {
+    var audiobookData = await getAudiobookFileData(folder, audiobook.fullPath, this.db.serverSettings)
+    if (!audiobookData) {
+      return ScanResult.NOTHING
+    }
+    var hasUpdated = false
+
+    var checkRes = audiobook.checkScanData(audiobookData, version)
+    if (checkRes.updated) hasUpdated = true
+
+    // Sync other files first so that local images are used as cover art
+    // TODO: Cleanup other file sync
+    var allOtherFiles = checkRes.newOtherFileData.concat(audiobook._otherFiles)
+    if (await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, this.db.serverSettings.scannerPreferOpfMetadata)) {
+      hasUpdated = true
+    }
+
+    // Scan all audio files
+    if (audiobookData.audioFiles.length) {
+      if (await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, this.db.serverSettings.scannerPreferAudioMetadata)) {
+        hasUpdated = true
+      }
+
+      // Extract embedded cover art if cover is not already in directory
+      if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
+        var outputCoverDirs = this.getCoverDirectory(audiobook)
+        var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
+        if (relativeDir) {
+          Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
+          hasUpdated = true
+        }
+      }
+    }
+
+    if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
+      audiobook.setInvalid()
+      hasUpdated = true
+    } else if (audiobook.isInvalid) {
+      audiobook.isInvalid = false
+      hasUpdated = true
+    }
+
+    if (hasUpdated) {
+      this.emitter('audiobook_updated', audiobook.toJSONExpanded())
+      await this.db.updateEntity('audiobook', audiobook)
+      return ScanResult.UPDATED
+    }
+    return ScanResult.UPTODATE
+  }
+
   async scan(libraryId, options = {}) {
-    if (this.librariesScanning.includes(libraryId)) {
+    if (this.isLibraryScanning(libraryId)) {
       Logger.error(`[Scanner] Already scanning ${libraryId}`)
       return
     }
@@ -66,172 +141,380 @@ class Scanner {
 
     var libraryScan = new LibraryScan()
     libraryScan.setData(library, scanOptions)
-    this.librariesScanning.push(libraryScan)
+    libraryScan.verbose = false
+    this.librariesScanning.push(libraryScan.getScanEmitData)
 
     this.emitter('scan_start', libraryScan.getScanEmitData)
 
     Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
 
-    await this.scanLibrary(libraryScan)
+    var canceled = await this.scanLibrary(libraryScan)
+
+    if (canceled) {
+      Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
+      delete this.cancelLibraryScan[libraryScan.libraryId]
+    }
 
     libraryScan.setComplete()
-    Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
 
+    Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp} | ${libraryScan.resultStats}`)
     this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
+
+    if (canceled && !libraryScan.totalResults) {
+      var emitData = libraryScan.getScanEmitData
+      emitData.results = null
+      this.emitter('scan_complete', emitData)
+      return
+    }
+
     this.emitter('scan_complete', libraryScan.getScanEmitData)
+
+    if (libraryScan.totalResults) {
+      libraryScan.saveLog(this.ScanLogPath)
+    }
   }
 
   async scanLibrary(libraryScan) {
     var audiobookDataFound = []
+
+    // Scan each library
     for (let i = 0; i < libraryScan.folders.length; i++) {
       var folder = libraryScan.folders[i]
       var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
-      Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
+      libraryScan.addLog(LogLevel.INFO, `${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
       audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
     }
 
+    if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+
     // Remove audiobooks with no inode
     audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
 
     var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
 
+    const NumScansPerChunk = 25
+    const audiobooksToUpdateChunks = []
+    const audiobookDataToRescanChunks = []
+    const newAudiobookDataToScanChunks = []
     var audiobooksToUpdate = []
-    var audiobookRescans = []
+    var audiobookDataToRescan = []
-    var newAudiobookScans = []
+    var newAudiobookDataToScan = []
+    var audiobooksToFindCovers = []
 
     // Check for existing & removed audiobooks
     for (let i = 0; i < audiobooksInLibrary.length; i++) {
       var audiobook = audiobooksInLibrary[i]
       var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
       if (!dataFound) {
-        Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
+        libraryScan.addLog(LogLevel.WARN, `Audiobook "${audiobook.title}" is missing`)
+        libraryScan.resultsMissing++
         audiobook.setMissing()
         audiobooksToUpdate.push(audiobook)
+        if (audiobooksToUpdate.length === NumScansPerChunk) {
+          audiobooksToUpdateChunks.push(audiobooksToUpdate)
+          audiobooksToUpdate = []
+        }
       } else {
-        var checkRes = audiobook.checkScanData(dataFound)
+        var checkRes = audiobook.checkScanData(dataFound, version)
-        if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
+        if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) { // Audiobook has new files
-          // existing audiobook has new files
           checkRes.audiobook = audiobook
           checkRes.bookScanData = dataFound
-          audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
+          audiobookDataToRescan.push(checkRes)
-          libraryScan.resultsMissing++
+          if (audiobookDataToRescan.length === NumScansPerChunk) {
-        } else if (checkRes.updated) {
+            audiobookDataToRescanChunks.push(audiobookDataToRescan)
-          audiobooksToUpdate.push(audiobook)
+            audiobookDataToRescan = []
+          }
+        } else if (libraryScan.findCovers && audiobook.book.shouldSearchForCover) {
           libraryScan.resultsUpdated++
+          audiobooksToFindCovers.push(audiobook)
+          audiobooksToUpdate.push(audiobook)
+          if (audiobooksToUpdate.length === NumScansPerChunk) {
+            audiobooksToUpdateChunks.push(audiobooksToUpdate)
+            audiobooksToUpdate = []
+          }
+        } else if (checkRes.updated) { // Updated but no scan required
+          libraryScan.resultsUpdated++
+          audiobooksToUpdate.push(audiobook)
+          if (audiobooksToUpdate.length === NumScansPerChunk) {
+            audiobooksToUpdateChunks.push(audiobooksToUpdate)
+            audiobooksToUpdate = []
+          }
         }
         audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
       }
     }
+    if (audiobooksToUpdate.length) audiobooksToUpdateChunks.push(audiobooksToUpdate)
+    if (audiobookDataToRescan.length) audiobookDataToRescanChunks.push(audiobookDataToRescan)
 
     // Potential NEW Audiobooks
     for (let i = 0; i < audiobookDataFound.length; i++) {
       var dataFound = audiobookDataFound[i]
       var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
       if (!hasEbook && !dataFound.audioFiles.length) {
-        Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
+        libraryScan.addLog(LogLevel.WARN, `Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
       } else {
-        newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
+        newAudiobookDataToScan.push(dataFound)
+        if (newAudiobookDataToScan.length === NumScansPerChunk) {
+          newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
+          newAudiobookDataToScan = []
+        }
       }
     }
+    if (newAudiobookDataToScan.length) newAudiobookDataToScanChunks.push(newAudiobookDataToScan)
+
+    // console.log('Num chunks to update', audiobooksToUpdateChunks.length)
+    // console.log('Num chunks to rescan', audiobookDataToRescanChunks.length)
+    // console.log('Num chunks to new scan', newAudiobookDataToScanChunks.length)
+
+    // Audiobooks not requiring a scan but require a search for cover
+    for (let i = 0; i < audiobooksToFindCovers.length; i++) {
+      var audiobook = audiobooksToFindCovers[i]
+      var updatedCover = await this.searchForCover(audiobook, libraryScan)
+      audiobook.book.updateLastCoverSearch(updatedCover)
+    }
 
-    if (audiobookRescans.length) {
+    for (let i = 0; i < audiobooksToUpdateChunks.length; i++) {
-      var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
+      await this.updateAudiobooksChunk(audiobooksToUpdateChunks[i])
-      if (updatedAudiobooks.length) {
+      if (this.cancelLibraryScan[libraryScan.libraryId]) return true
-        audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
+      // console.log('Update chunk done', i, 'of', audiobooksToUpdateChunks.length)
-        libraryScan.resultsUpdated += updatedAudiobooks.length
-      }
     }
-    if (audiobooksToUpdate.length) {
+    for (let i = 0; i < audiobookDataToRescanChunks.length; i++) {
-      Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
+      await this.rescanAudiobookDataChunk(audiobookDataToRescanChunks[i], libraryScan)
-      await this.db.updateEntities('audiobook', audiobooksToUpdate)
+      if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+      // console.log('Rescan chunk done', i, 'of', audiobookDataToRescanChunks.length)
     }
+    for (let i = 0; i < newAudiobookDataToScanChunks.length; i++) {
+      await this.scanNewAudiobookDataChunk(newAudiobookDataToScanChunks[i], libraryScan)
+      // console.log('New scan chunk done', i, 'of', newAudiobookDataToScanChunks.length)
+      if (this.cancelLibraryScan[libraryScan.libraryId]) return true
+    }
+  }
 
-    if (newAudiobookScans.length) {
+  async updateAudiobooksChunk(audiobooksToUpdate) {
-      var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
+    await this.db.updateEntities('audiobook', audiobooksToUpdate)
-      if (newAudiobooks.length) {
+    this.emitter('audiobooks_updated', audiobooksToUpdate.map(ab => ab.toJSONExpanded()))
-        Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
+  }
-        await this.db.insertEntities('audiobook', newAudiobooks)
+
-        libraryScan.resultsAdded = newAudiobooks.length
+  async rescanAudiobookDataChunk(audiobookDataToRescan, libraryScan) {
-      }
+    var audiobooksUpdated = await Promise.all(audiobookDataToRescan.map((abd) => {
-    }
+      return this.rescanAudiobook(abd, libraryScan)
+    }))
+    audiobooksUpdated = audiobooksUpdated.filter(ab => ab) // Filter out nulls
+    libraryScan.resultsUpdated += audiobooksUpdated.length
+    await this.db.updateEntities('audiobook', audiobooksUpdated)
+    this.emitter('audiobooks_updated', audiobooksUpdated.map(ab => ab.toJSONExpanded()))
+  }
+
+  async scanNewAudiobookDataChunk(newAudiobookDataToScan, libraryScan) {
+    var newAudiobooks = await Promise.all(newAudiobookDataToScan.map((abd) => {
+      return this.scanNewAudiobook(abd, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan)
+    }))
+    newAudiobooks = newAudiobooks.filter(ab => ab) // Filter out nulls
+    libraryScan.resultsAdded += newAudiobooks.length
+    await this.db.insertEntities('audiobook', newAudiobooks)
+    this.emitter('audiobooks_added', newAudiobooks.map(ab => ab.toJSONExpanded()))
   }
 
   async rescanAudiobook(audiobookCheckData, libraryScan) {
     const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
-    Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
+    libraryScan.addLog(LogLevel.DEBUG, `Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
+
+    // Sync other files first to use local images as cover before extracting audio file cover
+    if (newOtherFileData.length) {
+      // TODO: Cleanup other file sync
+      var allOtherFiles = newOtherFileData.concat(audiobook._otherFiles)
+      await audiobook.syncOtherFiles(allOtherFiles, this.MetadataPath, libraryScan.preferOpfMetadata)
+    }
 
     if (newAudioFileData.length) {
-      var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
+      await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData, audiobook, libraryScan.preferAudioMetadata, libraryScan)
-      Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
-      if (audioScanResult.audioFiles.length) {
-        var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
 
-        // validate & add audio files to audiobook
+      // Extract embedded cover art if cover is not already in directory
-        for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
+      if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
-          var newAF = audioScanResult.audioFiles[i]
+        var outputCoverDirs = this.getCoverDirectory(audiobook)
-          var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
+        var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
-          if (trackIndex !== null) {
+        if (relativeDir) {
-            if (audiobook.checkHasTrackNum(trackIndex)) {
+          libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
-              newAF.setDuplicateTrackNumber(trackIndex)
-            } else {
-              newAF.index = trackIndex
-            }
-          }
-          audiobook.addAudioFile(newAF)
         }
-
-        audiobook.rebuildTracks()
       }
     }
-    if (newOtherFileData.length) {
+
-      await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
+    if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) { // Audiobook is invalid
+      audiobook.setInvalid()
+    } else if (audiobook.isInvalid) {
+      audiobook.isInvalid = false
     }
+
+    // Scan for cover if enabled and has no cover
+    if (audiobook && libraryScan.findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
+      var updatedCover = await this.searchForCover(audiobook, libraryScan)
+      audiobook.book.updateLastCoverSearch(updatedCover)
+    }
+
     return audiobook
   }
 
-  async scanNewAudiobook(audiobookData, libraryScan) {
+  async scanNewAudiobook(audiobookData, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) {
-    Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
+    if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new book "${audiobookData.path}"`)
+    else Logger.debug(`[Scanner] Scanning new book "${audiobookData.path}"`)
+
     var audiobook = new Audiobook()
     audiobook.setData(audiobookData)
 
     if (audiobookData.audioFiles.length) {
-      var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
+      await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData, audiobook, preferAudioMetadata, libraryScan)
-      Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
+    }
-      if (audioScanResult.audioFiles.length) {
+
-        // validate & add audio files to audiobook
+    if (!audiobook.audioFilesToInclude.length && !audiobook.ebooks.length) {
-        for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
+      // Audiobook has no ebooks and no valid audio tracks do not continue
-          var newAF = audioScanResult.audioFiles[i]
+      Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
-          var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
+      return null
-          if (trackIndex !== null) {
-            if (audiobook.checkHasTrackNum(trackIndex)) {
-              newAF.setDuplicateTrackNumber(trackIndex)
-            } else {
-              newAF.index = trackIndex
-            }
-          }
-          audiobook.addAudioFile(newAF)
-        }
-        audiobook.rebuildTracks()
-      } else if (!audiobook.ebooks.length) {
-        // Audiobook has no ebooks and no valid audio tracks do not continue
-        Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
-        return null
-      }
     }
 
     // Look for desc.txt and reader.txt and update
-    await audiobook.saveDataFromTextFiles()
+    await audiobook.saveDataFromTextFiles(preferOpfMetadata)
 
     // Extract embedded cover art if cover is not already in directory
     if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
       var outputCoverDirs = this.getCoverDirectory(audiobook)
       var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
       if (relativeDir) {
-        Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
+        if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Saved embedded cover art "${relativeDir}"`)
+        else Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
       }
     }
 
+    // Scan for cover if enabled and has no cover
+    if (audiobook && findCovers && !audiobook.cover && audiobook.book.shouldSearchForCover) {
+      var updatedCover = await this.searchForCover(audiobook, libraryScan)
+      audiobook.book.updateLastCoverSearch(updatedCover)
+    }
+
     return audiobook
   }
+
+  getFileUpdatesGrouped(fileUpdates) {
+    var folderGroups = {}
+    fileUpdates.forEach((file) => {
+      if (folderGroups[file.folderId]) {
+        folderGroups[file.folderId].fileUpdates.push(file)
+      } else {
+        folderGroups[file.folderId] = {
+          libraryId: file.libraryId,
+          folderId: file.folderId,
+          fileUpdates: [file]
+        }
+      }
+    })
+    return folderGroups
+  }
+
+  async scanFilesChanged(fileUpdates) {
+    if (!fileUpdates.length) return
+    // files grouped by folder
+    var folderGroups = this.getFileUpdatesGrouped(fileUpdates)
+
+    for (const folderId in folderGroups) {
+      var libraryId = folderGroups[folderId].libraryId
+      var library = this.db.libraries.find(lib => lib.id === libraryId)
+      if (!library) {
+        Logger.error(`[Scanner] Library not found in files changed ${libraryId}`)
+        continue;
+      }
+      var folder = library.getFolderById(folderId)
+      if (!folder) {
+        Logger.error(`[Scanner] Folder is not in library in files changed "${folderId}", Library "${library.name}"`)
+        continue;
+      }
+      var relFilePaths = folderGroups[folderId].fileUpdates.map(fileUpdate => fileUpdate.relPath)
+      var fileUpdateBookGroup = groupFilesIntoAudiobookPaths(relFilePaths, true)
+      var folderScanResults = await this.scanFolderUpdates(library, folder, fileUpdateBookGroup)
+      Logger.debug(`[Scanner] Folder scan results`, folderScanResults)
+    }
+  }
+
+  async scanFolderUpdates(library, folder, fileUpdateBookGroup) {
+    Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
+
+    var bookGroupingResults = {}
+    for (const bookDir in fileUpdateBookGroup) {
+      var fullPath = Path.posix.join(folder.fullPath.replace(/\\/g, '/'), bookDir)
+
+      // Check if book dir group is already an audiobook or in a subdir of an audiobook
+      var existingAudiobook = this.db.audiobooks.find(ab => fullPath.startsWith(ab.fullPath))
+      if (existingAudiobook) {
+
+        // Is the audiobook exactly - check if was deleted
+        if (existingAudiobook.fullPath === fullPath) {
+          var exists = await fs.pathExists(fullPath)
+          if (!exists) {
+            Logger.info(`[Scanner] Scanning file update group and audiobook was deleted "${existingAudiobook.title}" - marking as missing`)
+            existingAudiobook.setMissing()
+            await this.db.updateAudiobook(existingAudiobook)
+            this.emitter('audiobook_updated', existingAudiobook.toJSONExpanded())
+
+            bookGroupingResults[bookDir] = ScanResult.REMOVED
+            continue;
+          }
+        }
+
+        // Scan audiobook for updates
+        Logger.debug(`[Scanner] Folder update for relative path "${bookDir}" is in audiobook "${existingAudiobook.title}" - scan for updates`)
+        bookGroupingResults[bookDir] = await this.scanAudiobook(folder, existingAudiobook)
+        continue;
+      }
+
+      // Check if an audiobook is a subdirectory of this dir
+      var childAudiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(fullPath))
+      if (childAudiobook) {
+        Logger.warn(`[Scanner] Files were modified in a parent directory of an audiobook "${childAudiobook.title}" - ignoring`)
+        bookGroupingResults[bookDir] = ScanResult.NOTHING
+        continue;
+      }
+
+      Logger.debug(`[Scanner] Folder update group must be a new book "${bookDir}" in library "${library.name}"`)
+      var newAudiobook = await this.scanPotentialNewAudiobook(folder, fullPath)
+      if (newAudiobook) {
+        await this.db.insertEntity('audiobook', newAudiobook)
+        this.emitter('audiobook_added', newAudiobook.toJSONExpanded())
+      }
+      bookGroupingResults[bookDir] = newAudiobook ? ScanResult.ADDED : ScanResult.NOTHING
+    }
+
+    return bookGroupingResults
+  }
+
+  async scanPotentialNewAudiobook(folder, fullPath) {
+    var audiobookData = await getAudiobookFileData(folder, fullPath, this.db.serverSettings)
+    if (!audiobookData) return null
+    var serverSettings = this.db.serverSettings
+    return this.scanNewAudiobook(audiobookData, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers)
+  }
+
+  async searchForCover(audiobook, libraryScan = null) {
+    var options = {
+      titleDistance: 2,
+      authorDistance: 2
+    }
+    var results = await this.bookFinder.findCovers('google', audiobook.title, audiobook.authorFL, options)
+    if (results.length) {
+      if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${audiobook.title}"`)
+      else Logger.debug(`[Scanner] Found best cover for "${audiobook.title}"`)
+
+      // If the first cover result fails, attempt to download the second
+      for (let i = 0; i < results.length && i < 2; i++) {
+
+        // Downloads and updates the book cover
+        var result = await this.coverController.downloadCoverFromUrl(audiobook, results[i])
+
+        if (result.error) {
+          Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
+        } else {
+          return true
+        }
+      }
+    }
+    return false
+  }
 }
 module.exports = Scanner

server/utils/constants.js

@@ -6,14 +6,6 @@ module.exports.ScanResult = {
   UPTODATE: 4
 }
 
-module.exports.ScanStatus = {
-  NOTHING: 0,
-  ADDED: 1,
-  UPDATED: 2,
-  REMOVED: 3,
-  UPTODATE: 4
-}
-
 module.exports.CoverDestination = {
   METADATA: 0,
   AUDIOBOOK: 1

server/utils/scandir.js

@@ -16,7 +16,7 @@ function isBookFile(path) {
 // TODO: Function needs to be re-done
 // Input: array of relative file paths
 // Output: map of files grouped into potential audiobook dirs
-function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) {
+function groupFilesIntoAudiobookPaths(paths) {
   // Step 1: Clean path, Remove leading "/", Filter out files in root dir
   var pathsFiltered = paths.map(path => {
     return path.startsWith('/') ? path.slice(1) : path