mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2024-10-06 02:01:58 +02:00
New data model scanner update and change scan chunks to be based on total file size
This commit is contained in:
parent
14a8f84446
commit
f00b120e96
@ -141,6 +141,11 @@ class LibraryItem {
|
||||
this.libraryFiles.forEach((lf) => total += lf.metadata.size)
|
||||
return total
|
||||
}
|
||||
get audioFileTotalSize() {
|
||||
var total = 0
|
||||
this.libraryFiles.filter(lf => lf.fileType == 'audio').forEach((lf) => total += lf.metadata.size)
|
||||
return total
|
||||
}
|
||||
get hasAudioFiles() {
|
||||
return this.libraryFiles.some(lf => lf.fileType === 'audio')
|
||||
}
|
||||
@ -347,7 +352,9 @@ class LibraryItem {
|
||||
return true
|
||||
})
|
||||
if (filesRemoved.length) {
|
||||
this.media.checkUpdateMissingTracks()
|
||||
if (this.media.audiobooks && this.media.audiobooks.length) {
|
||||
this.media.audiobooks.forEach(ab => ab.checkUpdateMissingTracks())
|
||||
}
|
||||
hasUpdated = true
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
const Path = require('path')
|
||||
const AudioFile = require('../files/AudioFile')
|
||||
const { areEquivalent, copyValue } = require('../../utils/index')
|
||||
const { areEquivalent, copyValue, getId } = require('../../utils/index')
|
||||
const AudioTrack = require('../files/AudioTrack')
|
||||
|
||||
class Audiobook {
|
||||
@ -93,6 +93,14 @@ class Audiobook {
|
||||
return this.audioFiles.some(af => af.embeddedCoverArt)
|
||||
}
|
||||
|
||||
setData(name, index) {
|
||||
this.id = getId('ab')
|
||||
this.name = name
|
||||
this.index = index
|
||||
this.addedAt = Date.now()
|
||||
this.updatedAt = Date.now()
|
||||
}
|
||||
|
||||
update(payload) {
|
||||
var json = this.toJSON()
|
||||
var hasUpdates = false
|
||||
|
@ -1,5 +1,5 @@
|
||||
const EBookFile = require('../files/EBookFile')
|
||||
const { areEquivalent, copyValue } = require('../../utils/index')
|
||||
const { areEquivalent, copyValue, getId } = require('../../utils/index')
|
||||
|
||||
class EBook {
|
||||
constructor(ebook) {
|
||||
@ -64,6 +64,15 @@ class EBook {
|
||||
return this.ebookFile.metadata.size
|
||||
}
|
||||
|
||||
setData(ebookFile, index) {
|
||||
this.id = getId('eb')
|
||||
this.name = ebookFile.metadata.filename
|
||||
this.index = index
|
||||
this.ebookFile = ebookFile
|
||||
this.addedAt = Date.now()
|
||||
this.updatedAt = Date.now()
|
||||
}
|
||||
|
||||
findFileWithInode(inode) {
|
||||
return this.ebookFile.ino === inode
|
||||
}
|
||||
|
@ -6,6 +6,7 @@ const { areEquivalent, copyValue } = require('../../utils/index')
|
||||
const { parseOpfMetadataXML } = require('../../utils/parseOpfMetadata')
|
||||
const { readTextFile } = require('../../utils/fileUtils')
|
||||
|
||||
const EBookFile = require('../files/EBookFile')
|
||||
const Audiobook = require('../entities/Audiobook')
|
||||
const EBook = require('../entities/EBook')
|
||||
|
||||
@ -267,9 +268,30 @@ class Book {
|
||||
}
|
||||
|
||||
addEbookFile(libraryFile) {
|
||||
// var newEbook = new EBookFile()
|
||||
// newEbook.setData(libraryFile)
|
||||
// this.ebookFiles.push(newEbook)
|
||||
var ebookFile = new EBookFile()
|
||||
ebookFile.setData(libraryFile)
|
||||
|
||||
var ebookIndex = this.ebooks.length + 1
|
||||
var newEBook = new EBook()
|
||||
newEBook.setData(ebookFile, ebookIndex)
|
||||
this.ebooks.push(newEBook)
|
||||
}
|
||||
|
||||
getCreateAudiobookVariant(variant) {
|
||||
if (this.audiobooks.length) {
|
||||
var ab = this.audiobooks.find(ab => ab.name == variantName)
|
||||
if (ab) return ab
|
||||
}
|
||||
var abIndex = this.audiobooks.length + 1
|
||||
var newAb = new Audiobook()
|
||||
newAb.setData(variant, abIndex)
|
||||
this.audiobooks.push(newAb)
|
||||
return newAb
|
||||
}
|
||||
|
||||
addAudioFileToAudiobook(audioFile, variant = 'default') { // Create if none
|
||||
var audiobook = this.getCreateAudiobookVariant(variant)
|
||||
audiobook.audioFiles.push(audioFile)
|
||||
}
|
||||
}
|
||||
module.exports = Book
|
@ -169,11 +169,12 @@ class AudioFileScanner {
|
||||
if (existingAF) {
|
||||
if (existingAF.updateFromScan) existingAF.updateFromScan(audioFiles[i])
|
||||
} else {
|
||||
libraryItem.media.audioFiles.push(audioFiles[i])
|
||||
libraryItem.media.addAudioFileToAudiobook(audioFiles[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: support for multiple audiobooks in a book item will need to pass an audiobook variant name here
|
||||
async scanAudioFiles(audioLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) {
|
||||
var hasUpdated = false
|
||||
|
||||
@ -195,14 +196,13 @@ class AudioFileScanner {
|
||||
if (totalAudioFilesToInclude === 1) {
|
||||
var af = audioScanResult.audioFiles[0]
|
||||
af.index = 1
|
||||
libraryItem.media.audioFiles.push(af)
|
||||
libraryItem.media.addAudioFileToAudiobook(af)
|
||||
hasUpdated = true
|
||||
} else {
|
||||
this.runSmartTrackOrder(libraryItem, audioScanResult.audioFiles)
|
||||
hasUpdated = true
|
||||
}
|
||||
} else {
|
||||
Logger.debug(`[AudioFileScanner] No audio track re-order required`)
|
||||
// Only update metadata not index
|
||||
audioScanResult.audioFiles.forEach((af) => {
|
||||
var existingAF = libraryItem.media.findFileWithInode(af.ino)
|
||||
@ -221,7 +221,12 @@ class AudioFileScanner {
|
||||
}
|
||||
|
||||
if (hasUpdated) {
|
||||
libraryItem.media.rebuildTracks()
|
||||
if (!libraryItem.media.audiobooks.length) {
|
||||
Logger.error(`[AudioFileScanner] Updates were made but library item has no audiobooks`, libraryItem)
|
||||
} else {
|
||||
var audiobook = libraryItem.media.audiobooks[0]
|
||||
audiobook.rebuildTracks()
|
||||
}
|
||||
}
|
||||
} // End Book media type
|
||||
}
|
||||
|
@ -3,14 +3,12 @@ const Path = require('path')
|
||||
|
||||
// Utils
|
||||
const Logger = require('../Logger')
|
||||
const { version } = require('../../package.json')
|
||||
const { groupFilesIntoLibraryItemPaths, getLibraryItemFileData, scanFolder } = require('../utils/scandir')
|
||||
const { comparePaths, getId } = require('../utils/index')
|
||||
const { comparePaths } = require('../utils/index')
|
||||
const { ScanResult, LogLevel } = require('../utils/constants')
|
||||
|
||||
const AudioFileScanner = require('./AudioFileScanner')
|
||||
const BookFinder = require('../finders/BookFinder')
|
||||
const Audiobook = require('../objects/legacy/Audiobook')
|
||||
const LibraryItem = require('../objects/LibraryItem')
|
||||
const LibraryScan = require('./LibraryScan')
|
||||
const ScanOptions = require('./ScanOptions')
|
||||
@ -181,13 +179,14 @@ class Scanner {
|
||||
libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino)
|
||||
var libraryItemsInLibrary = this.db.libraryItems.filter(li => li.libraryId === libraryScan.libraryId)
|
||||
|
||||
const NumScansPerChunk = 25
|
||||
const itemsToUpdateChunks = []
|
||||
const MaxSizePerChunk = 2.5e9
|
||||
const itemDataToRescanChunks = []
|
||||
const newItemDataToScanChunks = []
|
||||
var itemsToUpdate = []
|
||||
var itemDataToRescan = []
|
||||
var itemDataToRescanSize = 0
|
||||
var newItemDataToScan = []
|
||||
var newItemDataToScanSize = 0
|
||||
var itemsToFindCovers = []
|
||||
|
||||
// Check for existing & removed library items
|
||||
@ -200,40 +199,37 @@ class Scanner {
|
||||
libraryScan.resultsMissing++
|
||||
libraryItem.setMissing()
|
||||
itemsToUpdate.push(libraryItem)
|
||||
if (itemsToUpdate.length === NumScansPerChunk) {
|
||||
itemsToUpdateChunks.push(itemsToUpdate)
|
||||
itemsToUpdate = []
|
||||
}
|
||||
} else {
|
||||
var checkRes = libraryItem.checkScanData(dataFound)
|
||||
if (checkRes.newLibraryFiles.length || libraryScan.scanOptions.forceRescan) { // Item has new files
|
||||
checkRes.libraryItem = libraryItem
|
||||
checkRes.scanData = dataFound
|
||||
itemDataToRescan.push(checkRes)
|
||||
if (itemDataToRescan.length === NumScansPerChunk) {
|
||||
|
||||
// If this item will go over max size then push current chunk
|
||||
if (libraryItem.audioFileTotalSize + itemDataToRescanSize > MaxSizePerChunk && itemDataToRescan.length > 0) {
|
||||
itemDataToRescanChunks.push(itemDataToRescan)
|
||||
itemDataToRescanSize = 0
|
||||
itemDataToRescan = []
|
||||
}
|
||||
} else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) {
|
||||
|
||||
itemDataToRescan.push(checkRes)
|
||||
itemDataToRescanSize += libraryItem.audioFileTotalSize
|
||||
if (itemDataToRescanSize >= MaxSizePerChunk) {
|
||||
itemDataToRescanChunks.push(itemDataToRescan)
|
||||
itemDataToRescanSize = 0
|
||||
itemDataToRescan = []
|
||||
}
|
||||
} else if (libraryScan.findCovers && libraryItem.media.shouldSearchForCover) { // Search cover
|
||||
libraryScan.resultsUpdated++
|
||||
itemsToFindCovers.push(libraryItem)
|
||||
itemsToUpdate.push(libraryItem)
|
||||
if (itemsToUpdate.length === NumScansPerChunk) {
|
||||
itemsToUpdateChunks.push(itemsToUpdate)
|
||||
itemsToUpdate = []
|
||||
}
|
||||
} else if (checkRes.updated) { // Updated but no scan required
|
||||
libraryScan.resultsUpdated++
|
||||
itemsToUpdate.push(libraryItem)
|
||||
if (itemsToUpdate.length === NumScansPerChunk) {
|
||||
itemsToUpdateChunks.push(itemsToUpdate)
|
||||
itemsToUpdate = []
|
||||
}
|
||||
}
|
||||
libraryItemDataFound = libraryItemDataFound.filter(lid => lid.ino !== dataFound.ino)
|
||||
}
|
||||
}
|
||||
if (itemsToUpdate.length) itemsToUpdateChunks.push(itemsToUpdate)
|
||||
if (itemDataToRescan.length) itemDataToRescanChunks.push(itemDataToRescan)
|
||||
|
||||
// Potential NEW Library Items
|
||||
@ -244,9 +240,21 @@ class Scanner {
|
||||
if (!hasMediaFile) {
|
||||
libraryScan.addLog(LogLevel.WARN, `Directory found "${libraryItemDataFound.path}" has no media files`)
|
||||
} else {
|
||||
newItemDataToScan.push(dataFound)
|
||||
if (newItemDataToScan.length === NumScansPerChunk) {
|
||||
var audioFileSize = 0
|
||||
dataFound.libraryFiles.filter(lf => lf.fileType == 'audio').forEach(lf => audioFileSize += lf.metadata.size)
|
||||
|
||||
// If this item will go over max size then push current chunk
|
||||
if (audioFileSize + newItemDataToScanSize > MaxSizePerChunk && newItemDataToScan.length > 0) {
|
||||
newItemDataToScanChunks.push(newItemDataToScan)
|
||||
newItemDataToScanSize = 0
|
||||
newItemDataToScan = []
|
||||
}
|
||||
|
||||
newItemDataToScan.push(dataFound)
|
||||
newItemDataToScanSize += audioFileSize
|
||||
if (newItemDataToScanSize >= MaxSizePerChunk) {
|
||||
newItemDataToScanChunks.push(newItemDataToScan)
|
||||
newItemDataToScanSize = 0
|
||||
newItemDataToScan = []
|
||||
}
|
||||
}
|
||||
@ -260,10 +268,9 @@ class Scanner {
|
||||
libraryItem.media.updateLastCoverSearch(updatedCover)
|
||||
}
|
||||
|
||||
for (let i = 0; i < itemsToUpdateChunks.length; i++) {
|
||||
await this.updateLibraryItemChunk(itemsToUpdateChunks[i])
|
||||
if (itemsToUpdate.length) {
|
||||
await this.updateLibraryItemChunk(itemsToUpdate)
|
||||
if (this.cancelLibraryScan[libraryScan.libraryId]) return true
|
||||
// console.log('Update chunk done', i, 'of', itemsToUpdateChunks.length)
|
||||
}
|
||||
for (let i = 0; i < itemDataToRescanChunks.length; i++) {
|
||||
await this.rescanLibraryItemDataChunk(itemDataToRescanChunks[i], libraryScan)
|
||||
|
Loading…
Reference in New Issue
Block a user