mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2025-02-05 13:01:12 +01:00
New scanner updates with library scan data
This commit is contained in:
advplyr
parent
14128f3e29
commit
bf11d266dc
36
server/Db.js
36
server/Db.js
@ -201,6 +201,20 @@ class Db {
|
||||
})
|
||||
}
|
||||
|
||||
insertEntities(entityName, entities) {
|
||||
var entityDb = this.getEntityDb(entityName)
|
||||
return entityDb.insert(entities).then((results) => {
|
||||
Logger.debug(`[DB] Inserted ${results.inserted} ${entityName}`)
|
||||
|
||||
var arrayKey = this.getEntityArrayKey(entityName)
|
||||
this[arrayKey] = this[arrayKey].concat(entities)
|
||||
return true
|
||||
}).catch((error) => {
|
||||
Logger.error(`[DB] Failed to insert ${entityName}`, error)
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
insertEntity(entityName, entity) {
|
||||
var entityDb = this.getEntityDb(entityName)
|
||||
return entityDb.insert([entity]).then((results) => {
|
||||
@ -215,6 +229,26 @@ class Db {
|
||||
})
|
||||
}
|
||||
|
||||
updateEntities(entityName, entities) {
|
||||
var entityDb = this.getEntityDb(entityName)
|
||||
|
||||
var entityIds = entities.map(ent => ent.id)
|
||||
return entityDb.update((record) => entityIds.includes(record.id), (record) => {
|
||||
return entities.find(ent => ent.id === record.id)
|
||||
}).then((results) => {
|
||||
Logger.debug(`[DB] Updated ${entityName}: ${results.updated}`)
|
||||
var arrayKey = this.getEntityArrayKey(entityName)
|
||||
this[arrayKey] = this[arrayKey].map(e => {
|
||||
if (entityIds.includes(e.id)) return entities.find(_e => _e.id === e.id)
|
||||
return e
|
||||
})
|
||||
return true
|
||||
}).catch((error) => {
|
||||
Logger.error(`[DB] Update ${entityName} Failed: ${error}`)
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
updateEntity(entityName, entity) {
|
||||
var entityDb = this.getEntityDb(entityName)
|
||||
|
||||
@ -224,7 +258,7 @@ class Db {
|
||||
}
|
||||
|
||||
return entityDb.update((record) => record.id === entity.id, () => jsonEntity).then((results) => {
|
||||
Logger.debug(`[DB] Updated entity ${entityName}: ${results.updated}`)
|
||||
Logger.debug(`[DB] Updated ${entityName}: ${results.updated}`)
|
||||
var arrayKey = this.getEntityArrayKey(entityName)
|
||||
this[arrayKey] = this[arrayKey].map(e => {
|
||||
return e.id === entity.id ? entity : e
|
||||
|
||||
@ -6,8 +6,7 @@ const Logger = require('./Logger')
|
||||
const { version } = require('../package.json')
|
||||
const audioFileScanner = require('./utils/audioFileScanner')
|
||||
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('./utils/scandir')
|
||||
const { comparePaths, getIno, getId } = require('./utils/index')
|
||||
const { secondsToTimestamp } = require('./utils/fileUtils')
|
||||
const { comparePaths, getIno, getId, secondsToTimestamp } = require('./utils/index')
|
||||
const { ScanResult, CoverDestination } = require('./utils/constants')
|
||||
|
||||
const BookFinder = require('./BookFinder')
|
||||
@ -440,7 +439,6 @@ class Scanner {
|
||||
var scanPayload = {
|
||||
id: libraryId,
|
||||
name: library.name,
|
||||
scanType: 'library',
|
||||
folders: library.folders.length
|
||||
}
|
||||
this.emitter('scan_start', scanPayload)
|
||||
@ -489,7 +487,7 @@ class Scanner {
|
||||
Logger.info(`[Scanner] Canceling scan ${libraryId}`)
|
||||
delete this.cancelLibraryScan[libraryId]
|
||||
this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId)
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: null })
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, results: null })
|
||||
return null
|
||||
}
|
||||
|
||||
@ -516,7 +514,7 @@ class Scanner {
|
||||
Logger.info(`[Scanner] Canceling scan ${libraryId}`)
|
||||
delete this.cancelLibraryScan[libraryId]
|
||||
this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId)
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: scanResults })
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, results: scanResults })
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -532,7 +530,6 @@ class Scanner {
|
||||
this.emitter('scan_progress', {
|
||||
id: libraryId,
|
||||
name: library.name,
|
||||
scanType: 'library',
|
||||
progress: {
|
||||
total: audiobookDataFound.length,
|
||||
done: i + 1,
|
||||
@ -548,7 +545,7 @@ class Scanner {
|
||||
const scanElapsed = Math.floor((Date.now() - scanStart) / 1000)
|
||||
Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | ${scanResults.missing} missing | elapsed: ${secondsToTimestamp(scanElapsed)}`)
|
||||
this.librariesScanning = this.librariesScanning.filter(l => l.id !== libraryId)
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, scanType: 'library', results: scanResults })
|
||||
this.emitter('scan_complete', { id: libraryId, name: library.name, results: scanResults })
|
||||
}
|
||||
|
||||
async scanAudiobookById(audiobookId) {
|
||||
|
||||
@ -11,13 +11,14 @@ const { version } = require('../package.json')
|
||||
// Utils
|
||||
const { ScanResult } = require('./utils/constants')
|
||||
const filePerms = require('./utils/filePerms')
|
||||
const { secondsToTimestamp } = require('./utils/fileUtils')
|
||||
const { secondsToTimestamp } = require('./utils/index')
|
||||
const Logger = require('./Logger')
|
||||
|
||||
// Classes
|
||||
const Auth = require('./Auth')
|
||||
const Watcher = require('./Watcher')
|
||||
const Scanner = require('./Scanner')
|
||||
const Scanner2 = require('./scanner/Scanner')
|
||||
const Db = require('./Db')
|
||||
const BackupManager = require('./BackupManager')
|
||||
const LogManager = require('./LogManager')
|
||||
@ -49,6 +50,8 @@ class Server {
|
||||
this.watcher = new Watcher(this.AudiobookPath)
|
||||
this.coverController = new CoverController(this.db, this.MetadataPath, this.AudiobookPath)
|
||||
this.scanner = new Scanner(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this))
|
||||
this.scanner2 = new Scanner2(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this))
|
||||
|
||||
this.streamManager = new StreamManager(this.db, this.MetadataPath, this.emitter.bind(this), this.clientEmitter.bind(this))
|
||||
this.rssFeeds = new RssFeeds(this.Port, this.db)
|
||||
this.downloadManager = new DownloadManager(this.db, this.MetadataPath, this.AudiobookPath, this.emitter.bind(this))
|
||||
@ -314,7 +317,8 @@ class Server {
|
||||
|
||||
async scan(libraryId, forceAudioFileScan = false) {
|
||||
Logger.info('[Server] Starting Scan')
|
||||
await this.scanner.scan(libraryId, forceAudioFileScan)
|
||||
// await this.scanner2.scan(libraryId)
|
||||
await this.scanner(libraryId, forceAudioFileScan)
|
||||
Logger.info('[Server] Scan complete')
|
||||
}
|
||||
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
const { isNullOrNaN } = require('../utils/index')
|
||||
|
||||
const Logger = require('../Logger')
|
||||
const AudioFileMetadata = require('./AudioFileMetadata')
|
||||
|
||||
@ -154,7 +156,7 @@ class AudioFile {
|
||||
}
|
||||
|
||||
// New scanner creates AudioFile from AudioFileScanner
|
||||
setData2(fileData, probeData) {
|
||||
setDataFromProbe(fileData, probeData) {
|
||||
this.index = fileData.index || null
|
||||
this.ino = fileData.ino || null
|
||||
this.filename = fileData.filename
|
||||
@ -162,6 +164,42 @@ class AudioFile {
|
||||
this.path = fileData.path
|
||||
this.fullPath = fileData.fullPath
|
||||
this.addedAt = Date.now()
|
||||
|
||||
this.trackNumFromMeta = fileData.trackNumFromMeta || null
|
||||
this.trackNumFromFilename = fileData.trackNumFromFilename || null
|
||||
this.cdNumFromFilename = fileData.cdNumFromFilename || null
|
||||
|
||||
this.format = probeData.format
|
||||
this.duration = probeData.duration
|
||||
this.size = probeData.size
|
||||
this.bitRate = probeData.bitRate || null
|
||||
this.language = probeData.language
|
||||
this.codec = probeData.codec || null
|
||||
this.timeBase = probeData.timeBase
|
||||
this.channels = probeData.channels
|
||||
this.channelLayout = probeData.channelLayout
|
||||
this.chapters = probeData.chapters || []
|
||||
this.metadata = probeData.audioFileMetadata
|
||||
}
|
||||
|
||||
validateTrackIndex(isSingleTrack) {
|
||||
var numFromMeta = isNullOrNaN(this.trackNumFromMeta) ? null : Number(this.trackNumFromMeta)
|
||||
var numFromFilename = isNullOrNaN(this.trackNumFromFilename) ? null : Number(this.trackNumFromFilename)
|
||||
|
||||
if (isSingleTrack) { // Single audio track audiobook only use metadata tag and default to 1
|
||||
return numFromMeta ? numFromMeta : 1
|
||||
}
|
||||
if (numFromMeta !== null) return numFromMeta
|
||||
if (numFromFilename !== null) return numFromFilename
|
||||
|
||||
this.invalid = true
|
||||
this.error = 'Failed to get track number'
|
||||
return null
|
||||
}
|
||||
|
||||
setDuplicateTrackNumber(num) {
|
||||
this.invalid = true
|
||||
this.error = 'Duplicate track number "' + num + '"'
|
||||
}
|
||||
|
||||
syncChapters(updatedChapters) {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
const Path = require('path')
|
||||
const fs = require('fs-extra')
|
||||
const { bytesPretty, elapsedPretty, readTextFile } = require('../utils/fileUtils')
|
||||
const { comparePaths, getIno, getId } = require('../utils/index')
|
||||
const { bytesPretty, readTextFile } = require('../utils/fileUtils')
|
||||
const { comparePaths, getIno, getId, elapsedPretty } = require('../utils/index')
|
||||
const { parseOpfMetadataXML } = require('../utils/parseOpfMetadata')
|
||||
const { extractCoverArt } = require('../utils/ffmpegHelpers')
|
||||
const nfoGenerator = require('../utils/nfoGenerator')
|
||||
@ -128,6 +128,8 @@ class Audiobook {
|
||||
get _otherFiles() { return this.otherFiles || [] }
|
||||
get _tracks() { return this.tracks || [] }
|
||||
|
||||
get audioFilesToInclude() { return this._audioFiles.filter(af => !af.exclude) }
|
||||
|
||||
get ebooks() {
|
||||
return this.otherFiles.filter(file => file.filetype === 'ebook')
|
||||
}
|
||||
@ -346,6 +348,11 @@ class Audiobook {
|
||||
this.scanVersion = version
|
||||
}
|
||||
|
||||
setMissing() {
|
||||
this.isMissing = true
|
||||
this.lastUpdate = Date.now()
|
||||
}
|
||||
|
||||
setBook(data) {
|
||||
// Use first image file as cover
|
||||
if (this.otherFiles && this.otherFiles.length) {
|
||||
@ -353,7 +360,6 @@ class Audiobook {
|
||||
if (imageFile) {
|
||||
data.coverFullPath = imageFile.fullPath
|
||||
var relImagePath = imageFile.path.replace(this.path, '')
|
||||
console.log('SET BOOK PATH', imageFile.path, 'REPLACE', this.path, 'RESULT', relImagePath)
|
||||
data.cover = Path.posix.join(`/s/book/${this.id}`, relImagePath)
|
||||
}
|
||||
}
|
||||
@ -383,10 +389,15 @@ class Audiobook {
|
||||
}
|
||||
|
||||
addAudioFile(audioFileData) {
|
||||
var audioFile = new AudioFile()
|
||||
audioFile.setData(audioFileData)
|
||||
this.audioFiles.push(audioFile)
|
||||
return audioFile
|
||||
if (audioFileData instanceof AudioFile) {
|
||||
this.audioFiles.push(audioFileData)
|
||||
return audioFileData
|
||||
} else {
|
||||
var audioFile = new AudioFile()
|
||||
audioFile.setData(audioFileData)
|
||||
this.audioFiles.push(audioFile)
|
||||
return audioFile
|
||||
}
|
||||
}
|
||||
|
||||
addOtherFile(fileData) {
|
||||
@ -426,6 +437,10 @@ class Audiobook {
|
||||
return this.book.updateCover(cover, coverFullPath)
|
||||
}
|
||||
|
||||
checkHasTrackNum(trackNum) {
|
||||
return this.tracks.find(t => t.index === trackNum)
|
||||
}
|
||||
|
||||
updateAudioTracks(orderedFileData) {
|
||||
var index = 1
|
||||
this.audioFiles = orderedFileData.map((fileData) => {
|
||||
@ -444,8 +459,12 @@ class Audiobook {
|
||||
return audioFile
|
||||
})
|
||||
|
||||
this.audioFiles.sort((a, b) => a.index - b.index)
|
||||
this.rebuildTracks()
|
||||
}
|
||||
|
||||
// After audio files have been added/removed/updated this method sets tracks
|
||||
rebuildTracks() {
|
||||
this.audioFiles.sort((a, b) => a.index - b.index)
|
||||
this.tracks = []
|
||||
this.missingParts = []
|
||||
this.audioFiles.forEach((file) => {
|
||||
@ -570,7 +589,6 @@ class Audiobook {
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
var imageFiles = this.otherFiles.filter(f => f.filetype === 'image')
|
||||
|
||||
// OLD Path Check if cover was a local image and that it still exists
|
||||
@ -866,7 +884,7 @@ class Audiobook {
|
||||
return hasUpdated
|
||||
}
|
||||
|
||||
checkShouldScan(dataFound) {
|
||||
checkScanData(dataFound) {
|
||||
var hasUpdated = false
|
||||
|
||||
if (dataFound.ino !== this.ino) {
|
||||
|
||||
@ -12,6 +12,8 @@ class ServerSettings {
|
||||
// Scanner
|
||||
this.scannerParseSubtitle = false
|
||||
this.scannerFindCovers = false
|
||||
this.scannerPreferAudioMetadata = false
|
||||
this.scannerPreferOpfMetadata = false
|
||||
|
||||
// Metadata
|
||||
this.coverDestination = CoverDestination.METADATA
|
||||
|
||||
@ -3,8 +3,7 @@ const EventEmitter = require('events')
|
||||
const Path = require('path')
|
||||
const fs = require('fs-extra')
|
||||
const Logger = require('../Logger')
|
||||
const { getId } = require('../utils/index')
|
||||
const { secondsToTimestamp } = require('../utils/fileUtils')
|
||||
const { getId, secondsToTimestamp } = require('../utils/index')
|
||||
const { writeConcatFile } = require('../utils/ffmpegHelpers')
|
||||
const hlsPlaylistGenerator = require('../utils/hlsPlaylistGenerator')
|
||||
|
||||
|
||||
@ -1,22 +1,106 @@
|
||||
const Path = require('path')
|
||||
|
||||
const AudioFile = require('../objects/AudioFile')
|
||||
const AudioProbeData = require('./AudioProbeData')
|
||||
|
||||
const prober = require('../utils/prober')
|
||||
const Logger = require('../Logger')
|
||||
const { msToTimestamp } = require('../utils')
|
||||
|
||||
class AudioFileScanner {
|
||||
constructor() { }
|
||||
|
||||
async scan(audioFileData, verbose = false) {
|
||||
getTrackNumberFromMeta(scanData) {
|
||||
return !isNaN(scanData.trackNumber) && scanData.trackNumber !== null ? Math.trunc(Number(scanData.trackNumber)) : null
|
||||
}
|
||||
|
||||
getTrackNumberFromFilename(bookScanData, filename) {
|
||||
const { title, author, series, publishYear } = bookScanData
|
||||
var partbasename = Path.basename(filename, Path.extname(filename))
|
||||
|
||||
// Remove title, author, series, and publishYear from filename if there
|
||||
if (title) partbasename = partbasename.replace(title, '')
|
||||
if (author) partbasename = partbasename.replace(author, '')
|
||||
if (series) partbasename = partbasename.replace(series, '')
|
||||
if (publishYear) partbasename = partbasename.replace(publishYear)
|
||||
|
||||
// Remove eg. "disc 1" from path
|
||||
partbasename = partbasename.replace(/\bdisc \d\d?\b/i, '')
|
||||
|
||||
// Remove "cd01" or "cd 01" from path
|
||||
partbasename = partbasename.replace(/\bcd ?\d\d?\b/i, '')
|
||||
|
||||
var numbersinpath = partbasename.match(/\d{1,4}/g)
|
||||
if (!numbersinpath) return null
|
||||
|
||||
var number = numbersinpath.length ? parseInt(numbersinpath[0]) : null
|
||||
return number
|
||||
}
|
||||
|
||||
getCdNumberFromFilename(bookScanData, filename) {
|
||||
const { title, author, series, publishYear } = bookScanData
|
||||
var partbasename = Path.basename(filename, Path.extname(filename))
|
||||
|
||||
// Remove title, author, series, and publishYear from filename if there
|
||||
if (title) partbasename = partbasename.replace(title, '')
|
||||
if (author) partbasename = partbasename.replace(author, '')
|
||||
if (series) partbasename = partbasename.replace(series, '')
|
||||
if (publishYear) partbasename = partbasename.replace(publishYear)
|
||||
|
||||
var cdNumber = null
|
||||
|
||||
var cdmatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i)
|
||||
if (cdmatch && cdmatch.length > 2 && cdmatch[2]) {
|
||||
if (!isNaN(cdmatch[2])) {
|
||||
cdNumber = Number(cdmatch[2])
|
||||
}
|
||||
}
|
||||
|
||||
return cdNumber
|
||||
}
|
||||
|
||||
getAverageScanDurationMs(results) {
|
||||
if (!results.length) return 0
|
||||
var total = 0
|
||||
for (let i = 0; i < results.length; i++) total += results[i].elapsed
|
||||
return Math.floor(total / results.length)
|
||||
}
|
||||
|
||||
async scan(audioFileData, bookScanData, verbose = false) {
|
||||
var probeStart = Date.now()
|
||||
// Logger.debug(`[AudioFileScanner] Start Probe ${audioFileData.fullPath}`)
|
||||
var probeData = await prober.probe2(audioFileData.fullPath, verbose)
|
||||
if (probeData.error) {
|
||||
Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`)
|
||||
return null
|
||||
}
|
||||
// Logger.debug(`[AudioFileScanner] Finished Probe ${audioFileData.fullPath} elapsed ${msToTimestamp(Date.now() - probeStart, true)}`)
|
||||
|
||||
var audioFile = new AudioFile()
|
||||
// TODO: Build audio file
|
||||
return audioFile
|
||||
audioFileData.trackNumFromMeta = this.getTrackNumberFromMeta(probeData)
|
||||
audioFileData.trackNumFromFilename = this.getTrackNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFileData.cdNumFromFilename = this.getCdNumberFromFilename(bookScanData, audioFileData.filename)
|
||||
audioFile.setDataFromProbe(audioFileData, probeData)
|
||||
return {
|
||||
audioFile,
|
||||
elapsed: Date.now() - probeStart
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Returns array of { AudioFile, elapsed } from audio file scan objects
|
||||
async scanAudioFiles(audioFileDataArray, bookScanData) {
|
||||
var proms = []
|
||||
for (let i = 0; i < audioFileDataArray.length; i++) {
|
||||
var prom = this.scan(audioFileDataArray[i], bookScanData)
|
||||
proms.push(prom)
|
||||
}
|
||||
var scanStart = Date.now()
|
||||
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
|
||||
return {
|
||||
audioFiles: results.map(r => r.audioFile),
|
||||
elapsed: Date.now() - scanStart,
|
||||
averageScanDuration: this.getAverageScanDurationMs(results)
|
||||
}
|
||||
}
|
||||
}
|
||||
module.exports = new AudioFileScanner()
|
||||
@ -34,7 +34,7 @@ class AudioProbeData {
|
||||
}
|
||||
|
||||
setData(data) {
|
||||
var audioStream = getDefaultAudioStream(data.audio_streams)
|
||||
var audioStream = this.getDefaultAudioStream(data.audio_streams)
|
||||
|
||||
this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
|
||||
this.format = data.format
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
const Folder = require('../objects/Folder')
|
||||
const Constants = require('../utils/constants')
|
||||
|
||||
const { getId } = require('../utils/index')
|
||||
const { getId, secondsToTimestamp } = require('../utils/index')
|
||||
|
||||
class LibraryScan {
|
||||
constructor() {
|
||||
@ -13,22 +14,49 @@ class LibraryScan {
|
||||
|
||||
this.startedAt = null
|
||||
this.finishedAt = null
|
||||
this.elapsed = null
|
||||
|
||||
this.folderScans = []
|
||||
this.status = Constants.ScanStatus.NOTHING
|
||||
this.resultsMissing = 0
|
||||
this.resultsAdded = 0
|
||||
this.resultsUpdated = 0
|
||||
}
|
||||
|
||||
get _scanOptions() { return this.scanOptions || {} }
|
||||
get forceRescan() { return !!this._scanOptions.forceRescan }
|
||||
|
||||
get resultStats() {
|
||||
return `${this.resultsAdded} Added | ${this.resultsUpdated} Updated | ${this.resultsMissing} Missing`
|
||||
}
|
||||
get elapsedTimestamp() {
|
||||
return secondsToTimestamp(this.elapsed / 1000)
|
||||
}
|
||||
get getScanEmitData() {
|
||||
return {
|
||||
id: this.libraryId,
|
||||
name: this.libraryName,
|
||||
results: {
|
||||
added: this.resultsAdded,
|
||||
updated: this.resultsUpdated,
|
||||
missing: this.resultsMissing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setData(library, scanOptions) {
|
||||
this.id = getId('lscan')
|
||||
this.libraryId = library.id
|
||||
this.libraryName = library.name
|
||||
this.folders = library.folders.map(folder => Folder(folder.toJSON()))
|
||||
this.folders = library.folders.map(folder => new Folder(folder.toJSON()))
|
||||
|
||||
this.scanOptions = scanOptions
|
||||
|
||||
this.startedAt = Date.now()
|
||||
}
|
||||
|
||||
setComplete() {
|
||||
this.finishedAt = Date.now()
|
||||
this.elapsed = this.finishedAt - this.startedAt
|
||||
}
|
||||
}
|
||||
module.exports = LibraryScan
|
||||
@ -4,33 +4,35 @@ class ScanOptions {
|
||||
constructor(options) {
|
||||
this.forceRescan = false
|
||||
|
||||
this.metadataPrecedence = [
|
||||
{
|
||||
id: 'directory',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'reader-desc-txt',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'audio-file-metadata',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'metadata-opf',
|
||||
include: true
|
||||
},
|
||||
{
|
||||
id: 'external-source',
|
||||
include: false
|
||||
}
|
||||
]
|
||||
// this.metadataPrecedence = [
|
||||
// {
|
||||
// id: 'directory',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'reader-desc-txt',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'audio-file-metadata',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'metadata-opf',
|
||||
// include: true
|
||||
// },
|
||||
// {
|
||||
// id: 'external-source',
|
||||
// include: false
|
||||
// }
|
||||
// ]
|
||||
|
||||
// Server settings
|
||||
this.parseSubtitles = false
|
||||
this.findCovers = false
|
||||
this.coverDestination = CoverDestination.METADATA
|
||||
this.preferAudioMetadata = false
|
||||
this.preferOpfMetadata = false
|
||||
|
||||
if (options) {
|
||||
this.construct(options)
|
||||
@ -53,7 +55,9 @@ class ScanOptions {
|
||||
metadataPrecedence: this.metadataPrecedence,
|
||||
parseSubtitles: this.parseSubtitles,
|
||||
findCovers: this.findCovers,
|
||||
coverDestination: this.coverDestination
|
||||
coverDestination: this.coverDestination,
|
||||
preferAudioMetadata: this.preferAudioMetadata,
|
||||
preferOpfMetadata: this.preferOpfMetadata
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,6 +67,8 @@ class ScanOptions {
|
||||
this.parseSubtitles = !!serverSettings.scannerParseSubtitle
|
||||
this.findCovers = !!serverSettings.scannerFindCovers
|
||||
this.coverDestination = serverSettings.coverDestination
|
||||
this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata
|
||||
this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata
|
||||
}
|
||||
}
|
||||
module.exports = ScanOptions
|
||||
@ -6,8 +6,7 @@ const Logger = require('../Logger')
|
||||
const { version } = require('../../package.json')
|
||||
const audioFileScanner = require('../utils/audioFileScanner')
|
||||
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
|
||||
const { comparePaths, getIno, getId } = require('../utils/index')
|
||||
const { secondsToTimestamp } = require('../utils/fileUtils')
|
||||
const { comparePaths, getIno, getId, msToTimestamp } = require('../utils/index')
|
||||
const { ScanResult, CoverDestination } = require('../utils/constants')
|
||||
|
||||
const AudioFileScanner = require('./AudioFileScanner')
|
||||
@ -33,6 +32,20 @@ class Scanner {
|
||||
this.bookFinder = new BookFinder()
|
||||
}
|
||||
|
||||
getCoverDirectory(audiobook) {
|
||||
if (this.db.serverSettings.coverDestination === CoverDestination.AUDIOBOOK) {
|
||||
return {
|
||||
fullPath: audiobook.fullPath,
|
||||
relPath: '/s/book/' + audiobook.id
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
fullPath: Path.posix.join(this.BookMetadataPath, audiobook.id),
|
||||
relPath: Path.posix.join('/metadata', 'books', audiobook.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async scan(libraryId, options = {}) {
|
||||
if (this.librariesScanning.includes(libraryId)) {
|
||||
Logger.error(`[Scanner] Already scanning ${libraryId}`)
|
||||
@ -53,14 +66,19 @@ class Scanner {
|
||||
|
||||
var libraryScan = new LibraryScan()
|
||||
libraryScan.setData(library, scanOptions)
|
||||
this.librariesScanning.push(libraryScan)
|
||||
|
||||
this.emitter('scan_start', libraryScan.getScanEmitData)
|
||||
|
||||
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
|
||||
|
||||
var results = await this.scanLibrary(libraryScan)
|
||||
await this.scanLibrary(libraryScan)
|
||||
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`)
|
||||
libraryScan.setComplete()
|
||||
Logger.info(`[Scanner] Library scan ${libraryScan.id} completed in ${libraryScan.elapsedTimestamp}. ${libraryScan.resultStats}`)
|
||||
|
||||
return results
|
||||
this.librariesScanning = this.librariesScanning.filter(ls => ls.id !== library.id)
|
||||
this.emitter('scan_complete', libraryScan.getScanEmitData)
|
||||
}
|
||||
|
||||
async scanLibrary(libraryScan) {
|
||||
@ -77,9 +95,9 @@ class Scanner {
|
||||
|
||||
var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
|
||||
|
||||
const audiobooksToUpdate = []
|
||||
const audiobooksToRescan = []
|
||||
const newAudiobookData = []
|
||||
var audiobooksToUpdate = []
|
||||
var audiobookRescans = []
|
||||
var newAudiobookScans = []
|
||||
|
||||
// Check for existing & removed audiobooks
|
||||
for (let i = 0; i < audiobooksInLibrary.length; i++) {
|
||||
@ -87,21 +105,20 @@ class Scanner {
|
||||
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
|
||||
if (!dataFound) {
|
||||
Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
|
||||
audiobook.isMissing = true
|
||||
audiobook.lastUpdate = Date.now()
|
||||
scanResults.missing++
|
||||
audiobook.setMissing()
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
} else {
|
||||
var checkRes = audiobook.checkShouldRescan(dataFound)
|
||||
var checkRes = audiobook.checkScanData(dataFound)
|
||||
if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
|
||||
// existing audiobook has new files
|
||||
checkRes.audiobook = audiobook
|
||||
audiobooksToRescan.push(checkRes)
|
||||
checkRes.bookScanData = dataFound
|
||||
audiobookRescans.push(this.rescanAudiobook(checkRes, libraryScan))
|
||||
libraryScan.resultsMissing++
|
||||
} else if (checkRes.updated) {
|
||||
audiobooksToUpdate.push(audiobook)
|
||||
libraryScan.resultsUpdated++
|
||||
}
|
||||
|
||||
// Remove this abf
|
||||
audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
|
||||
}
|
||||
}
|
||||
@ -113,60 +130,108 @@ class Scanner {
|
||||
if (!hasEbook && !dataFound.audioFiles.length) {
|
||||
Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
|
||||
} else {
|
||||
newAudiobookData.push(dataFound)
|
||||
newAudiobookScans.push(this.scanNewAudiobook(dataFound, libraryScan))
|
||||
}
|
||||
}
|
||||
|
||||
var rescans = []
|
||||
for (let i = 0; i < audiobooksToRescan.length; i++) {
|
||||
var rescan = this.rescanAudiobook(audiobooksToRescan[i])
|
||||
rescans.push(rescan)
|
||||
if (audiobookRescans.length) {
|
||||
var updatedAudiobooks = (await Promise.all(audiobookRescans)).filter(ab => !!ab)
|
||||
if (updatedAudiobooks.length) {
|
||||
audiobooksToUpdate = audiobooksToUpdate.concat(updatedAudiobooks)
|
||||
libraryScan.resultsUpdated += updatedAudiobooks.length
|
||||
}
|
||||
}
|
||||
var newscans = []
|
||||
for (let i = 0; i < newAudiobookData.length; i++) {
|
||||
var newscan = this.scanNewAudiobook(newAudiobookData[i])
|
||||
newscans.push(newscan)
|
||||
if (audiobooksToUpdate.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" updating ${audiobooksToUpdate.length} books`)
|
||||
await this.db.updateEntities('audiobook', audiobooksToUpdate)
|
||||
}
|
||||
|
||||
var rescanResults = await Promise.all(rescans)
|
||||
|
||||
var newscanResults = await Promise.all(newscans)
|
||||
|
||||
// TODO: Return report
|
||||
return {
|
||||
updates: 0,
|
||||
additions: 0
|
||||
if (newAudiobookScans.length) {
|
||||
var newAudiobooks = (await Promise.all(newAudiobookScans)).filter(ab => !!ab)
|
||||
if (newAudiobooks.length) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" inserting ${newAudiobooks.length} books`)
|
||||
await this.db.insertEntities('audiobook', newAudiobooks)
|
||||
libraryScan.resultsAdded = newAudiobooks.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return scan result payload
|
||||
async rescanAudiobook(audiobookCheckData) {
|
||||
const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData
|
||||
async rescanAudiobook(audiobookCheckData, libraryScan) {
|
||||
const { newAudioFileData, newOtherFileData, audiobook, bookScanData } = audiobookCheckData
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Re-scanning "${audiobook.path}"`)
|
||||
|
||||
if (newAudioFileData.length) {
|
||||
var newAudioFiles = await this.scanAudioFiles(newAudioFileData)
|
||||
// TODO: Update audiobook tracks
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(newAudioFileData, bookScanData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobook.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
var totalAudioFilesToInclude = audiobook.audioFilesToInclude.length + audioScanResult.audioFiles.length
|
||||
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(totalAudioFilesToInclude === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
}
|
||||
|
||||
audiobook.rebuildTracks()
|
||||
}
|
||||
}
|
||||
if (newOtherFileData.length) {
|
||||
// TODO: Check other files
|
||||
}
|
||||
|
||||
return {
|
||||
updated: true
|
||||
await audiobook.syncOtherFiles(newOtherFileData, this.MetadataPath)
|
||||
}
|
||||
return audiobook
|
||||
}
|
||||
|
||||
async scanNewAudiobook(audiobookData) {
|
||||
// TODO: Return new audiobook
|
||||
return null
|
||||
}
|
||||
async scanNewAudiobook(audiobookData, libraryScan) {
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Scanning new "${audiobookData.path}"`)
|
||||
var audiobook = new Audiobook()
|
||||
audiobook.setData(audiobookData)
|
||||
|
||||
async scanAudioFiles(audioFileData) {
|
||||
var proms = []
|
||||
for (let i = 0; i < audioFileData.length; i++) {
|
||||
var prom = AudioFileScanner.scan(audioFileData[i])
|
||||
proms.push(prom)
|
||||
if (audiobookData.audioFiles.length) {
|
||||
var audioScanResult = await AudioFileScanner.scanAudioFiles(audiobookData.audioFiles, audiobookData)
|
||||
Logger.debug(`[Scanner] Library "${libraryScan.libraryName}" Book "${audiobookData.path}" Audio file scan took ${msToTimestamp(audioScanResult.elapsed, true)} for ${audioScanResult.audioFiles.length} with average time of ${msToTimestamp(audioScanResult.averageScanDuration, true)}`)
|
||||
if (audioScanResult.audioFiles.length) {
|
||||
// validate & add audio files to audiobook
|
||||
for (let i = 0; i < audioScanResult.audioFiles.length; i++) {
|
||||
var newAF = audioScanResult.audioFiles[i]
|
||||
var trackIndex = newAF.validateTrackIndex(audioScanResult.audioFiles.length === 1)
|
||||
if (trackIndex !== null) {
|
||||
if (audiobook.checkHasTrackNum(trackIndex)) {
|
||||
newAF.setDuplicateTrackNumber(trackIndex)
|
||||
} else {
|
||||
newAF.index = trackIndex
|
||||
}
|
||||
}
|
||||
audiobook.addAudioFile(newAF)
|
||||
}
|
||||
audiobook.rebuildTracks()
|
||||
} else if (!audiobook.ebooks.length) {
|
||||
// Audiobook has no ebooks and no valid audio tracks do not continue
|
||||
Logger.warn(`[Scanner] Audiobook has no ebooks and no valid audio tracks "${audiobook.path}"`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
return Promise.all(proms)
|
||||
|
||||
// Look for desc.txt and reader.txt and update
|
||||
await audiobook.saveDataFromTextFiles()
|
||||
|
||||
// Extract embedded cover art if cover is not already in directory
|
||||
if (audiobook.hasEmbeddedCoverArt && !audiobook.cover) {
|
||||
var outputCoverDirs = this.getCoverDirectory(audiobook)
|
||||
var relativeDir = await audiobook.saveEmbeddedCoverArt(outputCoverDirs.fullPath, outputCoverDirs.relPath)
|
||||
if (relativeDir) {
|
||||
Logger.debug(`[Scanner] Saved embedded cover art "${relativeDir}"`)
|
||||
}
|
||||
}
|
||||
|
||||
return audiobook
|
||||
}
|
||||
}
|
||||
module.exports = Scanner
|
||||
@ -6,6 +6,14 @@ module.exports.ScanResult = {
|
||||
UPTODATE: 4
|
||||
}
|
||||
|
||||
module.exports.ScanStatus = {
|
||||
NOTHING: 0,
|
||||
ADDED: 1,
|
||||
UPDATED: 2,
|
||||
REMOVED: 3,
|
||||
UPTODATE: 4
|
||||
}
|
||||
|
||||
module.exports.CoverDestination = {
|
||||
METADATA: 0,
|
||||
AUDIOBOOK: 1
|
||||
|
||||
@ -51,34 +51,6 @@ function bytesPretty(bytes, decimals = 0) {
|
||||
}
|
||||
module.exports.bytesPretty = bytesPretty
|
||||
|
||||
function elapsedPretty(seconds) {
|
||||
var minutes = Math.floor(seconds / 60)
|
||||
if (minutes < 70) {
|
||||
return `${minutes} min`
|
||||
}
|
||||
var hours = Math.floor(minutes / 60)
|
||||
minutes -= hours * 60
|
||||
if (!minutes) {
|
||||
return `${hours} hr`
|
||||
}
|
||||
return `${hours} hr ${minutes} min`
|
||||
}
|
||||
module.exports.elapsedPretty = elapsedPretty
|
||||
|
||||
function secondsToTimestamp(seconds) {
|
||||
var _seconds = seconds
|
||||
var _minutes = Math.floor(seconds / 60)
|
||||
_seconds -= _minutes * 60
|
||||
var _hours = Math.floor(_minutes / 60)
|
||||
_minutes -= _hours * 60
|
||||
_seconds = Math.floor(_seconds)
|
||||
if (!_hours) {
|
||||
return `${_minutes}:${_seconds.toString().padStart(2, '0')}`
|
||||
}
|
||||
return `${_hours}:${_minutes.toString().padStart(2, '0')}:${_seconds.toString().padStart(2, '0')}`
|
||||
}
|
||||
module.exports.secondsToTimestamp = secondsToTimestamp
|
||||
|
||||
function setFileOwner(path, uid, gid) {
|
||||
try {
|
||||
return fs.chown(path, uid, gid).then(() => true)
|
||||
|
||||
@ -45,6 +45,10 @@ module.exports.getIno = (path) => {
|
||||
})
|
||||
}
|
||||
|
||||
module.exports.isNullOrNaN = (num) => {
|
||||
return num === null || isNaN(num)
|
||||
}
|
||||
|
||||
const xmlToJSON = (xml) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
parseString(xml, (err, results) => {
|
||||
@ -63,4 +67,38 @@ module.exports.getId = (prepend = '') => {
|
||||
var _id = Math.random().toString(36).substring(2, 8) + Math.random().toString(36).substring(2, 8) + Math.random().toString(36).substring(2, 8)
|
||||
if (prepend) return prepend + '_' + _id
|
||||
return _id
|
||||
}
|
||||
}
|
||||
|
||||
function elapsedPretty(seconds) {
|
||||
var minutes = Math.floor(seconds / 60)
|
||||
if (minutes < 70) {
|
||||
return `${minutes} min`
|
||||
}
|
||||
var hours = Math.floor(minutes / 60)
|
||||
minutes -= hours * 60
|
||||
if (!minutes) {
|
||||
return `${hours} hr`
|
||||
}
|
||||
return `${hours} hr ${minutes} min`
|
||||
}
|
||||
module.exports.elapsedPretty = elapsedPretty
|
||||
|
||||
function secondsToTimestamp(seconds, includeMs = false) {
|
||||
var _seconds = seconds
|
||||
var _minutes = Math.floor(seconds / 60)
|
||||
_seconds -= _minutes * 60
|
||||
var _hours = Math.floor(_minutes / 60)
|
||||
_minutes -= _hours * 60
|
||||
|
||||
var ms = _seconds - Math.floor(seconds)
|
||||
_seconds = Math.floor(_seconds)
|
||||
|
||||
var msString = '.' + (includeMs ? ms.toFixed(3) : '0.0').split('.')[1]
|
||||
if (!_hours) {
|
||||
return `${_minutes}:${_seconds.toString().padStart(2, '0')}${msString}`
|
||||
}
|
||||
return `${_hours}:${_minutes.toString().padStart(2, '0')}:${_seconds.toString().padStart(2, '0')}${msString}`
|
||||
}
|
||||
module.exports.secondsToTimestamp = secondsToTimestamp
|
||||
|
||||
module.exports.msToTimestamp = (ms, includeMs) => secondsToTimestamp(ms / 1000, includeMs)
|
||||
Loading…
Reference in New Issue
Block a user