Update:Audiobook merge to set metadata with tone and replace m4b in library item #594

This commit is contained in:
advplyr 2022-09-26 18:07:31 -05:00
parent b7bdaac163
commit f36a5eae6d
6 changed files with 187 additions and 163 deletions

View File

@ -16,8 +16,8 @@
<ui-btn v-if="abmergeStatus !== $constants.DownloadStatus.READY" :loading="abmergeStatus === $constants.DownloadStatus.PENDING" :disabled="tempDisable" @click="startAudiobookMerge">Start Merge</ui-btn>
<div v-else>
<div class="flex">
<ui-btn @click="downloadWithProgress(abmergeDownload)">Download</ui-btn>
<ui-icon-btn small icon="delete" bg-color="error" class="ml-2" @click="removeDownload" />
<!-- <ui-btn @click="downloadWithProgress(abmergeDownload)">Download</ui-btn>
<ui-icon-btn small icon="delete" bg-color="error" class="ml-2" @click="removeDownload" /> -->
</div>
<p class="px-0.5 py-1 text-sm font-mono text-center">Size: {{ $bytesPretty(abmergeDownload.size) }}</p>
</div>
@ -34,18 +34,7 @@
</div>
<div class="flex-grow" />
<div>
<p v-if="abmergeStatus === $constants.DownloadStatus.FAILED" class="text-error mb-2">Download Failed</p>
<p v-if="abmergeStatus === $constants.DownloadStatus.READY" class="text-success mb-2">Download Ready!</p>
<p v-if="abmergeStatus === $constants.DownloadStatus.EXPIRED" class="text-error mb-2">Download Expired</p>
<ui-btn v-if="abmergeStatus !== $constants.DownloadStatus.READY" :loading="abmergeStatus === $constants.DownloadStatus.PENDING" :disabled="true" @click="startAudiobookMerge">Not yet implemented</ui-btn>
<div v-else>
<div class="flex">
<ui-btn @click="downloadWithProgress(abmergeDownload)">Download</ui-btn>
<ui-icon-btn small icon="delete" bg-color="error" class="ml-2" @click="removeDownload" />
</div>
<p class="px-0.5 py-1 text-sm font-mono text-center">Size: {{ $bytesPretty(abmergeDownload.size) }}</p>
</div>
<ui-btn :loading="abmergeStatus === $constants.DownloadStatus.PENDING" :disabled="true" @click="startAudiobookMerge">Not yet implemented</ui-btn>
</div>
</div>
</div>
@ -164,7 +153,7 @@ export default {
this.tempDisable = true
this.$axios
.$delete(`/api/download/${downloadId}`)
.$delete(`/api/ab-manager-tasks/${downloadId}`)
.then(() => {
this.tempDisable = false
this.$toast.success('Merge download deleted')
@ -192,7 +181,7 @@ export default {
},
downloadWithProgress(download) {
var downloadId = download.id
var downloadUrl = `${process.env.serverUrl}/api/download/${downloadId}`
var downloadUrl = `${process.env.serverUrl}/api/ab-manager-tasks/${downloadId}`
var filename = download.filename
this.isDownloading = true
@ -242,18 +231,18 @@ export default {
},
loadDownloads() {
this.$axios
.$get(`/api/downloads`)
.$get(`/api/ab-manager-tasks`)
.then((data) => {
var pendingDownloads = data.pendingDownloads.map((pd) => {
var pendingTasks = data.pendingTasks.map((pd) => {
pd.download.status = this.$constants.DownloadStatus.PENDING
return pd.download
})
var downloads = data.downloads.map((d) => {
var tasks = data.tasks.map((d) => {
d.status = this.$constants.DownloadStatus.READY
return d
})
var allDownloads = downloads.concat(pendingDownloads)
this.$store.commit('downloads/setDownloads', allDownloads)
var allTasks = tasks.concat(pendingTasks)
this.$store.commit('downloads/setDownloads', allTasks)
})
.catch((error) => {
console.error('Failed to load downloads', error)

View File

@ -110,53 +110,53 @@ class MiscController {
res.sendStatus(200)
}
// GET: api/download/:id
async getDownload(req, res) {
// GET: api/ab-manager-tasks/:id
async getAbManagerTask(req, res) {
if (!req.user.canDownload) {
Logger.error('User attempting to download without permission', req.user)
return res.sendStatus(403)
}
var downloadId = req.params.id
Logger.info('Download Request', downloadId)
var download = this.abMergeManager.getDownload(downloadId)
if (!download) {
Logger.error('Download request not found', downloadId)
var taskId = req.params.id
Logger.info('Download Request', taskId)
var task = this.abMergeManager.getTask(taskId)
if (!task) {
Logger.error('Ab manager task request not found', taskId)
return res.sendStatus(404)
}
var options = {
headers: {
'Content-Type': download.mimeType
'Content-Type': task.mimeType
}
}
res.download(download.path, download.filename, options, (err) => {
res.download(task.path, task.filename, options, (err) => {
if (err) {
Logger.error('Download Error', err)
}
})
}
// DELETE: api/download/:id
async removeDownload(req, res) {
// DELETE: api/ab-manager-tasks/:id
async removeAbManagerTask(req, res) {
if (!req.user.canDownload || !req.user.canDelete) {
Logger.error('User attempting to remove download without permission', req.user.username)
Logger.error('User attempting to remove ab manager task without permission', req.user.username)
return res.sendStatus(403)
}
this.abMergeManager.removeDownloadById(req.params.id)
this.abMergeManager.removeTaskById(req.params.id)
res.sendStatus(200)
}
// GET: api/downloads
async getDownloads(req, res) {
// GET: api/ab-manager-tasks
async getAbManagerTasks(req, res) {
if (!req.user.canDownload) {
Logger.error('User attempting to get downloads without permission', req.user.username)
Logger.error('User attempting to get ab manager tasks without permission', req.user.username)
return res.sendStatus(403)
}
var downloads = {
downloads: this.abMergeManager.downloads,
pendingDownloads: this.abMergeManager.pendingDownloads
var taskData = {
tasks: this.abMergeManager.tasks,
pendingTasks: this.abMergeManager.pendingTasks
}
res.json(downloads)
res.json(taskData)
}
// PATCH: api/settings (admin)

View File

@ -4,10 +4,11 @@ const fs = require('../libs/fsExtra')
const workerThreads = require('worker_threads')
const Logger = require('../Logger')
const Download = require('../objects/Download')
const AbManagerTask = require('../objects/AbManagerTask')
const filePerms = require('../utils/filePerms')
const { getId } = require('../utils/index')
const { writeConcatFile, writeMetadataFile } = require('../utils/ffmpegHelpers')
const { writeConcatFile } = require('../utils/ffmpegHelpers')
const toneHelpers = require('../utils/toneHelpers')
const { getFileSize } = require('../utils/fileUtils')
class AbMergeManager {
@ -18,8 +19,8 @@ class AbMergeManager {
this.downloadDirPath = Path.join(global.MetadataPath, 'downloads')
this.downloadDirPathExist = false
this.pendingDownloads = []
this.downloads = []
this.pendingTasks = []
this.tasks = []
}
async ensureDownloadDirPath() { // Creates download path if necessary and sets owner and permissions
@ -38,14 +39,14 @@ class AbMergeManager {
this.downloadDirPathExist = true
}
getDownload(downloadId) {
return this.downloads.find(d => d.id === downloadId)
getTask(taskId) {
return this.tasks.find(d => d.id === taskId)
}
removeDownloadById(downloadId) {
var download = this.getDownload(downloadId)
if (download) {
this.removeDownload(download)
removeTaskById(taskId) {
var task = this.getTask(taskId)
if (task) {
this.removeTask(task)
}
}
@ -68,46 +69,46 @@ class AbMergeManager {
}
async startAudiobookMerge(user, libraryItem) {
var downloadId = getId('abmerge')
var dlpath = Path.join(this.downloadDirPath, downloadId)
Logger.info(`Start audiobook merge for ${libraryItem.id} - DownloadId: ${downloadId} - ${dlpath}`)
var taskId = getId('abmerge')
var dlpath = Path.join(this.downloadDirPath, taskId)
Logger.info(`Start audiobook merge for ${libraryItem.id} - TaskId: ${taskId} - ${dlpath}`)
var audiobookDirname = Path.basename(libraryItem.path)
var filename = audiobookDirname + '.m4b'
var downloadData = {
id: downloadId,
var taskData = {
id: taskId,
libraryItemId: libraryItem.id,
type: 'abmerge',
dirpath: dlpath,
path: Path.join(dlpath, filename),
filename,
ext: '.m4b',
userId: user.id
userId: user.id,
libraryItemPath: libraryItem.path,
originalTrackPaths: libraryItem.media.tracks.map(t => t.metadata.path)
}
var download = new Download()
download.setData(downloadData)
download.setTimeoutTimer(this.downloadTimedOut.bind(this))
var abManagerTask = new AbManagerTask()
abManagerTask.setData(taskData)
abManagerTask.setTimeoutTimer(this.downloadTimedOut.bind(this))
try {
await fs.mkdir(download.dirpath)
await fs.mkdir(abManagerTask.dirpath)
} catch (error) {
Logger.error(`[AbMergeManager] Failed to make directory ${download.dirpath}`)
Logger.error(`[AbMergeManager] Failed to make directory ${abManagerTask.dirpath}`)
Logger.debug(`[AbMergeManager] Make directory error: ${error}`)
var downloadJson = download.toJSON()
this.clientEmitter(user.id, 'abmerge_failed', downloadJson)
var taskJson = abManagerTask.toJSON()
this.clientEmitter(user.id, 'abmerge_failed', taskJson)
return
}
this.clientEmitter(user.id, 'abmerge_started', download.toJSON())
this.runAudiobookMerge(libraryItem, download)
this.clientEmitter(user.id, 'abmerge_started', abManagerTask.toJSON())
this.runAudiobookMerge(libraryItem, abManagerTask)
}
async runAudiobookMerge(libraryItem, download) {
async runAudiobookMerge(libraryItem, abManagerTask) {
// If changing audio file type then encoding is needed
var audioTracks = libraryItem.media.tracks
var audioRequiresEncode = audioTracks[0].metadata.ext !== download.ext
var audioRequiresEncode = audioTracks[0].metadata.ext !== abManagerTask.ext
var shouldIncludeCover = libraryItem.media.coverPath
var firstTrackIsM4b = audioTracks[0].metadata.ext.toLowerCase() === '.m4b'
var isOneTrack = audioTracks.length === 1
@ -115,7 +116,7 @@ class AbMergeManager {
const ffmpegInputs = []
if (!isOneTrack) {
var concatFilePath = Path.join(download.dirpath, 'files.txt')
var concatFilePath = Path.join(abManagerTask.dirpath, 'files.txt')
console.log('Write files.txt', concatFilePath)
await writeConcatFile(audioTracks, concatFilePath)
ffmpegInputs.push({
@ -138,8 +139,8 @@ class AbMergeManager {
'-map 0:a',
'-acodec aac',
'-ac 2',
'-b:a 64k',
'-movflags use_metadata_tags'
'-b:a 64k'
// '-movflags use_metadata_tags'
])
} else {
ffmpegOptions.push('-max_muxing_queue_size 1000')
@ -150,34 +151,33 @@ class AbMergeManager {
ffmpegOptions.push('-c:a copy')
}
}
if (download.ext === '.m4b') {
if (abManagerTask.ext === '.m4b') {
ffmpegOutputOptions.push('-f mp4')
}
// Create ffmetadata file
var metadataFilePath = Path.join(download.dirpath, 'metadata.txt')
await writeMetadataFile(libraryItem, metadataFilePath)
ffmpegInputs.push({
input: metadataFilePath
})
ffmpegOptions.push('-map_metadata 1')
// Embed cover art
if (shouldIncludeCover) {
var coverPath = libraryItem.media.coverPath.replace(/\\/g, '/')
ffmpegInputs.push({
input: coverPath,
options: ['-f image2pipe']
})
ffmpegOptions.push('-c:v copy')
ffmpegOptions.push('-map 2:v')
var chaptersFilePath = null
if (libraryItem.media.chapters.length) {
chaptersFilePath = Path.join(abManagerTask.dirpath, 'chapters.txt')
try {
await toneHelpers.writeToneChaptersFile(libraryItem.media.chapters, chaptersFilePath)
} catch (error) {
Logger.error(`[AbMergeManager] Write chapters.txt failed`, error)
chaptersFilePath = null
}
}
const toneMetadataObject = toneHelpers.getToneMetadataObject(libraryItem, chaptersFilePath)
toneMetadataObject.TrackNumber = 1
abManagerTask.toneMetadataObject = toneMetadataObject
Logger.debug(`[AbMergeManager] Book "${libraryItem.media.metadata.title}" tone metadata object=`, toneMetadataObject)
var workerData = {
inputs: ffmpegInputs,
options: ffmpegOptions,
outputOptions: ffmpegOutputOptions,
output: download.path,
output: abManagerTask.path,
}
var worker = null
@ -186,19 +186,19 @@ class AbMergeManager {
worker = new workerThreads.Worker(workerPath, { workerData })
} catch (error) {
Logger.error(`[AbMergeManager] Start worker thread failed`, error)
if (download.userId) {
var downloadJson = download.toJSON()
this.clientEmitter(download.userId, 'abmerge_failed', downloadJson)
if (abManagerTask.userId) {
var taskJson = abManagerTask.toJSON()
this.clientEmitter(abManagerTask.userId, 'abmerge_failed', taskJson)
}
this.removeDownload(download)
this.removeTask(abManagerTask)
return
}
worker.on('message', (message) => {
if (message != null && typeof message === 'object') {
if (message.type === 'RESULT') {
if (!download.isTimedOut) {
this.sendResult(download, message)
if (!abManagerTask.isTimedOut) {
this.sendResult(abManagerTask, message)
}
} else if (message.type === 'FFMPEG') {
if (Logger[message.level]) {
@ -209,78 +209,114 @@ class AbMergeManager {
Logger.error('Invalid worker message', message)
}
})
this.pendingDownloads.push({
id: download.id,
download,
this.pendingTasks.push({
id: abManagerTask.id,
abManagerTask,
worker
})
}
async sendResult(download, result) {
download.clearTimeoutTimer()
async sendResult(abManagerTask, result) {
abManagerTask.clearTimeoutTimer()
// Remove pending download
this.pendingDownloads = this.pendingDownloads.filter(d => d.id !== download.id)
// Remove pending task
this.pendingTasks = this.pendingTasks.filter(d => d.id !== abManagerTask.id)
if (result.isKilled) {
if (download.userId) {
this.clientEmitter(download.userId, 'abmerge_killed', download.toJSON())
if (abManagerTask.userId) {
this.clientEmitter(abManagerTask.userId, 'abmerge_killed', abManagerTask.toJSON())
}
return
}
if (!result.success) {
if (download.userId) {
this.clientEmitter(download.userId, 'abmerge_failed', download.toJSON())
if (abManagerTask.userId) {
this.clientEmitter(abManagerTask.userId, 'abmerge_failed', abManagerTask.toJSON())
}
this.removeDownload(download)
this.removeTask(abManagerTask)
return
}
// Write metadata to merged file
const success = await toneHelpers.tagAudioFile(abManagerTask.path, abManagerTask.toneMetadataObject)
if (!success) {
Logger.error(`[AbMergeManager] Failed to write metadata to file "${abManagerTask.path}"`)
if (abManagerTask.userId) {
this.clientEmitter(abManagerTask.userId, 'abmerge_failed', abManagerTask.toJSON())
}
this.removeTask(abManagerTask)
return
}
// Move library item tracks to cache
const itemCacheDir = Path.join(global.MetadataPath, `cache/items/${abManagerTask.libraryItemId}`)
await fs.ensureDir(itemCacheDir)
for (const trackPath of abManagerTask.originalTrackPaths) {
const trackFilename = Path.basename(trackPath)
const moveToPath = Path.join(itemCacheDir, trackFilename)
Logger.debug(`[AbMergeManager] Backing up original track "${trackPath}" to ${moveToPath}`)
await fs.move(trackPath, moveToPath, { overwrite: true }).catch((err) => {
Logger.error(`[AbMergeManager] Failed to move track "${trackPath}" to "${moveToPath}"`, err)
})
}
// Set file permissions and ownership
await filePerms.setDefault(download.path)
await filePerms.setDefault(abManagerTask.path)
await filePerms.setDefault(itemCacheDir)
var filesize = await getFileSize(download.path)
download.setComplete(filesize)
if (download.userId) {
this.clientEmitter(download.userId, 'abmerge_ready', download.toJSON())
// Move merged file to library item
const moveToPath = Path.join(abManagerTask.libraryItemPath, abManagerTask.filename)
Logger.debug(`[AbMergeManager] Moving merged audiobook to library item at "${moveToPath}"`)
const moveSuccess = await fs.move(abManagerTask.path, moveToPath, { overwrite: true }).then(() => true).catch((err) => {
Logger.error(`[AbMergeManager] Failed to move merged audiobook from "${abManagerTask.path}" to "${moveToPath}"`, err)
return false
})
if (!moveSuccess) {
// TODO: Revert cached og files?
}
download.setExpirationTimer(this.downloadExpired.bind(this))
this.downloads.push(download)
Logger.info(`[AbMergeManager] Download Ready ${download.id}`)
var filesize = await getFileSize(abManagerTask.path)
abManagerTask.setComplete(filesize)
if (abManagerTask.userId) {
this.clientEmitter(abManagerTask.userId, 'abmerge_ready', abManagerTask.toJSON())
}
// abManagerTask.setExpirationTimer(this.downloadExpired.bind(this))
// this.tasks.push(abManagerTask)
await this.removeTask(abManagerTask)
Logger.info(`[AbMergeManager] Ab task finished ${abManagerTask.id}`)
}
async downloadExpired(download) {
Logger.info(`[AbMergeManager] Download ${download.id} expired`)
// async downloadExpired(abManagerTask) {
// Logger.info(`[AbMergeManager] Download ${abManagerTask.id} expired`)
if (download.userId) {
this.clientEmitter(download.userId, 'abmerge_expired', download.toJSON())
// if (abManagerTask.userId) {
// this.clientEmitter(abManagerTask.userId, 'abmerge_expired', abManagerTask.toJSON())
// }
// this.removeTask(abManagerTask)
// }
async downloadTimedOut(abManagerTask) {
Logger.info(`[AbMergeManager] Download ${abManagerTask.id} timed out (${abManagerTask.timeoutTimeMs}ms)`)
if (abManagerTask.userId) {
var taskJson = abManagerTask.toJSON()
taskJson.isTimedOut = true
this.clientEmitter(abManagerTask.userId, 'abmerge_failed', taskJson)
}
this.removeDownload(download)
this.removeTask(abManagerTask)
}
async downloadTimedOut(download) {
Logger.info(`[AbMergeManager] Download ${download.id} timed out (${download.timeoutTimeMs}ms)`)
async removeTask(abManagerTask) {
Logger.info('[AbMergeManager] Removing task ' + abManagerTask.id)
if (download.userId) {
var downloadJson = download.toJSON()
downloadJson.isTimedOut = true
this.clientEmitter(download.userId, 'abmerge_failed', downloadJson)
}
this.removeDownload(download)
}
abManagerTask.clearTimeoutTimer()
// abManagerTask.clearExpirationTimer()
async removeDownload(download) {
Logger.info('[AbMergeManager] Removing download ' + download.id)
download.clearTimeoutTimer()
download.clearExpirationTimer()
var pendingDl = this.pendingDownloads.find(d => d.id === download.id)
var pendingDl = this.pendingTasks.find(d => d.id === abManagerTask.id)
if (pendingDl) {
this.pendingDownloads = this.pendingDownloads.filter(d => d.id !== download.id)
this.pendingTasks = this.pendingTasks.filter(d => d.id !== abManagerTask.id)
Logger.warn(`[AbMergeManager] Removing download in progress - stopping worker`)
if (pendingDl.worker) {
try {
@ -291,12 +327,12 @@ class AbMergeManager {
}
}
await fs.remove(download.dirpath).then(() => {
Logger.info('[AbMergeManager] Deleted download', download.dirpath)
await fs.remove(abManagerTask.dirpath).then(() => {
Logger.info('[AbMergeManager] Deleted download', abManagerTask.dirpath)
}).catch((err) => {
Logger.error('[AbMergeManager] Failed to delete download', err)
})
this.downloads = this.downloads.filter(d => d.id !== download.id)
this.tasks = this.tasks.filter(d => d.id !== abManagerTask.id)
}
}
module.exports = AbMergeManager

View File

@ -43,10 +43,7 @@ class AudioMetadataMangaer {
// Write chapters file
var chaptersFilePath = null
var cachePath = Path.join(global.MetadataPath, 'cache/items')
console.log('Items Cache Path', cachePath)
var itemCacheDir = Path.join(cachePath, libraryItem.id)
const itemCacheDir = Path.join(global.MetadataPath, `cache/items/${libraryItem.id}`)
await fs.ensureDir(itemCacheDir)
if (libraryItem.media.chapters.length) {

View File

@ -3,10 +3,11 @@ const { getAudioMimeTypeFromExtname } = require('../utils/fileUtils')
const DEFAULT_EXPIRATION = 1000 * 60 * 60 // 60 minutes
const DEFAULT_TIMEOUT = 1000 * 60 * 30 // 30 minutes
class Download {
constructor(download) {
class AbManagerTask {
constructor() {
this.id = null
this.libraryItemId = null
this.libraryItemPath = null
this.type = null
this.dirpath = null
@ -14,6 +15,8 @@ class Download {
this.ext = null
this.filename = null
this.size = 0
this.toneMetadataObject = null
this.originalTrackPaths = []
this.userId = null
this.isReady = false
@ -28,10 +31,6 @@ class Download {
this.timeoutTimer = null
this.expirationTimer = null
if (download) {
this.construct(download)
}
}
get mimeType() {
@ -52,13 +51,21 @@ class Download {
isReady: this.isReady,
startedAt: this.startedAt,
finishedAt: this.finishedAt,
expirationSeconds: this.expirationSeconds
expirationSeconds: this.expirationSeconds,
toneMetadataObject: this.toneMetadataObject
}
}
setData(downloadData) {
downloadData.startedAt = Date.now()
downloadData.isProcessing = true
this.construct(downloadData)
}
construct(download) {
this.id = download.id
this.libraryItemId = download.libraryItemId
this.libraryItemPath = download.libraryItemPath
this.type = download.type
this.dirpath = download.dirpath
@ -66,6 +73,7 @@ class Download {
this.ext = download.ext
this.filename = download.filename
this.size = download.size || 0
this.originalTrackPaths = download.originalTrackPaths
this.userId = download.userId
this.isReady = !!download.isReady
@ -79,12 +87,6 @@ class Download {
this.expiresAt = download.expiresAt || null
}
setData(downloadData) {
downloadData.startedAt = Date.now()
downloadData.isProcessing = true
this.construct(downloadData)
}
setComplete(fileSize) {
this.finishedAt = Date.now()
this.size = fileSize
@ -117,4 +119,4 @@ class Download {
clearTimeout(this.expirationTimer)
}
}
module.exports = Download
module.exports = AbManagerTask

View File

@ -221,9 +221,9 @@ class ApiRouter {
//
this.router.post('/upload', MiscController.handleUpload.bind(this))
this.router.get('/audiobook-merge/:id', MiscController.mergeAudiobook.bind(this))
this.router.get('/download/:id', MiscController.getDownload.bind(this))
this.router.delete('/download/:id', MiscController.removeDownload.bind(this))
this.router.get('/downloads', MiscController.getDownloads.bind(this))
this.router.get('/ab-manager-tasks/:id', MiscController.getAbManagerTask.bind(this))
this.router.delete('/ab-manager-tasks/:id', MiscController.removeAbManagerTask.bind(this))
this.router.get('/ab-manager-tasks', MiscController.getAbManagerTasks.bind(this))
this.router.patch('/settings', MiscController.updateServerSettings.bind(this)) // Root only
this.router.post('/purgecache', MiscController.purgeCache.bind(this)) // Root only
this.router.post('/authorize', MiscController.authorize.bind(this))