extern/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2024-12-30 18:48:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #716 from jmt-gh/abs_overdrive

Browse Source 
Add support for leveraging chapter data directly from Overdrive mp3s during scanning
This commit is contained in:
advplyr 2022-06-13 18:25:44 -05:00 committed by GitHub
commit 8894f52439
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 203 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -11,6 +11,7 @@ test/
/client/.nuxt/
/client/dist/
/dist/
library/
sw.*
.DS_STORE
.DS_STORE

13
client/pages/config/index.vue
View File

@ -103,6 +103,16 @@
</ui-tooltip>
</div>
<div class="flex items-center py-2">
<ui-toggle-switch v-model="newServerSettings.scannerPreferOverdriveMediaMarker" :disabled="updatingServerSettings" @input="(val) => updateSettingsKey('scannerPreferOverdriveMediaMarker', val)" />
<ui-tooltip :text="tooltips.scannerPreferOverdriveMediaMarker">
<p class="pl-4 text-lg">
Scanner prefer Overdrive Media Markers for chapters
<span class="material-icons icon-text">info_outlined</span>
</p>
</ui-tooltip>
</div>
<div class="flex items-center py-2">
<ui-toggle-switch v-model="newServerSettings.scannerPreferOpfMetadata" :disabled="updatingServerSettings" @input="(val) => updateSettingsKey('scannerPreferOpfMetadata', val)" />
<ui-tooltip :text="tooltips.scannerPreferOpfMetadata">
@ -245,7 +255,8 @@ export default {
storeCoverWithItem: 'By default covers are stored in /metadata/items, enabling this setting will store covers in your library item folder. Only one file named "cover" will be kept',
storeMetadataWithItem: 'By default metadata files are stored in /metadata/items, enabling this setting will store metadata files in your library item folders. Uses .abs file extension',
coverAspectRatio: 'Prefer to use square covers over standard 1.6:1 book covers',
enableEReader: 'E-reader is still a work in progress, but use this setting to open it up to all your users (or use the "Experimental Features" toggle below just for you)'
enableEReader: 'E-reader is still a work in progress, but use this setting to open it up to all your users (or use the "Experimental Features" toggle below just for you)',
scannerPreferOverdriveMediaMarker: 'MP3 files from Overdrive come with chapter timings embedded as custom metadata. Enabling this will use these tags for chapter timings automatically'
},
showConfirmPurgeCache: false
}

2
server/controllers/PodcastController.js
View File

@ -103,7 +103,7 @@ class PodcastController {
Logger.error('Invalid podcast feed request response')
return res.status(500).send('Bad response from feed request')
}
Logger.debug(`[PdocastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`)
Logger.debug(`[PodcastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`)
var payload = await parsePodcastRssFeedXml(data.data, false, includeRaw)
if (!payload) {
return res.status(500).send('Invalid podcast RSS feed')

15
server/objects/mediaTypes/Book.js
View File

@ -3,6 +3,7 @@ const Logger = require('../../Logger')
const BookMetadata = require('../metadata/BookMetadata')
const { areEquivalent, copyValue } = require('../../utils/index')
const { parseOpfMetadataXML } = require('../../utils/parsers/parseOpfMetadata')
const { overdriveMediaMarkersExist, parseOverdriveMediaMarkersAsChapters } = require('../../utils/parsers/parseOverdriveMediaMarkers')
const abmetadataGenerator = require('../../utils/abmetadataGenerator')
const { readTextFile } = require('../../utils/fileUtils')
const AudioFile = require('../files/AudioFile')
@ -360,10 +361,11 @@ class Book {
this.rebuildTracks()
}
rebuildTracks() {
rebuildTracks(preferOverdriveMediaMarker) {
Logger.debug(`[Book] Tracks being rebuilt...!`)
this.audioFiles.sort((a, b) => a.index - b.index)
this.missingParts = []
this.setChapters()
this.setChapters(preferOverdriveMediaMarker)
this.checkUpdateMissingTracks()
}
@ -395,9 +397,16 @@ class Book {
return wasUpdated
}
setChapters() {
setChapters(preferOverdriveMediaMarker = false) {
// If 1 audio file without chapters, then no chapters will be set
var includedAudioFiles = this.audioFiles.filter(af => !af.exclude)
// If overdrive media markers are present and preferred, use those instead
if (preferOverdriveMediaMarker && overdriveMediaMarkersExist(includedAudioFiles)) {
Logger.info('[Book] Overdrive Media Markers and preference found! Using these for chapter definitions')
return this.chapters = parseOverdriveMediaMarkersAsChapters(includedAudioFiles)
}
if (includedAudioFiles.length === 1) {
// 1 audio file with chapters
if (includedAudioFiles[0].chapters) {

6
server/objects/metadata/AudioMetaTags.js
View File

@ -20,6 +20,7 @@ class AudioMetaTags {
this.tagIsbn = null
this.tagLanguage = null
this.tagASIN = null
this.tagOverdriveMediaMarker = null
if (metadata) {
this.construct(metadata)
@ -58,6 +59,7 @@ class AudioMetaTags {
this.tagIsbn = metadata.tagIsbn || null
this.tagLanguage = metadata.tagLanguage || null
this.tagASIN = metadata.tagASIN || null
this.tagOverdriveMediaMarker = metadata.tagOverdriveMediaMarker || null
}
// Data parsed in prober.js
@ -82,6 +84,7 @@ class AudioMetaTags {
this.tagIsbn = payload.file_tag_isbn || null
this.tagLanguage = payload.file_tag_language || null
this.tagASIN = payload.file_tag_asin || null
this.tagOverdriveMediaMarker = payload.file_tag_overdrive_media_marker || null
}
updateData(payload) {
@ -105,7 +108,8 @@ class AudioMetaTags {
tagEncodedBy: payload.file_tag_encodedby || null,
tagIsbn: payload.file_tag_isbn || null,
tagLanguage: payload.file_tag_language || null,
tagASIN: payload.file_tag_asin || null
tagASIN: payload.file_tag_asin || null,
tagOverdriveMediaMarker: payload.file_tag_overdrive_media_marker || null,
}
var hasUpdates = false

4
server/objects/metadata/BookMetadata.js
View File

@ -262,6 +262,10 @@ class BookMetadata {
{
tag: 'tagASIN',
key: 'asin'
},
{
tag: 'tagOverdriveMediaMarker',
key: 'overdriveMediaMarker'
}
]

3
server/objects/settings/ServerSettings.js
View File

@ -13,6 +13,7 @@ class ServerSettings {
this.scannerPreferOpfMetadata = false
this.scannerPreferMatchedMetadata = false
this.scannerDisableWatcher = false
this.scannerPreferOverdriveMediaMarker = false
// Metadata - choose to store inside users library item folder
this.storeCoverWithItem = false
@ -65,6 +66,7 @@ class ServerSettings {
this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata
this.scannerPreferMatchedMetadata = !!settings.scannerPreferMatchedMetadata
this.scannerDisableWatcher = !!settings.scannerDisableWatcher
this.scannerPreferOverdriveMediaMarker = !!settings.scannerPreferOverdriveMediaMarker
this.storeCoverWithItem = !!settings.storeCoverWithItem
if (settings.storeCoverWithBook != undefined) { // storeCoverWithBook was old name of setting < v2
@ -111,6 +113,7 @@ class ServerSettings {
scannerPreferOpfMetadata: this.scannerPreferOpfMetadata,
scannerPreferMatchedMetadata: this.scannerPreferMatchedMetadata,
scannerDisableWatcher: this.scannerDisableWatcher,
scannerPreferOverdriveMediaMarker: this.scannerPreferOverdriveMediaMarker,
storeCoverWithItem: this.storeCoverWithItem,
storeMetadataWithItem: this.storeMetadataWithItem,
rateLimitLoginRequests: this.rateLimitLoginRequests,

1
server/scanner/LibraryScan.js
View File

@ -34,6 +34,7 @@ class LibraryScan {
get forceRescan() { return !!this._scanOptions.forceRescan }
get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata }
get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata }
get preferOverdriveMediaMarker() { return !!this._scanOptions.preferOverdriveMediaMarker }
get findCovers() { return !!this._scanOptions.findCovers }
get timestamp() {
return (new Date()).toISOString()

5
server/scanner/MediaFileScanner.js
View File

@ -195,7 +195,7 @@ class MediaFileScanner {
}
}
async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) {
async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan = null) {
var hasUpdated = false
var mediaScanResult = await this.executeMediaFileScans(libraryItem.mediaType, mediaLibraryFiles, scanData)
@ -208,6 +208,7 @@ class MediaFileScanner {
} else if (mediaScanResult.audioFiles.length) {
if (libraryScan) {
libraryScan.addLog(LogLevel.DEBUG, `Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`)
Logger.debug(`Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`)
}
var totalAudioFilesToInclude = mediaScanResult.audioFiles.length
@ -247,7 +248,7 @@ class MediaFileScanner {
}
if (hasUpdated) {
libraryItem.media.rebuildTracks()
libraryItem.media.rebuildTracks(preferOverdriveMediaMarker)
}
} else { // Podcast Media Type
var existingAudioFiles = mediaScanResult.audioFiles.filter(af => libraryItem.media.findFileWithInode(af.ino))

5
server/scanner/ScanOptions.js
View File

@ -9,6 +9,7 @@ class ScanOptions {
this.preferAudioMetadata = false
this.preferOpfMetadata = false
this.preferMatchedMetadata = false
this.preferOverdriveMediaMarker = false
if (options) {
this.construct(options)
@ -34,7 +35,8 @@ class ScanOptions {
storeCoverWithItem: this.storeCoverWithItem,
preferAudioMetadata: this.preferAudioMetadata,
preferOpfMetadata: this.preferOpfMetadata,
preferMatchedMetadata: this.preferMatchedMetadata
preferMatchedMetadata: this.preferMatchedMetadata,
preferOverdriveMediaMarker: this.preferOverdriveMediaMarker
}
}
@ -47,6 +49,7 @@ class ScanOptions {
this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata
this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata
this.scannerPreferMatchedMetadata = serverSettings.scannerPreferMatchedMetadata
this.preferOverdriveMediaMarker = serverSettings.scannerPreferOverdriveMediaMarker
}
}
module.exports = ScanOptions

14
server/scanner/Scanner.js
View File

@ -80,7 +80,7 @@ class Scanner {
// Scan all audio files
if (libraryItem.hasAudioFiles) {
var libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio')
if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItemData, libraryItem, this.db.serverSettings.scannerPreferAudioMetadata)) {
if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItemData, libraryItem, this.db.serverSettings.scannerPreferAudioMetadata, this.db.serverSettings.scannerPreferOverdriveMediaMarker)) {
hasUpdated = true
}
@ -310,7 +310,7 @@ class Scanner {
async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) {
var newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => {
return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan)
return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan.preferOverdriveMediaMarker, libraryScan)
}))
newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls
@ -337,7 +337,7 @@ class Scanner {
// forceRescan all existing audio files - will probe and update ID3 tag metadata
var existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio')
if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) {
if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan)) {
if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) {
hasUpdated = true
}
}
@ -345,7 +345,7 @@ class Scanner {
var newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio')
var removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio')
if (newAudioFiles.length || removedAudioFiles.length) {
if (await MediaFileScanner.scanMediaFiles(newAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan)) {
if (await MediaFileScanner.scanMediaFiles(newAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) {
hasUpdated = true
}
}
@ -379,7 +379,7 @@ class Scanner {
return hasUpdated ? libraryItem : null
}
async scanNewLibraryItem(libraryItemData, libraryMediaType, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) {
async scanNewLibraryItem(libraryItemData, libraryMediaType, preferAudioMetadata, preferOpfMetadata, findCovers, preferOverdriveMediaMarker, libraryScan = null) {
if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`)
else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`)
@ -388,7 +388,7 @@ class Scanner {
var mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video')
if (mediaFiles.length) {
await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItemData, libraryItem, preferAudioMetadata, libraryScan)
await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItemData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan)
}
await libraryItem.syncFiles(preferOpfMetadata)
@ -608,7 +608,7 @@ class Scanner {
var libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, fullPath, isSingleMediaItem, this.db.serverSettings)
if (!libraryItemData) return null
var serverSettings = this.db.serverSettings
return this.scanNewLibraryItem(libraryItemData, libraryMediaType, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers)
return this.scanNewLibraryItem(libraryItemData, libraryMediaType, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers, serverSettings.scannerPreferOverdriveMediaMarker)
}
async searchForCover(libraryItem, libraryScan = null) {

148
server/utils/parsers/parseOverdriveMediaMarkers.js Normal file
View File

@ -0,0 +1,148 @@
const Logger = require('../../Logger')
// given a list of audio files, extract all of the Overdrive Media Markers metaTags, and return an array of them as XML
function extractOverdriveMediaMarkers(includedAudioFiles) {
Logger.debug('[parseOverdriveMediaMarkers] Extracting overdrive media markers')
var markers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(notUndefined => notUndefined !== undefined).filter(elem => { return elem !== null }) || []
return markers
}
// given the array of Overdrive Media Markers from generateOverdriveMediaMarkers()
// parse and clean them in to something a bit more usable
function cleanOverdriveMediaMarkers(overdriveMediaMarkers) {
Logger.debug('[parseOverdriveMediaMarkers] Cleaning up overdrive media markers')
/*
returns an array of arrays of objects. Each inner array corresponds to an audio track, with it's objects being a chapter:
[
[
{
"Name": "Chapter 1",
"Time": "0:00.000"
},
{
"Name": "Chapter 2",
"Time": "15:51.000"
},
{ etc }
]
]
*/
var parseString = require('xml2js').parseString; // function to convert xml to JSON
var parsedOverdriveMediaMarkers = []
overdriveMediaMarkers.forEach(function (item, index) {
var parsed_result
parseString(item, function (err, result) {
/*
result.Markers.Marker is the result of parsing the XML for the MediaMarker tags for the MP3 file (Part##.mp3)
it is shaped like this, and needs further cleaning below:
[
{
"Name": [
"Chapter 1: "
],
"Time": [
"0:00.000"
]
},
{
ANOTHER CHAPTER
},
]
*/
// The values for Name and Time in results.Markers.Marker are returned as Arrays from parseString and should be strings
parsed_result = objectValuesArrayToString(result.Markers.Marker)
})
parsedOverdriveMediaMarkers.push(parsed_result)
})
return removeExtraChapters(parsedOverdriveMediaMarkers)
}
// given an array of objects, convert any values that are arrays to strings
function objectValuesArrayToString(arrayOfObjects) {
Logger.debug('[parseOverdriveMediaMarkers] Converting Marker object values from arrays to strings')
arrayOfObjects.forEach((item) => {
Object.keys(item).forEach(key => {
item[key] = item[key].toString()
})
})
return arrayOfObjects
}
// Overdrive sometimes has weird chapters and subchapters defined
// These aren't necessary, so lets remove them
function removeExtraChapters(parsedOverdriveMediaMarkers) {
Logger.debug('[parseOverdriveMediaMarkers] Removing any unnecessary chapters')
const weirdChapterFilterRegex = /([(]\d|[cC]ontinued)/
var cleaned = []
parsedOverdriveMediaMarkers.forEach(function (item) {
cleaned.push(item.filter(chapter => !weirdChapterFilterRegex.test(chapter.Name)))
})
return cleaned
}
// Given a set of chapters from generateParsedChapters, add the end time to each one
function addChapterEndTimes(chapters, totalAudioDuration) {
Logger.debug('[parseOverdriveMediaMarkers] Adding chapter end times')
chapters.forEach((chapter, chapter_index) => {
if (chapter_index < chapters.length - 1) {
chapter.end = chapters[chapter_index + 1].start
} else {
chapter.end = totalAudioDuration
}
})
return chapters
}
// The function that actually generates the Chapters object that we update ABS with
function generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers) {
Logger.debug('[parseOverdriveMediaMarkers] Generating new chapters for ABS')
// logic ported over from benonymity's OverdriveChapterizer:
// https://github.com/benonymity/OverdriveChapterizer/blob/main/chapters.py
var parsedChapters = []
var length = 0.0
var index = 0
var time = 0.0
// cleanedOverdriveMediaMarkers is an array of array of objects, where the inner array matches to the included audio files tracks
// this allows us to leverage the individual track durations when calculating the start times of chapters in tracks after the first (using length)
includedAudioFiles.forEach((track, track_index) => {
cleanedOverdriveMediaMarkers[track_index].forEach((chapter) => {
time = chapter.Time.split(":")
time = length + parseFloat(time[0]) * 60 + parseFloat(time[1])
var newChapterData = {
id: index++,
start: time,
title: chapter.Name
}
parsedChapters.push(newChapterData)
})
length += track.duration
})
parsedChapters = addChapterEndTimes(parsedChapters, length) // we need all the start times sorted out before we can add the end times
return parsedChapters
}
module.exports.overdriveMediaMarkersExist = (includedAudioFiles) => {
return extractOverdriveMediaMarkers(includedAudioFiles).length > 1
}
module.exports.parseOverdriveMediaMarkersAsChapters = (includedAudioFiles) => {
Logger.info('[parseOverdriveMediaMarkers] Parsing of Overdrive Media Markers started')
var overdriveMediaMarkers = extractOverdriveMediaMarkers(includedAudioFiles)
var cleanedOverdriveMediaMarkers = cleanOverdriveMediaMarkers(overdriveMediaMarkers)
var parsedChapters = generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers)
return parsedChapters
}

1
server/utils/prober.js
View File

@ -192,6 +192,7 @@ function parseTags(format, verbose) {
file_tag_movement: tryGrabTags(format, 'movement', 'mvin'),
file_tag_genre1: tryGrabTags(format, 'tmp_genre1', 'genre1'),
file_tag_genre2: tryGrabTags(format, 'tmp_genre2', 'genre2'),
file_tag_overdrive_media_marker: tryGrabTags(format, 'OverDrive MediaMarkers'),
}
for (const key in tags) {
if (!tags[key]) {