diff --git a/.gitignore b/.gitignore index 769b0639..53a60bef 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ test/ /client/.nuxt/ /client/dist/ /dist/ +library/ sw.* -.DS_STORE \ No newline at end of file +.DS_STORE diff --git a/client/pages/config/index.vue b/client/pages/config/index.vue index a037ac71..3dda0a60 100644 --- a/client/pages/config/index.vue +++ b/client/pages/config/index.vue @@ -103,6 +103,16 @@ +
+ + +

+ Scanner prefer Overdrive Media Markers for chapters + info_outlined +

+
+
+
@@ -245,7 +255,8 @@ export default { storeCoverWithItem: 'By default covers are stored in /metadata/items, enabling this setting will store covers in your library item folder. Only one file named "cover" will be kept', storeMetadataWithItem: 'By default metadata files are stored in /metadata/items, enabling this setting will store metadata files in your library item folders. Uses .abs file extension', coverAspectRatio: 'Prefer to use square covers over standard 1.6:1 book covers', - enableEReader: 'E-reader is still a work in progress, but use this setting to open it up to all your users (or use the "Experimental Features" toggle below just for you)' + enableEReader: 'E-reader is still a work in progress, but use this setting to open it up to all your users (or use the "Experimental Features" toggle below just for you)', + scannerPreferOverdriveMediaMarker: 'MP3 files from Overdrive come with chapter timings embedded as custom metadata. Enabling this will use these tags for chapter timings automatically' }, showConfirmPurgeCache: false } diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 81029c95..9d1aa2c8 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -103,7 +103,7 @@ class PodcastController { Logger.error('Invalid podcast feed request response') return res.status(500).send('Bad response from feed request') } - Logger.debug(`[PdocastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`) + Logger.debug(`[PodcastController] Podcast feed size ${(data.data.length / 1024 / 1024).toFixed(2)}MB`) var payload = await parsePodcastRssFeedXml(data.data, false, includeRaw) if (!payload) { return res.status(500).send('Invalid podcast RSS feed') diff --git a/server/objects/mediaTypes/Book.js b/server/objects/mediaTypes/Book.js index 6f0f4a1d..e84980d0 100644 --- a/server/objects/mediaTypes/Book.js +++ b/server/objects/mediaTypes/Book.js @@ -3,6 +3,7 @@ const Logger = require('../../Logger') const BookMetadata = require('../metadata/BookMetadata') const { areEquivalent, copyValue } = require('../../utils/index') const { parseOpfMetadataXML } = require('../../utils/parsers/parseOpfMetadata') +const { overdriveMediaMarkersExist, parseOverdriveMediaMarkersAsChapters } = require('../../utils/parsers/parseOverdriveMediaMarkers') const abmetadataGenerator = require('../../utils/abmetadataGenerator') const { readTextFile } = require('../../utils/fileUtils') const AudioFile = require('../files/AudioFile') @@ -360,10 +361,11 @@ class Book { this.rebuildTracks() } - rebuildTracks() { + rebuildTracks(preferOverdriveMediaMarker) { + Logger.debug(`[Book] Tracks being rebuilt...!`) this.audioFiles.sort((a, b) => a.index - b.index) this.missingParts = [] - this.setChapters() + this.setChapters(preferOverdriveMediaMarker) this.checkUpdateMissingTracks() } @@ -395,9 +397,16 @@ class Book { return wasUpdated } - setChapters() { + setChapters(preferOverdriveMediaMarker = false) { // If 1 audio file without chapters, then no chapters will be set var includedAudioFiles = this.audioFiles.filter(af => !af.exclude) + + // If overdrive media markers are present and preferred, use those instead + if (preferOverdriveMediaMarker && overdriveMediaMarkersExist(includedAudioFiles)) { + Logger.info('[Book] Overdrive Media Markers and preference found! Using these for chapter definitions') + return this.chapters = parseOverdriveMediaMarkersAsChapters(includedAudioFiles) + } + if (includedAudioFiles.length === 1) { // 1 audio file with chapters if (includedAudioFiles[0].chapters) { diff --git a/server/objects/metadata/AudioMetaTags.js b/server/objects/metadata/AudioMetaTags.js index 65ac0d9c..13bb2a83 100644 --- a/server/objects/metadata/AudioMetaTags.js +++ b/server/objects/metadata/AudioMetaTags.js @@ -20,6 +20,7 @@ class AudioMetaTags { this.tagIsbn = null this.tagLanguage = null this.tagASIN = null + this.tagOverdriveMediaMarker = null if (metadata) { this.construct(metadata) @@ -58,6 +59,7 @@ class AudioMetaTags { this.tagIsbn = metadata.tagIsbn || null this.tagLanguage = metadata.tagLanguage || null this.tagASIN = metadata.tagASIN || null + this.tagOverdriveMediaMarker = metadata.tagOverdriveMediaMarker || null } // Data parsed in prober.js @@ -82,6 +84,7 @@ class AudioMetaTags { this.tagIsbn = payload.file_tag_isbn || null this.tagLanguage = payload.file_tag_language || null this.tagASIN = payload.file_tag_asin || null + this.tagOverdriveMediaMarker = payload.file_tag_overdrive_media_marker || null } updateData(payload) { @@ -105,7 +108,8 @@ class AudioMetaTags { tagEncodedBy: payload.file_tag_encodedby || null, tagIsbn: payload.file_tag_isbn || null, tagLanguage: payload.file_tag_language || null, - tagASIN: payload.file_tag_asin || null + tagASIN: payload.file_tag_asin || null, + tagOverdriveMediaMarker: payload.file_tag_overdrive_media_marker || null, } var hasUpdates = false diff --git a/server/objects/metadata/BookMetadata.js b/server/objects/metadata/BookMetadata.js index 6bcf38bf..4cedd1cb 100644 --- a/server/objects/metadata/BookMetadata.js +++ b/server/objects/metadata/BookMetadata.js @@ -262,6 +262,10 @@ class BookMetadata { { tag: 'tagASIN', key: 'asin' + }, + { + tag: 'tagOverdriveMediaMarker', + key: 'overdriveMediaMarker' } ] diff --git a/server/objects/settings/ServerSettings.js b/server/objects/settings/ServerSettings.js index 286e15c0..00c56b2a 100644 --- a/server/objects/settings/ServerSettings.js +++ b/server/objects/settings/ServerSettings.js @@ -13,6 +13,7 @@ class ServerSettings { this.scannerPreferOpfMetadata = false this.scannerPreferMatchedMetadata = false this.scannerDisableWatcher = false + this.scannerPreferOverdriveMediaMarker = false // Metadata - choose to store inside users library item folder this.storeCoverWithItem = false @@ -65,6 +66,7 @@ class ServerSettings { this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata this.scannerPreferMatchedMetadata = !!settings.scannerPreferMatchedMetadata this.scannerDisableWatcher = !!settings.scannerDisableWatcher + this.scannerPreferOverdriveMediaMarker = !!settings.scannerPreferOverdriveMediaMarker this.storeCoverWithItem = !!settings.storeCoverWithItem if (settings.storeCoverWithBook != undefined) { // storeCoverWithBook was old name of setting < v2 @@ -111,6 +113,7 @@ class ServerSettings { scannerPreferOpfMetadata: this.scannerPreferOpfMetadata, scannerPreferMatchedMetadata: this.scannerPreferMatchedMetadata, scannerDisableWatcher: this.scannerDisableWatcher, + scannerPreferOverdriveMediaMarker: this.scannerPreferOverdriveMediaMarker, storeCoverWithItem: this.storeCoverWithItem, storeMetadataWithItem: this.storeMetadataWithItem, rateLimitLoginRequests: this.rateLimitLoginRequests, diff --git a/server/scanner/LibraryScan.js b/server/scanner/LibraryScan.js index e089e844..d6d50b27 100644 --- a/server/scanner/LibraryScan.js +++ b/server/scanner/LibraryScan.js @@ -34,6 +34,7 @@ class LibraryScan { get forceRescan() { return !!this._scanOptions.forceRescan } get preferAudioMetadata() { return !!this._scanOptions.preferAudioMetadata } get preferOpfMetadata() { return !!this._scanOptions.preferOpfMetadata } + get preferOverdriveMediaMarker() { return !!this._scanOptions.preferOverdriveMediaMarker } get findCovers() { return !!this._scanOptions.findCovers } get timestamp() { return (new Date()).toISOString() diff --git a/server/scanner/MediaFileScanner.js b/server/scanner/MediaFileScanner.js index bccbae26..3d43d85e 100644 --- a/server/scanner/MediaFileScanner.js +++ b/server/scanner/MediaFileScanner.js @@ -195,7 +195,7 @@ class MediaFileScanner { } } - async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) { + async scanMediaFiles(mediaLibraryFiles, scanData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan = null) { var hasUpdated = false var mediaScanResult = await this.executeMediaFileScans(libraryItem.mediaType, mediaLibraryFiles, scanData) @@ -208,6 +208,7 @@ class MediaFileScanner { } else if (mediaScanResult.audioFiles.length) { if (libraryScan) { libraryScan.addLog(LogLevel.DEBUG, `Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`) + Logger.debug(`Library Item "${scanData.path}" Audio file scan took ${mediaScanResult.elapsed}ms for ${mediaScanResult.audioFiles.length} with average time of ${mediaScanResult.averageScanDuration}ms`) } var totalAudioFilesToInclude = mediaScanResult.audioFiles.length @@ -247,7 +248,7 @@ class MediaFileScanner { } if (hasUpdated) { - libraryItem.media.rebuildTracks() + libraryItem.media.rebuildTracks(preferOverdriveMediaMarker) } } else { // Podcast Media Type var existingAudioFiles = mediaScanResult.audioFiles.filter(af => libraryItem.media.findFileWithInode(af.ino)) diff --git a/server/scanner/ScanOptions.js b/server/scanner/ScanOptions.js index 3d9d4556..968f0723 100644 --- a/server/scanner/ScanOptions.js +++ b/server/scanner/ScanOptions.js @@ -9,6 +9,7 @@ class ScanOptions { this.preferAudioMetadata = false this.preferOpfMetadata = false this.preferMatchedMetadata = false + this.preferOverdriveMediaMarker = false if (options) { this.construct(options) @@ -34,7 +35,8 @@ class ScanOptions { storeCoverWithItem: this.storeCoverWithItem, preferAudioMetadata: this.preferAudioMetadata, preferOpfMetadata: this.preferOpfMetadata, - preferMatchedMetadata: this.preferMatchedMetadata + preferMatchedMetadata: this.preferMatchedMetadata, + preferOverdriveMediaMarker: this.preferOverdriveMediaMarker } } @@ -47,6 +49,7 @@ class ScanOptions { this.preferAudioMetadata = serverSettings.scannerPreferAudioMetadata this.preferOpfMetadata = serverSettings.scannerPreferOpfMetadata this.scannerPreferMatchedMetadata = serverSettings.scannerPreferMatchedMetadata + this.preferOverdriveMediaMarker = serverSettings.scannerPreferOverdriveMediaMarker } } module.exports = ScanOptions \ No newline at end of file diff --git a/server/scanner/Scanner.js b/server/scanner/Scanner.js index 677585e1..79d11258 100644 --- a/server/scanner/Scanner.js +++ b/server/scanner/Scanner.js @@ -80,7 +80,7 @@ class Scanner { // Scan all audio files if (libraryItem.hasAudioFiles) { var libraryAudioFiles = libraryItem.libraryFiles.filter(lf => lf.fileType === 'audio') - if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItemData, libraryItem, this.db.serverSettings.scannerPreferAudioMetadata)) { + if (await MediaFileScanner.scanMediaFiles(libraryAudioFiles, libraryItemData, libraryItem, this.db.serverSettings.scannerPreferAudioMetadata, this.db.serverSettings.scannerPreferOverdriveMediaMarker)) { hasUpdated = true } @@ -310,7 +310,7 @@ class Scanner { async scanNewLibraryItemDataChunk(newLibraryItemsData, libraryScan) { var newLibraryItems = await Promise.all(newLibraryItemsData.map((lid) => { - return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan) + return this.scanNewLibraryItem(lid, libraryScan.libraryMediaType, libraryScan.preferAudioMetadata, libraryScan.preferOpfMetadata, libraryScan.findCovers, libraryScan.preferOverdriveMediaMarker, libraryScan) })) newLibraryItems = newLibraryItems.filter(li => li) // Filter out nulls @@ -337,7 +337,7 @@ class Scanner { // forceRescan all existing audio files - will probe and update ID3 tag metadata var existingAudioFiles = existingLibraryFiles.filter(lf => lf.fileType === 'audio') if (libraryScan.scanOptions.forceRescan && existingAudioFiles.length) { - if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan)) { + if (await MediaFileScanner.scanMediaFiles(existingAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) { hasUpdated = true } } @@ -345,7 +345,7 @@ class Scanner { var newAudioFiles = newLibraryFiles.filter(lf => lf.fileType === 'audio') var removedAudioFiles = filesRemoved.filter(lf => lf.fileType === 'audio') if (newAudioFiles.length || removedAudioFiles.length) { - if (await MediaFileScanner.scanMediaFiles(newAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan)) { + if (await MediaFileScanner.scanMediaFiles(newAudioFiles, scanData, libraryItem, libraryScan.preferAudioMetadata, libraryScan.preferOverdriveMediaMarker, libraryScan)) { hasUpdated = true } } @@ -379,7 +379,7 @@ class Scanner { return hasUpdated ? libraryItem : null } - async scanNewLibraryItem(libraryItemData, libraryMediaType, preferAudioMetadata, preferOpfMetadata, findCovers, libraryScan = null) { + async scanNewLibraryItem(libraryItemData, libraryMediaType, preferAudioMetadata, preferOpfMetadata, findCovers, preferOverdriveMediaMarker, libraryScan = null) { if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Scanning new library item "${libraryItemData.path}"`) else Logger.debug(`[Scanner] Scanning new item "${libraryItemData.path}"`) @@ -388,7 +388,7 @@ class Scanner { var mediaFiles = libraryItemData.libraryFiles.filter(lf => lf.fileType === 'audio' || lf.fileType === 'video') if (mediaFiles.length) { - await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItemData, libraryItem, preferAudioMetadata, libraryScan) + await MediaFileScanner.scanMediaFiles(mediaFiles, libraryItemData, libraryItem, preferAudioMetadata, preferOverdriveMediaMarker, libraryScan) } await libraryItem.syncFiles(preferOpfMetadata) @@ -608,7 +608,7 @@ class Scanner { var libraryItemData = await getLibraryItemFileData(libraryMediaType, folder, fullPath, isSingleMediaItem, this.db.serverSettings) if (!libraryItemData) return null var serverSettings = this.db.serverSettings - return this.scanNewLibraryItem(libraryItemData, libraryMediaType, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers) + return this.scanNewLibraryItem(libraryItemData, libraryMediaType, serverSettings.scannerPreferAudioMetadata, serverSettings.scannerPreferOpfMetadata, serverSettings.scannerFindCovers, serverSettings.scannerPreferOverdriveMediaMarker) } async searchForCover(libraryItem, libraryScan = null) { diff --git a/server/utils/parsers/parseOverdriveMediaMarkers.js b/server/utils/parsers/parseOverdriveMediaMarkers.js new file mode 100644 index 00000000..a65e15c3 --- /dev/null +++ b/server/utils/parsers/parseOverdriveMediaMarkers.js @@ -0,0 +1,148 @@ +const Logger = require('../../Logger') + +// given a list of audio files, extract all of the Overdrive Media Markers metaTags, and return an array of them as XML +function extractOverdriveMediaMarkers(includedAudioFiles) { + Logger.debug('[parseOverdriveMediaMarkers] Extracting overdrive media markers') + var markers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(notUndefined => notUndefined !== undefined).filter(elem => { return elem !== null }) || [] + + return markers +} + +// given the array of Overdrive Media Markers from generateOverdriveMediaMarkers() +// parse and clean them in to something a bit more usable +function cleanOverdriveMediaMarkers(overdriveMediaMarkers) { + Logger.debug('[parseOverdriveMediaMarkers] Cleaning up overdrive media markers') + /* + returns an array of arrays of objects. Each inner array corresponds to an audio track, with it's objects being a chapter: + [ + [ + { + "Name": "Chapter 1", + "Time": "0:00.000" + }, + { + "Name": "Chapter 2", + "Time": "15:51.000" + }, + { etc } + ] + ] + */ + + var parseString = require('xml2js').parseString; // function to convert xml to JSON + var parsedOverdriveMediaMarkers = [] + + overdriveMediaMarkers.forEach(function (item, index) { + var parsed_result + parseString(item, function (err, result) { + /* + result.Markers.Marker is the result of parsing the XML for the MediaMarker tags for the MP3 file (Part##.mp3) + it is shaped like this, and needs further cleaning below: + [ + { + "Name": [ + "Chapter 1: " + ], + "Time": [ + "0:00.000" + ] + }, + { + ANOTHER CHAPTER + }, + ] + */ + + // The values for Name and Time in results.Markers.Marker are returned as Arrays from parseString and should be strings + parsed_result = objectValuesArrayToString(result.Markers.Marker) + }) + + parsedOverdriveMediaMarkers.push(parsed_result) + }) + + return removeExtraChapters(parsedOverdriveMediaMarkers) +} + +// given an array of objects, convert any values that are arrays to strings +function objectValuesArrayToString(arrayOfObjects) { + Logger.debug('[parseOverdriveMediaMarkers] Converting Marker object values from arrays to strings') + arrayOfObjects.forEach((item) => { + Object.keys(item).forEach(key => { + item[key] = item[key].toString() + }) + }) + + return arrayOfObjects +} + +// Overdrive sometimes has weird chapters and subchapters defined +// These aren't necessary, so lets remove them +function removeExtraChapters(parsedOverdriveMediaMarkers) { + Logger.debug('[parseOverdriveMediaMarkers] Removing any unnecessary chapters') + const weirdChapterFilterRegex = /([(]\d|[cC]ontinued)/ + var cleaned = [] + parsedOverdriveMediaMarkers.forEach(function (item) { + cleaned.push(item.filter(chapter => !weirdChapterFilterRegex.test(chapter.Name))) + }) + + return cleaned +} + +// Given a set of chapters from generateParsedChapters, add the end time to each one +function addChapterEndTimes(chapters, totalAudioDuration) { + Logger.debug('[parseOverdriveMediaMarkers] Adding chapter end times') + chapters.forEach((chapter, chapter_index) => { + if (chapter_index < chapters.length - 1) { + chapter.end = chapters[chapter_index + 1].start + } else { + chapter.end = totalAudioDuration + } + }) + + return chapters +} + +// The function that actually generates the Chapters object that we update ABS with +function generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers) { + Logger.debug('[parseOverdriveMediaMarkers] Generating new chapters for ABS') + // logic ported over from benonymity's OverdriveChapterizer: + // https://github.com/benonymity/OverdriveChapterizer/blob/main/chapters.py + var parsedChapters = [] + var length = 0.0 + var index = 0 + var time = 0.0 + + // cleanedOverdriveMediaMarkers is an array of array of objects, where the inner array matches to the included audio files tracks + // this allows us to leverage the individual track durations when calculating the start times of chapters in tracks after the first (using length) + includedAudioFiles.forEach((track, track_index) => { + cleanedOverdriveMediaMarkers[track_index].forEach((chapter) => { + time = chapter.Time.split(":") + time = length + parseFloat(time[0]) * 60 + parseFloat(time[1]) + var newChapterData = { + id: index++, + start: time, + title: chapter.Name + } + parsedChapters.push(newChapterData) + }) + length += track.duration + }) + + parsedChapters = addChapterEndTimes(parsedChapters, length) // we need all the start times sorted out before we can add the end times + + return parsedChapters +} + +module.exports.overdriveMediaMarkersExist = (includedAudioFiles) => { + return extractOverdriveMediaMarkers(includedAudioFiles).length > 1 +} + +module.exports.parseOverdriveMediaMarkersAsChapters = (includedAudioFiles) => { + Logger.info('[parseOverdriveMediaMarkers] Parsing of Overdrive Media Markers started') + + var overdriveMediaMarkers = extractOverdriveMediaMarkers(includedAudioFiles) + var cleanedOverdriveMediaMarkers = cleanOverdriveMediaMarkers(overdriveMediaMarkers) + var parsedChapters = generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers) + + return parsedChapters +} \ No newline at end of file diff --git a/server/utils/prober.js b/server/utils/prober.js index d7d60c2f..890899b8 100644 --- a/server/utils/prober.js +++ b/server/utils/prober.js @@ -192,6 +192,7 @@ function parseTags(format, verbose) { file_tag_movement: tryGrabTags(format, 'movement', 'mvin'), file_tag_genre1: tryGrabTags(format, 'tmp_genre1', 'genre1'), file_tag_genre2: tryGrabTags(format, 'tmp_genre2', 'genre2'), + file_tag_overdrive_media_marker: tryGrabTags(format, 'OverDrive MediaMarkers'), } for (const key in tags) { if (!tags[key]) {