From 68621e0c07bc5cbfe4f628b45998f0b09ae2a7ae Mon Sep 17 00:00:00 2001 From: yuuzhan Date: Thu, 2 Feb 2023 12:43:48 -0500 Subject: [PATCH 1/4] Update abmetadataGenerator.js --- server/utils/abmetadataGenerator.js | 39 ++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/server/utils/abmetadataGenerator.js b/server/utils/abmetadataGenerator.js index 2bb118ed..be343979 100644 --- a/server/utils/abmetadataGenerator.js +++ b/server/utils/abmetadataGenerator.js @@ -2,7 +2,7 @@ const fs = require('../libs/fsExtra') const filePerms = require('./filePerms') const package = require('../../package.json') const Logger = require('../Logger') -const { getId } = require('./index') +const { getId, copyValue } = require('./index') const CurrentAbMetadataVersion = 2 @@ -136,6 +136,7 @@ function generate(libraryItem, outputPath) { const mediaType = libraryItem.mediaType fileString += `media=${mediaType}\n` + fileString += `tags=`+JSON.stringify(libraryItem.media.tags)+`\n\n` const metadataMapper = metadataMappers[mediaType] var mediaMetadata = libraryItem.media.metadata @@ -159,7 +160,6 @@ function generate(libraryItem, outputPath) { fileString += `title=${chapter.title}\n` }) } - return fs.writeFile(outputPath, fileString).then(() => { return filePerms.setDefault(outputPath, true).then(() => true) }).catch((error) => { @@ -263,7 +263,16 @@ function parseAbMetadataText(text, mediaType) { } else { Logger.warn(`No media type found in abmetadata file - expecting ${mediaType}`) } - + const abTags = []; + try{ + if (detailLines[0].toLowerCase().startsWith('tags=')) { + var tagLine = detailLines.shift() + var tagsStr = tagLine.substring(5, tagLine.len) + JSON.parse(tagsStr).forEach((loadedTag) => { abTags.push(loadedTag) }) + } + }catch(err){ + Logger.error("Error parsing TAGS:", err.message) + } const metadataMapper = metadataMappers[mediaType] // Put valid book detail values into map const mediaMetadataDetails = {} @@ -301,7 +310,8 @@ function parseAbMetadataText(text, mediaType) { return { metadata: mediaMetadataDetails, - chapters + chapters, + tags: abTags } } module.exports.parse = parseAbMetadataText @@ -377,12 +387,12 @@ function checkArraysChanged(abmetadataArray, mediaArray) { } // Input text from abmetadata file and return object of metadata changes from media metadata -function parseAndCheckForUpdates(text, mediaMetadata, mediaType) { - if (!text || !mediaMetadata || !mediaType) { +function parseAndCheckForUpdates(text, media, mediaType) { + if (!text || !media || !media.metadata || !mediaType) { Logger.error(`Invalid inputs to parseAndCheckForUpdates`) return null } - + const mediaMetadata = media.metadata var updatePayload = {} // Only updated key/values var abmetadataData = parseAbMetadataText(text, mediaType) @@ -411,7 +421,18 @@ function parseAndCheckForUpdates(text, mediaMetadata, mediaType) { Logger.warn('[abmetadataGenerator] Invalid key', key) } } - + try{ + if(abmetadataData.tags.length > 0){ + abmetadataData.tags.forEach((tag) => { + if(media.tags.includes(tag) == false){ + media.tags.push(copyValue(tag)) + } + }) + } + } + catch(err){ + Logger.error("[abmetadataGenerator] Error parsing tags", err.message) + } return updatePayload } -module.exports.parseAndCheckForUpdates = parseAndCheckForUpdates \ No newline at end of file +module.exports.parseAndCheckForUpdates = parseAndCheckForUpdates From 7a751b8f91df5d94de1b04b5a538940dbdd2d4a9 Mon Sep 17 00:00:00 2001 From: yuuzhan Date: Thu, 2 Feb 2023 12:46:22 -0500 Subject: [PATCH 2/4] Updated function parseAndCheckForUpdates to pass Library Item rather then just the metadata object --- server/objects/mediaTypes/Book.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/objects/mediaTypes/Book.js b/server/objects/mediaTypes/Book.js index bc34bbb8..321d0170 100644 --- a/server/objects/mediaTypes/Book.js +++ b/server/objects/mediaTypes/Book.js @@ -252,7 +252,7 @@ class Book { if (metadataAbs) { Logger.debug(`[Book] Found metadata.abs file for "${this.metadata.title}"`) const metadataText = await readTextFile(metadataAbs.metadata.path) - const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this.metadata, 'book') + const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'book') if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) { Logger.debug(`[Book] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates) metadataUpdatePayload = { @@ -489,4 +489,4 @@ class Book { return this.metadata.authorName } } -module.exports = Book \ No newline at end of file +module.exports = Book From 639b6005707e0f1f6a2e8fa47adffe84cc18ba9e Mon Sep 17 00:00:00 2001 From: yuuzhan Date: Thu, 2 Feb 2023 12:47:12 -0500 Subject: [PATCH 3/4] Updated parseAndCheckForUpdates to pass in LibraryItem instead of Metadata Object --- server/objects/mediaTypes/Podcast.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/objects/mediaTypes/Podcast.js b/server/objects/mediaTypes/Podcast.js index c94c8653..f279be88 100644 --- a/server/objects/mediaTypes/Podcast.js +++ b/server/objects/mediaTypes/Podcast.js @@ -193,7 +193,7 @@ class Podcast { var metadataAbs = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.abs') if (metadataAbs) { var metadataText = await readTextFile(metadataAbs.metadata.path) - var abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this.metadata, 'podcast') + var abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'podcast') if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) { Logger.debug(`[Podcast] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates) metadataUpdatePayload = abmetadataUpdates @@ -305,4 +305,4 @@ class Podcast { return this.episodes.find(ep => ep.id == episodeId) } } -module.exports = Podcast \ No newline at end of file +module.exports = Podcast From 08f765fa518d4e241505f5c7a5bd17ef17ac1073 Mon Sep 17 00:00:00 2001 From: advplyr Date: Thu, 2 Feb 2023 17:13:22 -0600 Subject: [PATCH 4/4] Update parsing and using tags from abmetadata file --- server/objects/mediaTypes/Book.js | 13 ++- server/objects/mediaTypes/Podcast.js | 25 ++++-- server/utils/abmetadataGenerator.js | 128 ++++++++++++++------------- 3 files changed, 96 insertions(+), 70 deletions(-) diff --git a/server/objects/mediaTypes/Book.js b/server/objects/mediaTypes/Book.js index 321d0170..3fa4c1be 100644 --- a/server/objects/mediaTypes/Book.js +++ b/server/objects/mediaTypes/Book.js @@ -255,9 +255,16 @@ class Book { const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'book') if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) { Logger.debug(`[Book] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates) - metadataUpdatePayload = { - ...metadataUpdatePayload, - ...abmetadataUpdates + + if (abmetadataUpdates.tags) { // Set media tags if updated + this.tags = abmetadataUpdates.tags + tagsUpdated = true + } + if (abmetadataUpdates.metadata) { + metadataUpdatePayload = { + ...metadataUpdatePayload, + ...abmetadataUpdates.metadata + } } } } diff --git a/server/objects/mediaTypes/Podcast.js b/server/objects/mediaTypes/Podcast.js index f279be88..2f45cc4f 100644 --- a/server/objects/mediaTypes/Podcast.js +++ b/server/objects/mediaTypes/Podcast.js @@ -188,22 +188,33 @@ class Podcast { } async syncMetadataFiles(textMetadataFiles, opfMetadataOverrideDetails) { - var metadataUpdatePayload = {} + let metadataUpdatePayload = {} + let tagsUpdated = false - var metadataAbs = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.abs') + const metadataAbs = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.abs') if (metadataAbs) { - var metadataText = await readTextFile(metadataAbs.metadata.path) - var abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'podcast') + const metadataText = await readTextFile(metadataAbs.metadata.path) + const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'podcast') if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) { Logger.debug(`[Podcast] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates) - metadataUpdatePayload = abmetadataUpdates + + if (abmetadataUpdates.tags) { // Set media tags if updated + this.tags = abmetadataUpdates.tags + tagsUpdated = true + } + if (abmetadataUpdates.metadata) { + metadataUpdatePayload = { + ...metadataUpdatePayload, + ...abmetadataUpdates.metadata + } + } } } if (Object.keys(metadataUpdatePayload).length) { - return this.metadata.update(metadataUpdatePayload) + return this.metadata.update(metadataUpdatePayload) || tagsUpdated } - return false + return tagsUpdated } searchEpisodes(query) { diff --git a/server/utils/abmetadataGenerator.js b/server/utils/abmetadataGenerator.js index be343979..bf638492 100644 --- a/server/utils/abmetadataGenerator.js +++ b/server/utils/abmetadataGenerator.js @@ -130,13 +130,13 @@ const metadataMappers = { } function generate(libraryItem, outputPath) { - var fileString = `;ABMETADATA${CurrentAbMetadataVersion}\n` + let fileString = `;ABMETADATA${CurrentAbMetadataVersion}\n` fileString += `#audiobookshelf v${package.version}\n\n` const mediaType = libraryItem.mediaType fileString += `media=${mediaType}\n` - fileString += `tags=`+JSON.stringify(libraryItem.media.tags)+`\n\n` + fileString += `tags=${JSON.stringify(libraryItem.media.tags)}\n` const metadataMapper = metadataMappers[mediaType] var mediaMetadata = libraryItem.media.metadata @@ -223,17 +223,31 @@ function parseChapterLines(lines) { return chapter } +function parseTags(value) { + if (!value) return null + try { + const parsedTags = [] + JSON.parse(value).forEach((loadedTag) => { + if (loadedTag.trim()) parsedTags.push(loadedTag) // Only push tags that are non-empty + }) + return parsedTags + } catch (err) { + Logger.error(`[abmetadataGenerator] Error parsing TAGS "${value}":`, err.message) + return null + } +} + function parseAbMetadataText(text, mediaType) { if (!text) return null - var lines = text.split(/\r?\n/) + let lines = text.split(/\r?\n/) // Check first line and get abmetadata version number - var firstLine = lines.shift().toLowerCase() + const firstLine = lines.shift().toLowerCase() if (!firstLine.startsWith(';abmetadata')) { Logger.error(`Invalid abmetadata file first line is not ;abmetadata "${firstLine}"`) return null } - var abmetadataVersion = Number(firstLine.replace(';abmetadata', '').trim()) + const abmetadataVersion = Number(firstLine.replace(';abmetadata', '').trim()) if (isNaN(abmetadataVersion) || abmetadataVersion != CurrentAbMetadataVersion) { Logger.warn(`Invalid abmetadata version ${abmetadataVersion} - must use version ${CurrentAbMetadataVersion}`) return null @@ -244,9 +258,9 @@ function parseAbMetadataText(text, mediaType) { lines = lines.filter(line => !!line.trim() && !ignoreFirstChars.includes(line[0])) // Get lines that map to book details (all lines before the first chapter or description section) - var firstSectionLine = lines.findIndex(l => l.startsWith('[')) - var detailLines = firstSectionLine > 0 ? lines.slice(0, firstSectionLine) : lines - var remainingLines = firstSectionLine > 0 ? lines.slice(firstSectionLine) : [] + const firstSectionLine = lines.findIndex(l => l.startsWith('[')) + const detailLines = firstSectionLine > 0 ? lines.slice(0, firstSectionLine) : lines + const remainingLines = firstSectionLine > 0 ? lines.slice(firstSectionLine) : [] if (!detailLines.length) { Logger.error(`Invalid abmetadata file no detail lines`) @@ -255,64 +269,57 @@ function parseAbMetadataText(text, mediaType) { // Check the media type saved for this abmetadata file show warning if not matching expected if (detailLines[0].toLowerCase().startsWith('media=')) { - var mediaLine = detailLines.shift() // Remove media line - var abMediaType = mediaLine.toLowerCase().split('=')[1].trim() + const mediaLine = detailLines.shift() // Remove media line + const abMediaType = mediaLine.toLowerCase().split('=')[1].trim() if (abMediaType != mediaType) { Logger.warn(`Invalid media type in abmetadata file ${abMediaType} expecting ${mediaType}`) } } else { Logger.warn(`No media type found in abmetadata file - expecting ${mediaType}`) } - const abTags = []; - try{ - if (detailLines[0].toLowerCase().startsWith('tags=')) { - var tagLine = detailLines.shift() - var tagsStr = tagLine.substring(5, tagLine.len) - JSON.parse(tagsStr).forEach((loadedTag) => { abTags.push(loadedTag) }) - } - }catch(err){ - Logger.error("Error parsing TAGS:", err.message) - } + const metadataMapper = metadataMappers[mediaType] // Put valid book detail values into map - const mediaMetadataDetails = {} + const mediaDetails = { + metadata: {}, + chapters: [], + tags: null // When tags are null it will not be used + } + for (let i = 0; i < detailLines.length; i++) { - var line = detailLines[i] - var keyValue = line.split('=') + const line = detailLines[i] + const keyValue = line.split('=') if (keyValue.length < 2) { Logger.warn('abmetadata invalid line has no =', line) - } else if (!metadataMapper[keyValue[0].trim()]) { + } else if (keyValue[0].trim() === 'tags') { // Parse tags + const value = keyValue.slice(1).join('=').trim() // Everything after "tags=" + mediaDetails.tags = parseTags(value) + } else if (!metadataMapper[keyValue[0].trim()]) { // Ensure valid media metadata key Logger.warn(`abmetadata key "${keyValue[0].trim()}" is not a valid ${mediaType} metadata key`) } else { - var key = keyValue.shift().trim() - var value = keyValue.join('=').trim() - mediaMetadataDetails[key] = metadataMapper[key].from(value) + const key = keyValue.shift().trim() + const value = keyValue.join('=').trim() + mediaDetails.metadata[key] = metadataMapper[key].from(value) } } - const chapters = [] - // Parse sections for description and chapters - var sections = parseSections(remainingLines) + const sections = parseSections(remainingLines) sections.forEach((section) => { - var sectionHeader = section.shift() + const sectionHeader = section.shift() if (sectionHeader.toLowerCase().startsWith('[description]')) { - mediaMetadataDetails.description = section.join('\n') + mediaDetails.metadata.description = section.join('\n') } else if (sectionHeader.toLowerCase().startsWith('[chapter]')) { - var chapter = parseChapterLines(section) + const chapter = parseChapterLines(section) if (chapter) { - chapters.push(chapter) + mediaDetails.chapters.push(chapter) } } }) - chapters.sort((a, b) => a.start - b.start) + mediaDetails.chapters.sort((a, b) => a.start - b.start) - return { - metadata: mediaMetadataDetails, - chapters, - tags: abTags - } + return mediaDetails } module.exports.parse = parseAbMetadataText @@ -386,53 +393,54 @@ function checkArraysChanged(abmetadataArray, mediaArray) { return abmetadataArray.join(',') != mediaArray.join(',') } -// Input text from abmetadata file and return object of metadata changes from media metadata +// Input text from abmetadata file and return object of media changes +// only returns object of changes. empty object means no changes function parseAndCheckForUpdates(text, media, mediaType) { if (!text || !media || !media.metadata || !mediaType) { Logger.error(`Invalid inputs to parseAndCheckForUpdates`) return null } const mediaMetadata = media.metadata - var updatePayload = {} // Only updated key/values + const metadataUpdatePayload = {} // Only updated key/values - var abmetadataData = parseAbMetadataText(text, mediaType) + const abmetadataData = parseAbMetadataText(text, mediaType) if (!abmetadataData || !abmetadataData.metadata) { return null } - var abMetadata = abmetadataData.metadata // Metadata from abmetadata file - + const abMetadata = abmetadataData.metadata // Metadata from abmetadata file for (const key in abMetadata) { if (mediaMetadata[key] !== undefined) { if (key === 'authors') { - var authorUpdatePayload = checkUpdatedBookAuthors(abMetadata[key], mediaMetadata[key]) - if (authorUpdatePayload.hasUpdates) updatePayload.authors = authorUpdatePayload.authors + const authorUpdatePayload = checkUpdatedBookAuthors(abMetadata[key], mediaMetadata[key]) + if (authorUpdatePayload.hasUpdates) metadataUpdatePayload.authors = authorUpdatePayload.authors } else if (key === 'series') { - var seriesUpdatePayload = checkUpdatedBookSeries(abMetadata[key], mediaMetadata[key]) - if (seriesUpdatePayload.hasUpdates) updatePayload.series = seriesUpdatePayload.series + const seriesUpdatePayload = checkUpdatedBookSeries(abMetadata[key], mediaMetadata[key]) + if (seriesUpdatePayload.hasUpdates) metadataUpdatePayload.series = seriesUpdatePayload.series } else if (key === 'genres' || key === 'narrators') { // Compare array differences if (checkArraysChanged(abMetadata[key], mediaMetadata[key])) { - updatePayload[key] = abMetadata[key] + metadataUpdatePayload[key] = abMetadata[key] } } else if (abMetadata[key] !== mediaMetadata[key]) { - updatePayload[key] = abMetadata[key] + metadataUpdatePayload[key] = abMetadata[key] } } else { Logger.warn('[abmetadataGenerator] Invalid key', key) } } - try{ - if(abmetadataData.tags.length > 0){ - abmetadataData.tags.forEach((tag) => { - if(media.tags.includes(tag) == false){ - media.tags.push(copyValue(tag)) - } - }) + + const updatePayload = {} // Only updated key/values + // Check update tags + if (abmetadataData.tags) { + if (checkArraysChanged(abmetadataData.tags, media.tags)) { + updatePayload.tags = abmetadataData.tags } } - catch(err){ - Logger.error("[abmetadataGenerator] Error parsing tags", err.message) + + if (Object.keys(metadataUpdatePayload).length) { + updatePayload.metadata = metadataUpdatePayload } + return updatePayload } module.exports.parseAndCheckForUpdates = parseAndCheckForUpdates