Added support for custom metadata providers

WiP but already open to feedback
This commit is contained in:
Barnabas Ratki 2024-01-03 01:36:56 +01:00
parent 8c6a2ac5dd
commit 8027c4a06f
14 changed files with 642 additions and 4 deletions

View File

@ -109,6 +109,11 @@ export default {
id: 'config-authentication',
title: this.$strings.HeaderAuthentication,
path: '/config/authentication'
},
{
id: 'config-custom-metadata-providers',
title: this.$strings.HeaderCustomMetadataProviders,
path: '/config/custom-metadata-providers'
}
]

View File

@ -0,0 +1,104 @@
<template>
<modals-modal ref="modal" v-model="show" name="custom-metadata-provider" :width="600" :height="'unset'" :processing="processing">
<template #outer>
<div class="absolute top-0 left-0 p-5 w-2/3 overflow-hidden">
<p class="text-3xl text-white truncate">Add custom metadata provider</p>
</div>
</template>
<form @submit.prevent="submitForm">
<div class="px-4 w-full text-sm py-6 rounded-lg bg-bg shadow-lg border border-black-300 overflow-y-auto overflow-x-hidden" style="min-height: 400px; max-height: 80vh">
<div class="w-full p-8">
<div class="w-full mb-4">
<ui-text-input-with-label v-model="newName" :label="$strings.LabelName" />
</div>
<div class="w-full mb-4">
<ui-text-input-with-label v-model="newUrl" :label="$strings.LabelUrl" />
</div>
<div class="w-full mb-4">
<ui-text-input-with-label v-model="newApiKey" :label="$strings.LabelApiKey" type="password" />
</div>
<div class="flex pt-4 px-2">
<div class="flex-grow" />
<ui-btn color="success" type="submit">{{ $strings.ButtonAdd }}</ui-btn>
</div>
</div>
</div>
</form>
</modals-modal>
</template>
<script>
export default {
props: {
value: Boolean,
},
data() {
return {
processing: false,
newName: "",
newUrl: "",
newApiKey: "",
}
},
watch: {
show: {
handler(newVal) {
if (newVal) {
this.init()
}
}
}
},
computed: {
show: {
get() {
return this.value
},
set(val) {
this.$emit('input', val)
}
},
},
methods: {
close() {
// Force close when navigating - used in the table
if (this.$refs.modal) this.$refs.modal.setHide()
},
submitForm() {
if (!this.newName || !this.newUrl || !this.newApiKey) {
this.$toast.error('Must add name, url and API key')
return
}
this.processing = true
this.$axios
.$patch('/api/custom-metadata-providers/admin', {
name: this.newName,
url: this.newUrl,
apiKey: this.newApiKey,
})
.then((data) => {
this.processing = false
if (data.error) {
this.$toast.error(`Failed to add provider: ${data.error}`)
} else {
this.$toast.success('New provider added')
this.show = false
}
})
.catch((error) => {
this.processing = false
console.error('Failed to add provider', error)
this.$toast.error('Failed to add provider')
})
},
init() {
this.processing = false
this.newName = ""
this.newUrl = ""
this.newApiKey = ""
}
},
mounted() {}
}
</script>

View File

@ -0,0 +1,150 @@
<template>
<div>
<div class="text-center">
<table id="providers">
<tr>
<th>{{ $strings.LabelName }}</th>
<th>{{ $strings.LabelUrl }}</th>
<th>{{ $strings.LabelApiKey }}</th>
<th class="w-12"></th>
</tr>
<tr v-for="provider in providers" :key="provider.id">
<td class="text-sm">{{ provider.name }}</td>
<td class="text-sm">{{ provider.url }}</td>
<td class="text-sm">
<span class="custom-provider-api-key">{{ provider.apiKey }}</span>
</td>
<td class="py-0">
<div class="h-8 w-8 flex items-center justify-center text-white text-opacity-50 hover:text-error cursor-pointer" @click.stop="removeProvider(provider)">
<button type="button" :aria-label="$strings.ButtonDelete" class="material-icons text-base">delete</button>
</div>
</td>
</tr>
</table>
</div>
</div>
</template>
<script>
export default {
data() {
return {
providers: [],
}
},
methods: {
addedProvider(provider) {
if (!Array.isArray(this.providers)) return
this.providers.push(provider)
},
removedProvider(provider) {
this.providers = this.providers.filter((p) => p.id !== provider.id)
},
removeProvider(provider) {
this.$axios
.$delete(`/api/custom-metadata-providers/admin/${provider.id}`)
.then((data) => {
if (data.error) {
this.$toast.error(`Failed to remove provider: ${data.error}`)
} else {
this.$toast.success('Provider removed')
}
})
.catch((error) => {
console.error('Failed to remove provider', error)
this.$toast.error('Failed to remove provider')
})
},
loadProviders() {
this.$axios.$get('/api/custom-metadata-providers/admin')
.then((res) => {
this.providers = res.providers
})
.catch((error) => {
console.error('Failed', error)
})
},
init(attempts = 0) {
if (!this.$root.socket) {
if (attempts > 10) {
return console.error('Failed to setup socket listeners')
}
setTimeout(() => {
this.init(++attempts)
}, 250)
return
}
this.$root.socket.on('custom_metadata_provider_added', this.addedProvider)
this.$root.socket.on('custom_metadata_provider_removed', this.removedProvider)
}
},
mounted() {
this.loadProviders()
this.init()
},
beforeDestroy() {
if (this.$refs.addModal) {
this.$refs.addModal.close()
}
if (this.$root.socket) {
this.$root.socket.off('custom_metadata_provider_added', this.addedProvider)
this.$root.socket.off('custom_metadata_provider_removed', this.removedProvider)
}
}
}
</script>
<style>
#providers {
table-layout: fixed;
border-collapse: collapse;
border: 1px solid #474747;
width: 100%;
}
#providers td,
#providers th {
/* border: 1px solid #2e2e2e; */
padding: 8px 8px;
text-align: left;
}
#providers td.py-0 {
padding: 0px 8px;
}
#providers tr:nth-child(even) {
background-color: #373838;
}
#providers tr:nth-child(odd) {
background-color: #2f2f2f;
}
#providers tr:hover {
background-color: #444;
}
#providers th {
font-size: 0.8rem;
font-weight: 600;
padding-top: 5px;
padding-bottom: 5px;
background-color: #272727;
}
.custom-provider-api-key {
padding: 1px;
background-color: #272727;
border-radius: 4px;
color: transparent;
transition: color, background-color 0.5s ease;
}
.custom-provider-api-key:hover {
background-color: transparent;
color: white;
}
</style>

View File

@ -328,6 +328,9 @@ export default {
this.$store.commit('libraries/setEReaderDevices', data.ereaderDevices)
},
customMetadataProvidersChanged() {
this.$store.dispatch('scanners/reFetchCustom')
},
initializeSocket() {
this.socket = this.$nuxtSocket({
name: process.env.NODE_ENV === 'development' ? 'dev' : 'prod',
@ -406,6 +409,10 @@ export default {
this.socket.on('batch_quickmatch_complete', this.batchQuickMatchComplete)
this.socket.on('admin_message', this.adminMessageEvt)
// Custom metadata provider Listeners
this.socket.on('custom_metadata_provider_added', this.customMetadataProvidersChanged)
this.socket.on('custom_metadata_provider_removed', this.customMetadataProvidersChanged)
},
showUpdateToast(versionData) {
var ignoreVersion = localStorage.getItem('ignoreVersion')
@ -541,6 +548,7 @@ export default {
window.addEventListener('keydown', this.keyDown)
this.$store.dispatch('libraries/load')
this.$store.dispatch('scanners/reFetchCustom')
this.initLocalStorage()

View File

@ -0,0 +1,45 @@
<template>
<div id="authentication-settings">
<app-settings-content :header-text="$strings.HeaderCustomMetadataProviders">
<template #header-items>
<ui-tooltip :text="$strings.LabelClickForMoreInfo" class="inline-flex ml-2">
<a href="https://www.audiobookshelf.org/guides/#" target="_blank" class="inline-flex">
<span class="material-icons text-xl w-5 text-gray-200">help_outline</span>
</a>
</ui-tooltip>
<div class="flex-grow" />
<ui-btn color="primary" small @click="setShowAddModal()">{{ $strings.ButtonAdd }}</ui-btn>
</template>
<tables-custom-metadata-provider-table class="pt-2" />
<modals-add-custom-metadata-provider-modal ref="addModal" v-model="showAddModal" />
</app-settings-content>
</div>
</template>
<script>
export default {
async asyncData({ store, redirect }) {
if (!store.getters['user/getIsAdminOrUp']) {
redirect('/')
return
}
return {}
},
data() {
return {
showAddModal: false,
}
},
methods: {
setShowAddModal() {
this.showAddModal = true
}
}
}
</script>
<style>
</style>

View File

@ -73,6 +73,33 @@ export const state = () => ({
export const getters = {}
export const actions = {}
export const actions = {
reFetchCustom({ dispatch, commit }) {
return this.$axios
.$get(`/api/custom-metadata-providers`)
.then((data) => {
const providers = data.providers
export const mutations = {}
commit('setCustomProviders', providers)
return data
})
.catch((error) => {
console.error('Failed', error)
return false
})
},
}
export const mutations = {
setCustomProviders(state, providers) {
// clear previous values, and add new values to the end
state.providers = state.providers.filter((p) => !p.value.startsWith("custom-"));
state.providers = [
...state.providers,
...providers.map((p) => {return {
text: p.name,
value: p.slug,
}})
]
},
}

View File

@ -96,6 +96,7 @@
"HeaderAudiobookTools": "Audiobook File Management Tools",
"HeaderAudioTracks": "Audio Tracks",
"HeaderAuthentication": "Authentication",
"HeaderCustomMetadataProviders": "Custom metadata providers",
"HeaderBackups": "Backups",
"HeaderChangePassword": "Change Password",
"HeaderChapters": "Chapters",
@ -540,6 +541,8 @@
"LabelYourBookmarks": "Your Bookmarks",
"LabelYourPlaylists": "Your Playlists",
"LabelYourProgress": "Your Progress",
"LabelUrl": "URL",
"LabelApiKey": "API Key",
"MessageAddToPlayerQueue": "Add to player queue",
"MessageAppriseDescription": "To use this feature you will need to have an instance of <a href=\"https://github.com/caronc/apprise-api\" target=\"_blank\">Apprise API</a> running or an api that will handle those same requests. <br />The Apprise API Url should be the full URL path to send the notification, e.g., if your API instance is served at <code>http://192.168.1.1:8337</code> then you would put <code>http://192.168.1.1:8337/notify</code>.",
"MessageBackupsDescription": "Backups include users, user progress, library item details, server settings, and images stored in <code>/metadata/items</code> & <code>/metadata/authors</code>. Backups <strong>do not</strong> include any files stored in your library folders.",

View File

@ -132,6 +132,11 @@ class Database {
return this.models.playbackSession
}
/** @type {typeof import('./models/CustomMetadataProvider')} */
get customMetadataProviderModel() {
return this.models.customMetadataProvider
}
/**
* Check if db file exists
* @returns {boolean}
@ -245,6 +250,7 @@ class Database {
require('./models/Feed').init(this.sequelize)
require('./models/FeedEpisode').init(this.sequelize)
require('./models/Setting').init(this.sequelize)
require('./models/CustomMetadataProvider').init(this.sequelize)
return this.sequelize.sync({ force, alter: false })
}
@ -694,6 +700,45 @@ class Database {
})
}
/**
* Returns true if a custom provider with the given slug exists
* @param {string} providerSlug
* @return {boolean}
*/
async doesCustomProviderExistBySlug(providerSlug) {
const id = providerSlug.split("custom-")[1]
if (!id) {
return false
}
return !!await this.customMetadataProviderModel.findByPk(id)
}
/**
* Removes a custom metadata provider
* @param {string} id
*/
async removeCustomMetadataProviderById(id) {
// destroy metadta provider
await this.customMetadataProviderModel.destroy({
where: {
id,
}
})
const slug = `custom-${id}`;
// fallback libraries using it to google
await this.libraryModel.update({
provider: "google",
}, {
where: {
provider: slug,
}
});
}
/**
* Clean invalid records in database
* Series should have atleast one Book

View File

@ -51,6 +51,11 @@ class LibraryController {
}
}
// Validate that the custom provider exists if given any
if (newLibraryPayload.provider && newLibraryPayload.provider.startsWith("custom-")) {
await Database.doesCustomProviderExistBySlug(newLibraryPayload.provider)
}
const library = new Library()
let currentLargestDisplayOrder = await Database.libraryModel.getMaxDisplayOrder()
@ -175,6 +180,11 @@ class LibraryController {
}
}
// Validate that the custom provider exists if given any
if (req.body.provider && req.body.provider.startsWith("custom-")) {
await Database.doesCustomProviderExistBySlug(req.body.provider)
}
const hasUpdates = library.update(req.body)
// TODO: Should check if this is an update to folder paths or name only
if (hasUpdates) {

View File

@ -717,5 +717,95 @@ class MiscController {
const stats = await adminStats.getStatsForYear(year)
res.json(stats)
}
/**
* GET: /api/custom-metadata-providers
*
* @param {import('express').Request} req
* @param {import('express').Response} res
*/
async getCustomMetadataProviders(req, res) {
const providers = await Database.customMetadataProviderModel.findAll()
res.json({
providers: providers.map((p) => p.toUserJson()),
})
}
/**
* GET: /api/custom-metadata-providers/admin
*
* @param {import('express').Request} req
* @param {import('express').Response} res
*/
async getAdminCustomMetadataProviders(req, res) {
if (!req.user.isAdminOrUp) {
Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
return res.sendStatus(403)
}
const providers = await Database.customMetadataProviderModel.findAll()
res.json({
providers,
})
}
/**
* PATCH: /api/custom-metadata-providers/admin
*
* @param {import('express').Request} req
* @param {import('express').Response} res
*/
async addCustomMetadataProviders(req, res) {
if (!req.user.isAdminOrUp) {
Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
return res.sendStatus(403)
}
const { name, url, apiKey } = req.body;
if (!name || !url || !apiKey) {
return res.status(500).send(`Invalid patch data`)
}
const provider = await Database.customMetadataProviderModel.create({
name,
url,
apiKey,
})
SocketAuthority.adminEmitter('custom_metadata_provider_added', provider)
res.json({
provider,
})
}
/**
* DELETE: /api/custom-metadata-providers/admin/:id
*
* @param {import('express').Request} req
* @param {import('express').Response} res
*/
async deleteCustomMetadataProviders(req, res) {
if (!req.user.isAdminOrUp) {
Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get admin custom metadata providers`)
return res.sendStatus(403)
}
const { id } = req.params;
if (!id) {
return res.status(500).send(`Invalid delete data`)
}
const provider = await Database.customMetadataProviderModel.findByPk(id);
await Database.removeCustomMetadataProviderById(id);
SocketAuthority.adminEmitter('custom_metadata_provider_removed', provider)
res.json({})
}
}
module.exports = new MiscController()

View File

@ -5,6 +5,7 @@ const iTunes = require('../providers/iTunes')
const Audnexus = require('../providers/Audnexus')
const FantLab = require('../providers/FantLab')
const AudiobookCovers = require('../providers/AudiobookCovers')
const CustomProviderAdapter = require('../providers/CustomProviderAdapter')
const Logger = require('../Logger')
const { levenshteinDistance, escapeRegExp } = require('../utils/index')
@ -17,6 +18,7 @@ class BookFinder {
this.audnexus = new Audnexus()
this.fantLab = new FantLab()
this.audiobookCovers = new AudiobookCovers()
this.customProviderAdapter = new CustomProviderAdapter()
this.providers = ['google', 'itunes', 'openlibrary', 'fantlab', 'audiobookcovers', 'audible', 'audible.ca', 'audible.uk', 'audible.au', 'audible.fr', 'audible.de', 'audible.jp', 'audible.it', 'audible.in', 'audible.es']
@ -147,6 +149,13 @@ class BookFinder {
return books
}
async getCustomProviderResults(title, author, providerSlug) {
const books = await this.customProviderAdapter.search(title, author, providerSlug)
if (this.verbose) Logger.debug(`Custom provider '${providerSlug}' Search Results: ${books.length || 0}`)
return books
}
static TitleCandidates = class {
constructor(cleanAuthor) {
@ -315,6 +324,11 @@ class BookFinder {
const maxFuzzySearches = !isNaN(options.maxFuzzySearches) ? Number(options.maxFuzzySearches) : 5
let numFuzzySearches = 0
// Custom providers are assumed to be correct
if (provider.startsWith("custom-")) {
return await this.getCustomProviderResults(title, author, provider)
}
if (!title)
return books
@ -397,8 +411,7 @@ class BookFinder {
books = await this.getFantLabResults(title, author)
} else if (provider === 'audiobookcovers') {
books = await this.getAudiobookCoversResults(title)
}
else {
} else {
books = await this.getGoogleBooksResults(title, author)
}
return books

View File

@ -0,0 +1,58 @@
const { DataTypes, Model, Sequelize } = require('sequelize')
class CustomMetadataProvider extends Model {
constructor(values, options) {
super(values, options)
/** @type {UUIDV4} */
this.id
/** @type {string} */
this.name
/** @type {string} */
this.url
/** @type {string} */
this.apiKey
}
getSlug() {
return `custom-${this.id}`
}
toUserJson() {
return {
name: this.name,
id: this.id,
slug: this.getSlug()
}
}
static findByPk(id) {
this.findOne({
where: {
id,
}
})
}
/**
* Initialize model
* @param {import('../Database').sequelize} sequelize
*/
static init(sequelize) {
super.init({
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true
},
name: DataTypes.STRING,
url: DataTypes.STRING,
apiKey: DataTypes.STRING
}, {
sequelize,
modelName: 'customMetadataProvider'
})
}
}
module.exports = CustomMetadataProvider

View File

@ -0,0 +1,76 @@
const Database = require('../Database')
const axios = require("axios");
const Logger = require("../Logger");
class CustomProviderAdapter {
constructor() {
}
async search(title, author, providerSlug) {
const providerId = providerSlug.split("custom-")[1]
console.log(providerId)
const provider = await Database.customMetadataProviderModel.findOne({
where: {
id: providerId,
}
});
if (!provider) {
throw new Error("Custom provider not found for the given id");
}
const matches = await axios.get(`${provider.url}/search?query=${encodeURIComponent(title)}${!!author ? `&author=${encodeURIComponent(author)}` : ""}`, {
headers: {
"Authorization": provider.apiKey,
},
}).then((res) => {
if (!res || !res.data || !Array.isArray(res.data.matches)) return null
return res.data.matches
}).catch(error => {
Logger.error('[CustomMetadataProvider] Search error', error)
return []
})
if (matches === null) {
throw new Error("Custom provider returned malformed response");
}
// re-map keys to throw out
return matches.map(({
title,
subtitle,
author,
narrator,
publisher,
published_year,
description,
cover,
isbn,
asin,
genres,
tags,
language,
duration,
}) => {
return {
title,
subtitle,
author,
narrator,
publisher,
publishedYear: published_year,
description,
cover,
isbn,
asin,
genres,
tags: tags.join(","),
language,
duration,
}
})
}
}
module.exports = CustomProviderAdapter

View File

@ -318,6 +318,10 @@ class ApiRouter {
this.router.patch('/auth-settings', MiscController.updateAuthSettings.bind(this))
this.router.post('/watcher/update', MiscController.updateWatchedPath.bind(this))
this.router.get('/stats/year/:year', MiscController.getAdminStatsForYear.bind(this))
this.router.get('/custom-metadata-providers', MiscController.getCustomMetadataProviders.bind(this))
this.router.get('/custom-metadata-providers/admin', MiscController.getAdminCustomMetadataProviders.bind(this))
this.router.patch('/custom-metadata-providers/admin', MiscController.addCustomMetadataProviders.bind(this))
this.router.delete('/custom-metadata-providers/admin/:id', MiscController.deleteCustomMetadataProviders.bind(this))
}
async getDirectories(dir, relpath, excludedDirs, level = 0) {