extern/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2024-10-05 09:42:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support accent-insensitive matching using the sqlean sqlite3 unicode extension

Browse Source
This commit is contained in:
mikiher 2024-07-27 21:56:07 +03:00
parent 6c379fc3a7
commit dedf6e5d4b
4 changed files with 75 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
server/Database.js
View File

@ -207,6 +207,7 @@ class Database {
try {
await this.sequelize.authenticate()
await this.loadExtensions([process.env.SQLEAN_UNICODE_PATH])
Logger.info(`[Database] Db connection was successful`)
return true
} catch (error) {
@ -215,6 +216,30 @@ class Database {
}
}
async loadExtensions(extensions) {
// This is a hack to get the db connection for loading extensions.
// The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize.
// See https://github.com/sequelize/sequelize/issues/12487
// This is not a public API and may break in the future.
const db = await this.sequelize.dialect.connectionManager.getConnection()
if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions')
for (const ext of extensions) {
Logger.info(`[Database] Loading extension ${ext}`)
await new Promise((resolve, reject) => {
db.loadExtension(ext, (err) => {
if (err) {
Logger.error(`[Database] Failed to load extension ${ext}`, err)
reject(err)
return
}
Logger.info(`[Database] Successfully loaded extension ${ext}`)
resolve()
})
})
}
}
/**
* Disconnect from db
*/
@ -801,6 +826,23 @@ class Database {
Logger.warn(`Removed ${badSessionsRemoved} sessions that were 3 seconds or less`)
}
}
normalize(value) {
return `lower(unaccent(${value}))`
}
async getNormalizedQuery(query) {
const escapedQuery = this.sequelize.escape(query)
const normalizedQuery = this.normalize(escapedQuery)
const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQuery} as normalized_query`)
return normalizedQueryResult[0][0].normalized_query
}
matchExpression(column, normalizedQuery) {
const normalizedPattern = this.sequelize.escape(`%${normalizedQuery}%`)
const normalizedColumn = this.normalize(column)
return `${normalizedColumn} LIKE ${normalizedPattern}`
}
}
module.exports = new Database()

6
server/utils/queries/authorFilters.js
View File

@ -60,12 +60,10 @@ module.exports = {
* @returns {Promise<Object[]>} oldAuthor with numBooks
*/
async search(libraryId, query, limit, offset) {
const matchAuthor = Database.matchExpression('name', query)
const authors = await Database.authorModel.findAll({
where: {
name: {
[Sequelize.Op.substring]: query
},
libraryId
[Sequelize.Op.and]: [Sequelize.literal(matchAuthor), { libraryId }]
},
attributes: {
include: [[Sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 'numBooks']]

41
server/utils/queries/libraryItemsBookFilters.js
View File

@ -975,21 +975,18 @@ module.exports = {
async search(oldUser, oldLibrary, query, limit, offset) {
const userPermissionBookWhere = this.getUserPermissionBookWhereQuery(oldUser)
const normalizedQuery = await Database.getNormalizedQuery(query)
const matchTitle = Database.matchExpression('title', normalizedQuery)
const matchSubtitle = Database.matchExpression('subtitle', normalizedQuery)
// Search title, subtitle, asin, isbn
const books = await Database.bookModel.findAll({
where: [
{
[Sequelize.Op.or]: [
{
title: {
[Sequelize.Op.substring]: query
}
},
{
subtitle: {
[Sequelize.Op.substring]: query
}
},
Sequelize.literal(matchTitle),
Sequelize.literal(matchSubtitle),
{
asin: {
[Sequelize.Op.substring]: query
@ -1044,11 +1041,12 @@ module.exports = {
})
}
const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
// Search narrators
const narratorMatches = []
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
replacements: {
query: `%${query}%`,
libraryId: oldLibrary.id,
limit,
offset
@ -1064,9 +1062,8 @@ module.exports = {
// Search tags
const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
query: `%${query}%`,
libraryId: oldLibrary.id,
limit,
offset
@ -1082,9 +1079,8 @@ module.exports = {
// Search genres
const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
query: `%${query}%`,
libraryId: oldLibrary.id,
limit,
offset
@ -1099,12 +1095,15 @@ module.exports = {
}
// Search series
const matchName = Database.matchExpression('name', normalizedQuery)
const allSeries = await Database.seriesModel.findAll({
where: {
name: {
[Sequelize.Op.substring]: query
},
libraryId: oldLibrary.id
[Sequelize.Op.and]: [
Sequelize.literal(matchName),
{
libraryId: oldLibrary.id
}
]
},
replacements: userPermissionBookWhere.replacements,
include: {
@ -1137,7 +1136,7 @@ module.exports = {
}
// Search authors
const authorMatches = await authorFilters.search(oldLibrary.id, query, limit, offset)
const authorMatches = await authorFilters.search(oldLibrary.id, normalizedQuery, limit, offset)
return {
book: itemMatches,

25
server/utils/queries/libraryItemsPodcastFilters.js
View File

@ -313,21 +313,18 @@ module.exports = {
*/
async search(oldUser, oldLibrary, query, limit, offset) {
const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser)
const normalizedQuery = await Database.getNormalizedQuery(query)
const matchTitle = Database.matchExpression('title', normalizedQuery)
const matchAuthor = Database.matchExpression('author', normalizedQuery)
// Search title, author, itunesId, itunesArtistId
const podcasts = await Database.podcastModel.findAll({
where: [
{
[Sequelize.Op.or]: [
{
title: {
[Sequelize.Op.substring]: query
}
},
{
author: {
[Sequelize.Op.substring]: query
}
},
Sequelize.literal(matchTitle),
Sequelize.literal(matchAuthor),
{
itunesId: {
[Sequelize.Op.substring]: query
@ -368,11 +365,12 @@ module.exports = {
})
}
const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
// Search tags
const tagMatches = []
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND json_each.value LIKE :query AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
query: `%${query}%`,
libraryId: oldLibrary.id,
limit,
offset
@ -388,9 +386,8 @@ module.exports = {
// Search genres
const genreMatches = []
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND json_each.value LIKE :query AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
replacements: {
query: `%${query}%`,
libraryId: oldLibrary.id,
limit,
offset