Auto add/update/remove audiobooks, update screenshots

This commit is contained in:
advplyr 2021-09-06 20:14:04 -05:00
parent 2e82370408
commit 07a2a0aefd
12 changed files with 335 additions and 168 deletions

View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf-client",
"version": "1.0.8",
"version": "1.1.0",
"description": "Audiobook manager and player",
"main": "index.js",
"scripts": {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 196 KiB

After

Width:  |  Height:  |  Size: 168 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 MiB

After

Width:  |  Height:  |  Size: 1.2 MiB

View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf",
"version": "1.0.8",
"version": "1.1.0",
"description": "Self-hosted audiobook server for managing and playing audiobooks.",
"main": "index.js",
"scripts": {

View File

@ -28,8 +28,6 @@ will store "With a Subtitle" as the subtitle
#### Features coming soon:
* Auto add and update audiobooks (currently you need to press scan)
* User permissions & editing users
* Support different views to see more details of each audiobook
* Option to download all files in a zip file
* iOS App (Android is in beta [here](https://play.google.com/store/apps/details?id=com.audiobookshelf.app))

View File

@ -1,10 +1,13 @@
const fs = require('fs-extra')
const Logger = require('./Logger')
const BookFinder = require('./BookFinder')
const Audiobook = require('./objects/Audiobook')
const audioFileScanner = require('./utils/audioFileScanner')
const { getAllAudiobookFiles } = require('./utils/scandir')
const { getAllAudiobookFileData, getAudiobookFileData } = require('./utils/scandir')
const { comparePaths, getIno } = require('./utils/index')
const { secondsToTimestamp } = require('./utils/fileUtils')
const { ScanResult } = require('./utils/constants')
class Scanner {
constructor(AUDIOBOOK_PATH, METADATA_PATH, db, emitter) {
@ -60,6 +63,110 @@ class Scanner {
return audiobookDataAudioFiles.filter(abdFile => !!abdFile.ino)
}
async scanAudiobookData(audiobookData) {
var existingAudiobook = this.audiobooks.find(a => a.ino === audiobookData.ino)
Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`)
if (existingAudiobook) {
// REMOVE: No valid audio files
if (!audiobookData.audioFiles.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
return ScanResult.REMOVED
}
audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
// Check for audio files that were removed
var abdAudioFileInos = audiobookData.audioFiles.map(af => af.ino)
var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !abdAudioFileInos.includes(file.ino))
if (removedAudioFiles.length) {
Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
}
// Check for new audio files and sync existing audio files
var newAudioFiles = []
var hasUpdatedAudioFiles = false
audiobookData.audioFiles.forEach((file) => {
var existingAudioFile = existingAudiobook.getAudioFileByIno(file.ino)
if (existingAudioFile) { // Audio file exists, sync paths
if (existingAudiobook.syncAudioFile(existingAudioFile, file)) {
hasUpdatedAudioFiles = true
}
} else {
newAudioFiles.push(file)
}
})
if (newAudioFiles.length) {
Logger.info(`[Scanner] ${newAudioFiles.length} new audio files were found for audiobook "${existingAudiobook.title}"`)
// Scan new audio files found - sets tracks
await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
}
// REMOVE: No valid audio tracks
if (!existingAudiobook.tracks.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
return ScanResult.REMOVED
}
var hasUpdates = removedAudioFiles.length || newAudioFiles.length || hasUpdatedAudioFiles
if (existingAudiobook.checkUpdateMissingParts()) {
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
hasUpdates = true
}
if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
hasUpdates = true
}
// Syncs path and fullPath
if (existingAudiobook.syncPaths(audiobookData)) {
hasUpdates = true
}
if (hasUpdates) {
Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
existingAudiobook.lastUpdate = Date.now()
await this.db.updateAudiobook(existingAudiobook)
this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
return ScanResult.UPDATED
}
return ScanResult.UPTODATE
}
// NEW: Check new audiobook
if (!audiobookData.audioFiles.length) {
Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path)
return ScanResult.NOTHING
}
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles)
if (!audiobook.tracks.length) {
Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
return ScanResult.NOTHING
}
audiobook.checkUpdateMissingParts()
Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
await this.db.insertAudiobook(audiobook)
this.emitter('audiobook_added', audiobook.toJSONMinified())
return ScanResult.ADDED
}
async scan() {
// TEMP - fix relative file paths
// TEMP - update ino for each audiobook
@ -80,7 +187,7 @@ class Scanner {
}
const scanStart = Date.now()
var audiobookDataFound = await getAllAudiobookFiles(this.AudiobookPath, this.db.serverSettings)
var audiobookDataFound = await getAllAudiobookFileData(this.AudiobookPath, this.db.serverSettings)
// Set ino for each ab data as a string
audiobookDataFound = await this.setAudiobookDataInos(audiobookDataFound)
@ -112,97 +219,14 @@ class Scanner {
}
}
// Check for new and updated audiobooks
for (let i = 0; i < audiobookDataFound.length; i++) {
var audiobookData = audiobookDataFound[i]
var existingAudiobook = this.audiobooks.find(a => a.ino === audiobookData.ino)
Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`)
var result = await this.scanAudiobookData(audiobookData)
if (result === ScanResult.ADDED) scanResults.added++
if (result === ScanResult.REMOVED) scanResults.removed++
if (result === ScanResult.UPDATED) scanResults.updated++
if (existingAudiobook) {
if (!audiobookData.audioFiles.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
scanResults.removed++
} else {
audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
var abdAudioFileInos = audiobookData.audioFiles.map(af => af.ino)
// Check for audio files that were removed
var removedAudioFiles = existingAudiobook.audioFiles.filter(file => !abdAudioFileInos.includes(file.ino))
if (removedAudioFiles.length) {
Logger.info(`[Scanner] ${removedAudioFiles.length} audio files removed for audiobook "${existingAudiobook.title}"`)
removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
}
// Check for new audio files and sync existing audio files
var newAudioFiles = []
var hasUpdatedAudioFiles = false
audiobookData.audioFiles.forEach((file) => {
var existingAudioFile = existingAudiobook.getAudioFileByIno(file.ino)
if (existingAudioFile) { // Audio file exists, sync paths
if (existingAudiobook.syncAudioFile(existingAudioFile, file)) {
hasUpdatedAudioFiles = true
}
} else {
newAudioFiles.push(file)
}
})
if (newAudioFiles.length) {
Logger.info(`[Scanner] ${newAudioFiles.length} new audio files were found for audiobook "${existingAudiobook.title}"`)
// Scan new audio files found
await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
}
if (!existingAudiobook.tracks.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
await this.db.removeEntity('audiobook', existingAudiobook.id)
this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
} else {
var hasUpdates = removedAudioFiles.length || newAudioFiles.length || hasUpdatedAudioFiles
if (existingAudiobook.checkUpdateMissingParts()) {
Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
hasUpdates = true
}
if (existingAudiobook.syncOtherFiles(audiobookData.otherFiles)) {
hasUpdates = true
}
// Syncs path and fullPath
if (existingAudiobook.syncPaths(audiobookData)) {
hasUpdates = true
}
if (hasUpdates) {
Logger.info(`[Scanner] "${existingAudiobook.title}" was updated - saving`)
existingAudiobook.lastUpdate = Date.now()
await this.db.updateAudiobook(existingAudiobook)
this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
scanResults.updated++
}
}
} // end if update existing
} else {
if (!audiobookData.audioFiles.length) {
Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path)
} else {
var audiobook = new Audiobook()
audiobook.setData(audiobookData)
await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles)
if (!audiobook.tracks.length) {
Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
} else {
audiobook.checkUpdateMissingParts()
Logger.info(`[Scanner] Audiobook "${audiobook.title}" Scanned (${audiobook.sizePretty}) [${audiobook.durationPretty}]`)
await this.db.insertAudiobook(audiobook)
this.emitter('audiobook_added', audiobook.toJSONMinified())
scanResults.added++
}
} // end if add new
}
var progress = Math.round(100 * (i + 1) / audiobookDataFound.length)
this.emitter('scan_progress', {
scanType: 'files',
@ -222,6 +246,29 @@ class Scanner {
return scanResults
}
async scanAudiobook(audiobookPath) {
var exists = await fs.pathExists(audiobookPath)
if (!exists) {
// Audiobook was deleted, TODO: Should confirm this better
var audiobook = this.db.audiobooks.find(ab => ab.fullPath === audiobookPath)
if (audiobook) {
var audiobookJSON = audiobook.toJSONMinified()
await this.db.removeEntity('audiobook', audiobook.id)
this.emitter('audiobook_removed', audiobookJSON)
return ScanResult.REMOVED
}
Logger.warn('Path was deleted but no audiobook found', audiobookPath)
return ScanResult.NOTHING
}
var audiobookData = await getAudiobookFileData(this.AudiobookPath, audiobookPath, this.db.serverSettings)
if (!audiobookData) {
return ScanResult.NOTHING
}
audiobookData.ino = await getIno(audiobookData.fullPath)
return this.scanAudiobookData(audiobookData)
}
async fetchMetadata(id, trackIndex = 0) {
var audiobook = this.audiobooks.find(a => a.id === id)
if (!audiobook) {

View File

@ -14,6 +14,7 @@ const StreamManager = require('./StreamManager')
const RssFeeds = require('./RssFeeds')
const DownloadManager = require('./DownloadManager')
const Logger = require('./Logger')
const { ScanResult } = require('./utils/constants')
class Server {
constructor(PORT, CONFIG_PATH, METADATA_PATH, AUDIOBOOK_PATH) {
@ -75,8 +76,21 @@ class Server {
})
}
async fileAddedUpdated({ path, fullPath }) { }
async fileRemoved({ path, fullPath }) { }
async newFilesAdded({ dir, files }) {
Logger.info(files.length, 'New Files Added in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('New Files Added result', result)
}
async filesRemoved({ dir, files }) {
Logger.info(files.length, 'Files Removed in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('Files Removed result', result)
}
async filesRenamed({ dir, files }) {
Logger.info(files.length, 'Files Renamed in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('Files Renamed result', result)
}
async scan() {
Logger.info('[Server] Starting Scan')
@ -112,9 +126,9 @@ class Server {
this.auth.init()
this.watcher.initWatcher()
this.watcher.on('file_added', this.fileAddedUpdated.bind(this))
this.watcher.on('file_removed', this.fileRemoved.bind(this))
this.watcher.on('file_updated', this.fileAddedUpdated.bind(this))
this.watcher.on('new_files', this.newFilesAdded.bind(this))
this.watcher.on('removed_files', this.filesRemoved.bind(this))
this.watcher.on('renamed_files', this.filesRenamed.bind(this))
}
authMiddleware(req, res, next) {

View File

@ -1,6 +1,8 @@
var EventEmitter = require('events')
var Logger = require('./Logger')
var Watcher = require('watcher')
const Path = require('path')
const EventEmitter = require('events')
const Watcher = require('watcher')
const Logger = require('./Logger')
const { getIno } = require('./utils/index')
class FolderWatcher extends EventEmitter {
constructor(audiobookPath) {
@ -8,6 +10,11 @@ class FolderWatcher extends EventEmitter {
this.AudiobookPath = audiobookPath
this.folderMap = {}
this.watcher = null
this.pendingBatchDelay = 4000
// Audiobook paths with changes
this.pendingBatch = {}
}
initWatcher() {
@ -46,32 +53,69 @@ class FolderWatcher extends EventEmitter {
return this.watcher.close()
}
onNewFile(path) {
// After [pendingBatchDelay] seconds emit batch
async onNewFile(path) {
Logger.debug('FolderWatcher: New File', path)
this.emit('file_added', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path
})
var dir = Path.dirname(path)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(path)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [path]
}
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('new_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
}
onFileRemoved(path) {
Logger.debug('[FolderWatcher] File Removed', path)
this.emit('file_removed', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path
})
var dir = Path.dirname(path)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(path)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [path]
}
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('removed_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
}
onFileUpdated(path) {
Logger.debug('[FolderWatcher] Updated File', path)
this.emit('file_updated', {
path: path.replace(this.AudiobookPath, ''),
fullPath: path
})
}
onRename(pathFrom, pathTo) {
Logger.debug(`[FolderWatcher] Rename ${pathFrom} => ${pathTo}`)
var dir = Path.dirname(pathTo)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(pathTo)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [pathTo]
}
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('renamed_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
}
}
module.exports = FolderWatcher

View File

@ -0,0 +1,7 @@
module.exports.ScanResult = {
NOTHING: 0,
ADDED: 1,
UPDATED: 2,
REMOVED: 3,
UPTODATE: 4
}

View File

@ -32,17 +32,20 @@ module.exports.levenshteinDistance = levenshteinDistance
const cleanString = (str) => {
if (!str) return ''
// Now supporting all utf-8 characters, can remove this method in future
// replace accented characters: https://stackoverflow.com/a/49901740/7431543
str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "")
// str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "")
const availableChars = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
const cleanChar = (char) => availableChars.indexOf(char) < 0 ? '?' : char
// const availableChars = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
// const cleanChar = (char) => availableChars.indexOf(char) < 0 ? '?' : char
var cleaned = ''
for (let i = 0; i < str.length; i++) {
cleaned += cleanChar(str[i])
}
return cleaned
// var cleaned = ''
// for (let i = 0; i < str.length; i++) {
// cleaned += cleanChar(str[i])
// }
return cleaned.trim()
}
module.exports.cleanString = cleanString

View File

@ -30,7 +30,54 @@ function getFileType(ext) {
return 'unknown'
}
async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
// Input relative filepath, output all details that can be parsed
function getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle = false) {
var pathformat = Path.parse(relpath)
var path = pathformat.dir
if (!path) {
Logger.error('Ignoring file in root dir', relpath)
return null
}
// If relative file directory has 3 folders, then the middle folder will be series
var splitDir = path.split(Path.sep)
var author = null
if (splitDir.length > 1) author = splitDir.shift()
var series = null
if (splitDir.length > 1) series = splitDir.shift()
var title = splitDir.shift()
var publishYear = null
var subtitle = null
// If Title is of format 1999 - Title, then use 1999 as publish year
var publishYearMatch = title.match(/^([0-9]{4}) - (.+)/)
if (publishYearMatch && publishYearMatch.length > 2) {
if (!isNaN(publishYearMatch[1])) {
publishYear = publishYearMatch[1]
title = publishYearMatch[2]
}
}
if (parseSubtitle && title.includes(' - ')) {
var splitOnSubtitle = title.split(' - ')
title = splitOnSubtitle.shift()
subtitle = splitOnSubtitle.join(' - ')
}
return {
author,
title,
subtitle,
series,
publishYear,
path, // relative audiobook path i.e. /Author Name/Book Name/..
fullPath: Path.join(abRootPath, path) // i.e. /audiobook/Author Name/Book Name/..
}
}
async function getAllAudiobookFileData(abRootPath, serverSettings = {}) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
var paths = await getPaths(abRootPath)
@ -38,59 +85,26 @@ async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
paths.files.forEach((filepath) => {
var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
var pathformat = Path.parse(relpath)
var path = pathformat.dir
if (!path) {
Logger.error('Ignoring file in root dir', filepath)
return
}
// If relative file directory has 3 folders, then the middle folder will be series
var splitDir = pathformat.dir.split(Path.sep)
var author = null
if (splitDir.length > 1) author = splitDir.shift()
var series = null
if (splitDir.length > 1) series = splitDir.shift()
var title = splitDir.shift()
var publishYear = null
var subtitle = null
// If Title is of format 1999 - Title, then use 1999 as publish year
var publishYearMatch = title.match(/^([0-9]{4}) - (.+)/)
if (publishYearMatch && publishYearMatch.length > 2) {
if (!isNaN(publishYearMatch[1])) {
publishYear = publishYearMatch[1]
title = publishYearMatch[2]
}
}
if (parseSubtitle && title.includes(' - ')) {
var splitOnSubtitle = title.split(' - ')
title = splitOnSubtitle.shift()
subtitle = splitOnSubtitle.join(' - ')
}
var parsed = Path.parse(relpath)
var path = parsed.dir
if (!audiobooks[path]) {
var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
if (!audiobookData) return
audiobooks[path] = {
author,
title,
subtitle,
series: cleanString(series),
publishYear: publishYear,
path: path,
fullPath: Path.join(abRootPath, path),
...audiobookData,
audioFiles: [],
otherFiles: []
}
}
var fileObj = {
filetype: getFileType(pathformat.ext),
filename: pathformat.base,
filetype: getFileType(parsed.ext),
filename: parsed.base,
path: relpath,
fullPath: filepath,
ext: pathformat.ext
ext: parsed.ext
}
if (fileObj.filetype === 'audio') {
audiobooks[path].audioFiles.push(fileObj)
@ -100,4 +114,44 @@ async function getAllAudiobookFiles(abRootPath, serverSettings = {}) {
})
return Object.values(audiobooks)
}
module.exports.getAllAudiobookFiles = getAllAudiobookFiles
module.exports.getAllAudiobookFileData = getAllAudiobookFileData
async function getAudiobookFileData(abRootPath, audiobookPath, serverSettings = {}) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
var paths = await getPaths(audiobookPath)
var audiobook = null
paths.files.forEach((filepath) => {
var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
if (!audiobook) {
var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
if (!audiobookData) return
audiobook = {
...audiobookData,
audioFiles: [],
otherFiles: []
}
}
var extname = Path.extname(filepath)
var basename = Path.basename(filepath)
var fileObj = {
filetype: getFileType(extname),
filename: basename,
path: relpath,
fullPath: filepath,
ext: extname
}
if (fileObj.filetype === 'audio') {
audiobook.audioFiles.push(fileObj)
} else {
audiobook.otherFiles.push(fileObj)
}
})
return audiobook
}
module.exports.getAudiobookFileData = getAudiobookFileData