Merge remote-tracking branch 'origin/master' into auth_passportjs

server/Server.js

@@ -1,4 +1,5 @@
 const Path = require('path')
+const Sequelize = require('sequelize')
 const express = require('express')
 const http = require('http')
 const fs = require('./libs/fsExtra')
@@ -8,7 +9,6 @@ const rateLimit = require('./libs/expressRateLimit')
 const { version } = require('../package.json')
 
 // Utils
-const dbMigration = require('./utils/dbMigration')
 const filePerms = require('./utils/filePerms')
 const fileUtils = require('./utils/fileUtils')
 const Logger = require('./Logger')
@@ -16,14 +16,16 @@ const Logger = require('./Logger')
 const Auth = require('./Auth')
 const Watcher = require('./Watcher')
 const Scanner = require('./scanner/Scanner')
-const Db = require('./Db')
+const Database = require('./Database')
 const SocketAuthority = require('./SocketAuthority')
 
+const routes = require('./routes/index')
+
 const ApiRouter = require('./routers/ApiRouter')
 const HlsRouter = require('./routers/HlsRouter')
-const StaticRouter = require('./routers/StaticRouter')
 
 const NotificationManager = require('./managers/NotificationManager')
+const EmailManager = require('./managers/EmailManager')
 const CoverManager = require('./managers/CoverManager')
 const AbMergeManager = require('./managers/AbMergeManager')
 const CacheManager = require('./managers/CacheManager')
@@ -63,30 +65,29 @@ class Server {
       filePerms.setDefaultDirSync(global.MetadataPath, false)
     }
 
-    this.db = new Db()
     this.watcher = new Watcher()
-    this.auth = new Auth(this.db)
+    this.auth = new Auth()
 
     // Managers
     this.taskManager = new TaskManager()
-    this.notificationManager = new NotificationManager(this.db)
-    this.backupManager = new BackupManager(this.db)
-    this.logManager = new LogManager(this.db)
+    this.notificationManager = new NotificationManager()
+    this.emailManager = new EmailManager()
+    this.backupManager = new BackupManager()
+    this.logManager = new LogManager()
     this.cacheManager = new CacheManager()
-    this.abMergeManager = new AbMergeManager(this.db, this.taskManager)
-    this.playbackSessionManager = new PlaybackSessionManager(this.db)
-    this.coverManager = new CoverManager(this.db, this.cacheManager)
-    this.podcastManager = new PodcastManager(this.db, this.watcher, this.notificationManager, this.taskManager)
-    this.audioMetadataManager = new AudioMetadataMangaer(this.db, this.taskManager)
-    this.rssFeedManager = new RssFeedManager(this.db)
+    this.abMergeManager = new AbMergeManager(this.taskManager)
+    this.playbackSessionManager = new PlaybackSessionManager()
+    this.coverManager = new CoverManager(this.cacheManager)
+    this.podcastManager = new PodcastManager(this.watcher, this.notificationManager, this.taskManager)
+    this.audioMetadataManager = new AudioMetadataMangaer(this.taskManager)
+    this.rssFeedManager = new RssFeedManager()
 
-    this.scanner = new Scanner(this.db, this.coverManager)
-    this.cronManager = new CronManager(this.db, this.scanner, this.podcastManager)
+    this.scanner = new Scanner(this.coverManager, this.taskManager)
+    this.cronManager = new CronManager(this.scanner, this.podcastManager)
 
     // Routers
     this.apiRouter = new ApiRouter(this)
-    this.hlsRouter = new HlsRouter(this.db, this.auth, this.playbackSessionManager)
-    this.staticRouter = new StaticRouter(this.db)
+    this.hlsRouter = new HlsRouter(this.auth, this.playbackSessionManager)
 
     Logger.logManager = this.logManager
 
@@ -98,42 +99,37 @@ class Server {
     this.auth.isAuthenticated(req, res, next)
   }
 
+  /**
+   * Initialize database, backups, logs, rss feeds, cron jobs & watcher
+   * Cleanup stale/invalid data
+   */
   async init() {
     Logger.info('[Server] Init v' + version)
     await this.playbackSessionManager.removeOrphanStreams()
 
-    const previousVersion = await this.db.checkPreviousVersion() // Returns null if same server version
-    if (previousVersion) {
-      Logger.debug(`[Server] Upgraded from previous version ${previousVersion}`)
-    }
-    if (previousVersion && previousVersion.localeCompare('2.0.0') < 0) { // Old version data model migration
-      Logger.debug(`[Server] Previous version was < 2.0.0 - migration required`)
-      await dbMigration.migrate(this.db)
-    } else {
-      await this.db.init()
-    }
+    await Database.init(false)
 
     // Create token secret if does not exist (Added v2.1.0)
-    if (!this.db.serverSettings.tokenSecret) {
+    if (!Database.serverSettings.tokenSecret) {
       await this.auth.initTokenSecret()
     }
 
     await this.cleanUserData() // Remove invalid user item progress
-    await this.purgeMetadata() // Remove metadata folders without library item
-    await this.playbackSessionManager.removeInvalidSessions()
     await this.cacheManager.ensureCachePaths()
 
     await this.backupManager.init()
     await this.logManager.init()
-    await this.apiRouter.checkRemoveEmptySeries(this.db.series) // Remove empty series
+    await this.apiRouter.checkRemoveEmptySeries(Database.series) // Remove empty series
     await this.rssFeedManager.init()
-    this.cronManager.init()
 
-    if (this.db.serverSettings.scannerDisableWatcher) {
+    const libraries = await Database.models.library.getAllOldLibraries()
+    this.cronManager.init(libraries)
+
+    if (Database.serverSettings.scannerDisableWatcher) {
       Logger.info(`[Server] Watcher is disabled`)
       this.watcher.disabled = true
     } else {
-      this.watcher.initWatcher(this.db.libraries)
+      this.watcher.initWatcher(libraries)
       this.watcher.on('files', this.filesChanged.bind(this))
     }
   }
@@ -169,49 +165,37 @@ class Server {
 
     this.server = http.createServer(app)
 
-    router.use(fileUpload())
-    router.use(express.urlencoded({ extended: true, limit: "5mb" }))
+    router.use(this.auth.cors)
+    router.use(fileUpload({
+      defCharset: 'utf8',
+      defParamCharset: 'utf8',
+      useTempFiles: true,
+      tempFileDir: Path.join(global.MetadataPath, 'tmp')
+    }))
+    router.use(express.urlencoded({ extended: true, limit: "5mb" }));
     router.use(express.json({ limit: "5mb" }))
 
     // Static path to generated nuxt
     const distPath = Path.join(global.appRoot, '/client/dist')
     router.use(express.static(distPath))
 
-    // Metadata folder static path
-    router.use('/metadata', this.authMiddleware.bind(this), express.static(global.MetadataPath))
-
     // Static folder
     router.use(express.static(Path.join(global.appRoot, 'static')))
 
+    // router.use('/api/v1', routes) // TODO: New routes
     router.use('/api', this.authMiddleware.bind(this), this.apiRouter.router)
     router.use('/hls', this.authMiddleware.bind(this), this.hlsRouter.router)
-    router.use('/s', this.authMiddleware.bind(this), this.staticRouter.router)
-
-    // Auth routes
-    this.auth.initAuthRoutes(router)
-
-    // EBook static file routes
-    router.get('/ebook/:library/:folder/*', (req, res) => {
-      const library = this.db.libraries.find(lib => lib.id === req.params.library)
-      if (!library) return res.sendStatus(404)
-      const folder = library.folders.find(fol => fol.id === req.params.folder)
-      if (!folder) return res.status(404).send('Folder not found')
-
-      const remainingPath = req.params['0']
-      const fullPath = Path.join(folder.fullPath, remainingPath)
-      res.sendFile(fullPath)
-    })
 
     // RSS Feed temp route
-    router.get('/feed/:id', (req, res) => {
-      Logger.info(`[Server] Requesting rss feed ${req.params.id}`)
+    router.get('/feed/:slug', (req, res) => {
+      Logger.info(`[Server] Requesting rss feed ${req.params.slug}`)
       this.rssFeedManager.getFeed(req, res)
     })
-    router.get('/feed/:id/cover', (req, res) => {
+    router.get('/feed/:slug/cover', (req, res) => {
       this.rssFeedManager.getFeedCover(req, res)
     })
-    router.get('/feed/:id/item/:episodeId/*', (req, res) => {
-      Logger.debug(`[Server] Requesting rss feed episode ${req.params.id}/${req.params.episodeId}`)
+    router.get('/feed/:slug/item/:episodeId/*', (req, res) => {
+      Logger.debug(`[Server] Requesting rss feed episode ${req.params.slug}/${req.params.episodeId}`)
       this.rssFeedManager.getFeedItem(req, res)
     })
 
@@ -240,7 +224,7 @@ class Server {
     // router.post('/login', passport.authenticate('local', this.auth.login), this.auth.loginResult.bind(this))
     // router.post('/logout', this.authMiddleware.bind(this), this.logout.bind(this))
     router.post('/init', (req, res) => {
-      if (this.db.hasRootUser) {
+      if (Database.hasRootUser) {
         Logger.error(`[Server] attempt to init server when server already has a root user`)
         return res.sendStatus(500)
       }
@@ -250,8 +234,8 @@ class Server {
       // status check for client to see if server has been initialized
       // server has been initialized if a root user exists
       const payload = {
-        isInit: this.db.hasRootUser,
-        language: this.db.serverSettings.language
+        isInit: Database.hasRootUser,
+        language: Database.serverSettings.language
       }
       if (!payload.isInit) {
         payload.ConfigPath = global.ConfigPath
@@ -277,10 +261,10 @@ class Server {
   async initializeServer(req, res) {
     Logger.info(`[Server] Initializing new server`)
     const newRoot = req.body.newRoot
-    let rootPash = newRoot.password ? await this.auth.hashPass(newRoot.password) : ''
+    const rootUsername = newRoot.username || 'root'
+    const rootPash = newRoot.password ? await this.auth.hashPass(newRoot.password) : ''
     if (!rootPash) Logger.warn(`[Server] Creating root user with no password`)
-    let rootToken = await this.auth.generateAccessToken({ userId: 'root', username: newRoot.username })
-    await this.db.createRootUser(newRoot.username, rootPash, rootToken)
+    await Database.createRootUser(rootUsername, rootPash, this.auth)
 
     res.sendStatus(200)
   }
@@ -290,54 +274,49 @@ class Server {
     await this.scanner.scanFilesChanged(fileUpdates)
   }
 
-  // Remove unused /metadata/items/{id} folders
-  async purgeMetadata() {
-    const itemsMetadata = Path.join(global.MetadataPath, 'items')
-    if (!(await fs.pathExists(itemsMetadata))) return
-    const foldersInItemsMetadata = await fs.readdir(itemsMetadata)
-
-    let purged = 0
-    await Promise.all(foldersInItemsMetadata.map(async foldername => {
-      const hasMatchingItem = this.db.libraryItems.find(ab => ab.id === foldername)
-      if (!hasMatchingItem) {
-        const folderPath = Path.join(itemsMetadata, foldername)
-        Logger.debug(`[Server] Purging unused metadata ${folderPath}`)
-
-        await fs.remove(folderPath).then(() => {
-          purged++
-        }).catch((err) => {
-          Logger.error(`[Server] Failed to delete folder path ${folderPath}`, err)
-        })
-      }
-    }))
-    if (purged > 0) {
-      Logger.info(`[Server] Purged ${purged} unused library item metadata`)
-    }
-    return purged
-  }
-
-  // Remove user media progress with items that no longer exist & remove seriesHideFrom that no longer exist
+  /**
+   * Remove user media progress for items that no longer exist & remove seriesHideFrom that no longer exist
+   */
   async cleanUserData() {
-    for (let i = 0; i < this.db.users.length; i++) {
-      const _user = this.db.users[i]
-      let hasUpdated = false
-      if (_user.mediaProgress.length) {
-        const lengthBefore = _user.mediaProgress.length
-        _user.mediaProgress = _user.mediaProgress.filter(mp => {
-          const libraryItem = this.db.libraryItems.find(li => li.id === mp.libraryItemId)
-          if (!libraryItem) return false
-          if (mp.episodeId && (libraryItem.mediaType !== 'podcast' || !libraryItem.media.checkHasEpisode(mp.episodeId))) return false // Episode not found
-          return true
-        })
-
-        if (lengthBefore > _user.mediaProgress.length) {
-          Logger.debug(`[Server] Removing ${_user.mediaProgress.length - lengthBefore} media progress data from user ${_user.username}`)
-          hasUpdated = true
+    // Get all media progress without an associated media item
+    const mediaProgressToRemove = await Database.models.mediaProgress.findAll({
+      where: {
+        '$podcastEpisode.id$': null,
+        '$book.id$': null
+      },
+      attributes: ['id'],
+      include: [
+        {
+          model: Database.models.book,
+          attributes: ['id']
+        },
+        {
+          model: Database.models.podcastEpisode,
+          attributes: ['id']
         }
+      ]
+    })
+    if (mediaProgressToRemove.length) {
+      // Remove media progress
+      const mediaProgressRemoved = await Database.models.mediaProgress.destroy({
+        where: {
+          id: {
+            [Sequelize.Op.in]: mediaProgressToRemove.map(mp => mp.id)
+          }
+        }
+      })
+      if (mediaProgressRemoved) {
+        Logger.info(`[Server] Removed ${mediaProgressRemoved} media progress for media items that no longer exist in db`)
       }
+    }
+
+    // Remove series from hide from continue listening that no longer exist
+    const users = await Database.models.user.getOldUsers()
+    for (const _user of users) {
+      let hasUpdated = false
       if (_user.seriesHideFromContinueListening.length) {
         _user.seriesHideFromContinueListening = _user.seriesHideFromContinueListening.filter(seriesId => {
-          if (!this.db.series.some(se => se.id === seriesId)) { // Series removed
+          if (!Database.series.some(se => se.id === seriesId)) { // Series removed
             hasUpdated = true
             return false
           }
@@ -345,7 +324,7 @@ class Server {
         })
       }
       if (hasUpdated) {
-        await this.db.updateEntity('user', _user)
+        await Database.updateUser(_user)
       }
     }
   }
@@ -358,8 +337,8 @@ class Server {
 
   getLoginRateLimiter() {
     return rateLimit({
-      windowMs: this.db.serverSettings.rateLimitLoginWindow, // 5 minutes
-      max: this.db.serverSettings.rateLimitLoginRequests,
+      windowMs: Database.serverSettings.rateLimitLoginWindow, // 5 minutes
+      max: Database.serverSettings.rateLimitLoginRequests,
       skipSuccessfulRequests: true,
       onLimitReached: this.loginLimitReached
     })