diff --git a/server/Server.js b/server/Server.js
index 87b8e167..678691c2 100644
--- a/server/Server.js
+++ b/server/Server.js
@@ -114,7 +114,6 @@ class Server {
 
     await this.backupManager.init()
     await this.logManager.init()
-    await this.apiRouter.checkRemoveEmptySeries(Database.series) // Remove empty series
     await this.rssFeedManager.init()
 
     const libraries = await Database.models.library.getAllOldLibraries()
diff --git a/server/controllers/LibraryItemController.js b/server/controllers/LibraryItemController.js
index bdfc9322..2667128b 100644
--- a/server/controllers/LibraryItemController.js
+++ b/server/controllers/LibraryItemController.js
@@ -135,7 +135,7 @@ class LibraryItemController {
       if (seriesRemoved.length) {
         // Check remove empty series
         Logger.debug(`[LibraryItemController] Series was removed from book. Check if series is now empty.`)
-        await this.checkRemoveEmptySeries(seriesRemoved)
+        await this.checkRemoveEmptySeries(libraryItem.media.id, seriesRemoved.map(se => se.id))
       }
 
       if (isPodcastAutoDownloadUpdated) {
@@ -351,8 +351,21 @@ class LibraryItemController {
 
       await this.createAuthorsAndSeriesForItemUpdate(mediaPayload, libraryItem.libraryId)
 
+      let seriesRemoved = []
+      if (libraryItem.isBook && mediaPayload.metadata?.series) {
+        const seriesIdsInUpdate = (mediaPayload.metadata?.series || []).map(se => se.id)
+        seriesRemoved = libraryItem.media.metadata.series.filter(se => !seriesIdsInUpdate.includes(se.id))
+      }
+
       if (libraryItem.media.update(mediaPayload)) {
         Logger.debug(`[LibraryItemController] Updated library item media ${libraryItem.media.metadata.title}`)
+
+        if (seriesRemoved.length) {
+          // Check remove empty series
+          Logger.debug(`[LibraryItemController] Series was removed from book. Check if series is now empty.`)
+          await this.checkRemoveEmptySeries(libraryItem.media.id, seriesRemoved.map(se => se.id))
+        }
+
         await Database.updateLibraryItem(libraryItem)
         SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded())
         itemsUpdated++
diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js
index c4046688..69179006 100644
--- a/server/routers/ApiRouter.js
+++ b/server/routers/ApiRouter.js
@@ -371,6 +371,7 @@ class ApiRouter {
 
     // Remove series if empty
     if (mediaType === 'book') {
+      // TODO: update filter data
       const bookSeries = await Database.models.bookSeries.findAll({
         where: {
           bookId: mediaItemIds[0]
@@ -440,17 +441,33 @@ class ApiRouter {
     })
   }
 
-  async checkRemoveEmptySeries(seriesToCheck, excludeLibraryItemId = null) {
-    if (!seriesToCheck?.length) return
+  /**
+   * Used when a series is removed from a book
+   * Series is removed if it only has 1 book
+   * TODO: Update filter data
+   * @param {UUIDV4} bookId
+   * @param {UUIDV4[]} seriesIds 
+   */
+  async checkRemoveEmptySeries(bookId, seriesIds) {
+    if (!seriesIds?.length) return
 
-    for (const series of seriesToCheck) {
-      const otherLibraryItemsInSeries = Database.libraryItems.filter(li => li.id !== excludeLibraryItemId && li.isBook && li.media.metadata.hasSeries(series.id))
-      if (!otherLibraryItemsInSeries.length) {
-        // Close open RSS feed for series
-        await this.rssFeedManager.closeFeedForEntityId(series.id)
-        Logger.info(`[ApiRouter] Series "${series.name}" is now empty. Removing series`)
-        await Database.removeSeries(series.id)
-        // TODO: Socket events for series?
+    const bookSeries = await Database.models.bookSeries.findAll({
+      where: {
+        bookId,
+        seriesId: seriesIds
+      },
+      include: [
+        {
+          model: Database.models.series,
+          include: {
+            model: Database.models.book
+          }
+        }
+      ]
+    })
+    for (const bs of bookSeries) {
+      if (bs.series.books.length === 1) {
+        await this.removeEmptySeries(bs.series)
       }
     }
   }