diff --git a/ui/media/js/image-modifiers.js b/ui/media/js/image-modifiers.js
index 24347fc4..8ecb6a26 100644
--- a/ui/media/js/image-modifiers.js
+++ b/ui/media/js/image-modifiers.js
@@ -87,9 +87,9 @@ function createModifierGroup(modifierGroup, initiallyExpanded) {
             modifiersEl.appendChild(modifierCard)
 
             modifierCard.addEventListener('click', () => {
-                if (activeTags.map(x => x.name).includes(modifierName)) {
+                if (activeTags.map(x => trimModifiers(x.name)).includes(trimModifiers(modifierName))) {
                     // remove modifier from active array
-                    activeTags = activeTags.filter(x => x.name != modifierName)
+                    activeTags = activeTags.filter(x => trimModifiers(x.name) != trimModifiers(modifierName))
                     modifierCard.classList.remove(activeCardClass)
 
                     modifierCard.querySelector('.modifier-card-image-overlay').innerText = '+'
@@ -125,6 +125,10 @@ function createModifierGroup(modifierGroup, initiallyExpanded) {
     return e
 }
 
+function trimModifiers(tag) {
+    return tag.replace(/^\(+|\)+$/g, '').replace(/^\[+|\]+$/g, '')
+}
+
 async function loadModifiers() {
     try {
         let res = await fetch('/get/modifiers')