diff --git a/Makefile b/Makefile
index 53f880e8..424933c5 100644
--- a/Makefile
+++ b/Makefile
@@ -297,10 +297,10 @@ ggml-cuda/%.o: ggml-cuda/%.cu ggml-cuda/%.cuh ggml.h ggml-common.h ggml-cuda/com
 
 ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml.h ggml-backend.h ggml-backend-impl.h ggml-common.h $(wildcard ggml-cuda/*.cuh)
 	$(NVCC) $(NVCCFLAGS) $(CXXFLAGS) -Wno-pedantic -c $< -o $@
-endif
 
 whisper-mel-cuda.o: whisper-mel-cuda.cu whisper.h ggml.h ggml-backend.h whisper-mel.hpp whisper-mel-cuda.hpp
 	$(NVCC) $(NVCCFLAGS) $(CXXFLAGS) -Wno-pedantic -c $< -o $@
+endif
 
 ifdef WHISPER_HIPBLAS
 	ROCM_PATH   ?= /opt/rocm
diff --git a/whisper.cpp b/whisper.cpp
index 457fef9f..a08f15ff 100644
--- a/whisper.cpp
+++ b/whisper.cpp
@@ -3167,7 +3167,7 @@ struct mel_calc_cpu : public whisper_mel_calc {
 }
 
 whisper_mel_calc * whisper_mel_calc_create(ggml_backend_t backend, const whisper_filters & filters) {
-#if GGML_USE_CUDA
+#if defined(GGML_USE_CUDA) && !defined(GGML_USE_HIPBLAS)
     if (ggml_backend_is_cuda(backend)) {
         auto ret = whisper_mel_calc_create_cuda(backend, filters);
         // run a warmup to avoid the first kernel launch overhead (thus we get the best perf even on the first run)