diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 12b2495834..e793114f30 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -741,6 +741,7 @@ class LlamaCppBackend: LlamaCppBackend._codec_mgr.unload() LlamaCppBackend._codec_mgr = None import torch + if torch.cuda.is_available(): torch.cuda.empty_cache() return True diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 2d4e3a609b..b6f82bfe08 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -545,9 +545,12 @@ async def generate_audio( if llama_backend.is_loaded and getattr(llama_backend, "_is_audio", False): model_name = llama_backend.model_identifier gen = lambda: llama_backend.generate_audio_response( - text = text, audio_type = llama_backend._audio_type, - temperature = payload.temperature, top_p = payload.top_p, - top_k = payload.top_k, min_p = payload.min_p, + text = text, + audio_type = llama_backend._audio_type, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = payload.top_k, + min_p = payload.min_p, max_new_tokens = payload.max_tokens or 2048, repetition_penalty = payload.repetition_penalty, )