From ef7a60e2d2b4dad4a9c2af7b03a5ff41f9ca31de Mon Sep 17 00:00:00 2001 From: Hendrik Langer Date: Mon, 17 Apr 2023 00:43:10 +0200 Subject: [PATCH] remove chunk window for gpu services --- matrix_pygmalion_bot/ai/model_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/matrix_pygmalion_bot/ai/model_helpers.py b/matrix_pygmalion_bot/ai/model_helpers.py index 3810355..5b9e2d0 100644 --- a/matrix_pygmalion_bot/ai/model_helpers.py +++ b/matrix_pygmalion_bot/ai/model_helpers.py @@ -136,7 +136,10 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history, model_name: str #prompt += f"{ai_name}:" MAX_TOKENS = 2048 - WINDOW = 600 + if bot.service_text == "koboldcpp": + WINDOW = 600 + else: + WINDOW = 0 max_new_tokens = 200 total_num_tokens = await num_tokens(prompt, model_name) input_num_tokens = await num_tokens(f"{user_name}: {simple_prompt}\n{ai_name}:", model_name)