diff --git a/matrix_pygmalion_bot/ai/koboldcpp.py b/matrix_pygmalion_bot/ai/koboldcpp.py index afde2db..54e7f1b 100644 --- a/matrix_pygmalion_bot/ai/koboldcpp.py +++ b/matrix_pygmalion_bot/ai/koboldcpp.py @@ -12,8 +12,8 @@ import io import base64 from PIL import Image, PngImagePlugin -from .pygmalion_helpers import get_full_prompt, num_tokens -#from .llama_helpers import get_full_prompt, num_tokens +#from .pygmalion_helpers import get_full_prompt, num_tokens +from .llama_helpers import get_full_prompt, num_tokens logger = logging.getLogger(__name__) @@ -76,7 +76,7 @@ async def generate_sync( complete_reply += partial_reply tokens += input_data["max_length"] await typing_fn() - if not partial_reply or tokens >= max_new_tokens: + if not partial_reply or tokens >= max_new_tokens +100: # ToDo: is a hundred past the limit okay? complete = True break for t in [f"\nYou:", f"\n### Human:", f"\n{bot.user_name}:", '<|endoftext|>']: diff --git a/matrix_pygmalion_bot/ai/llama_helpers.py b/matrix_pygmalion_bot/ai/llama_helpers.py index 07e5d2a..5fa412e 100644 --- a/matrix_pygmalion_bot/ai/llama_helpers.py +++ b/matrix_pygmalion_bot/ai/llama_helpers.py @@ -31,23 +31,26 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history): #prompt += f"{bot.name} is also very curious and will ask the user a lot of questions about themselves and their life, she will also try to make the user like her.\n" #prompt += f"\n" - prompt = f"Write a script for a dialogue between the user {bot.user_name} and the fictional character {bot.name}.\n" - prompt += bot.persona + "\n" - prompt += "Scenario: " + bot.scenario + "\n" + prompt = f"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n" + prompt += f"### Instruction:\nGiven the following character description and scenario, write a script for a dialogue between the human user {bot.user_name} and the fictional AI assistant {bot.name}. Play the role of the character {bot.name}\n\n" + prompt += "### Input:\n" + prompt += bot.name + "'s Persona: " + bot.persona + "\n" + prompt += "Scenario: " + bot.scenario + "\n\n" + prompt += "### Response:\n" for dialogue_item in bot.example_dialogue: - dialogue_item = dialogue_item.replace('{{user}}', f"{bot.user_name}") - dialogue_item = dialogue_item.replace('{{char}}', bot.name) + dialogue_item = dialogue_item.replace('{{user}}', f"### Human") + dialogue_item = dialogue_item.replace('{{char}}', f"### Assistant") prompt += dialogue_item + "\n\n" prompt += "" + "\n" #prompt += bot.name + ": " + bot.greeting + "\n" - #prompt += f"{bot.user_name}: " + simple_prompt + "\n" - #prompt += f"{bot.name}:" + #prompt += f"### Human: " + simple_prompt + "\n" + #prompt += f"### Assistant:" MAX_TOKENS = 2048 max_new_tokens = 200 total_num_tokens = await num_tokens(prompt) - total_num_tokens += await num_tokens(f"{bot.user_name}: " + simple_prompt + "\n{bot.name}:") + total_num_tokens += await num_tokens(f"### Human: " + simple_prompt + f"\n### Assistant:") visible_history = [] current_message = True for key, chat_item in reversed(chat_history.chat_history.items()): @@ -63,7 +66,7 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history): #if chat_item.message["en"] == bot.greeting: # continue if chat_item.num_tokens == None: - chat_item.num_tokens = await num_tokens("{}: {}".format(chat_item.user_name, chat_item.message["en"])) + chat_item.num_tokens = await num_tokens("{}: {}".format("### Human", chat_item.message["en"])) # TODO: is it MAX_TOKENS or MAX_TOKENS - max_new_tokens?? logger.debug(f"History: " + str(chat_item) + " [" + str(chat_item.num_tokens) + "]") if total_num_tokens + chat_item.num_tokens < MAX_TOKENS - max_new_tokens: @@ -76,15 +79,15 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history): if not hasattr(bot, "greeting_num_tokens"): bot.greeting_num_tokens = await num_tokens(bot.greeting) if total_num_tokens + bot.greeting_num_tokens < MAX_TOKENS - max_new_tokens: - prompt += bot.name + ": " + bot.greeting + "\n" + prompt += "### Assistant: " + bot.greeting + "\n" for chat_item in visible_history: if chat_item.is_own_message: - prompt += bot.name + ": " + chat_item.message["en"] + "\n" + prompt += "### Assistant: " + chat_item.message["en"] + "\n" else: - prompt += f"{bot.user_name}: " + chat_item.message["en"] + "\n" - prompt += f"{bot.user_name}: " + simple_prompt + "\n" - prompt += f"{bot.name}:" + prompt += f"### Human: " + chat_item.message["en"] + "\n" + prompt += f"### Human: " + simple_prompt + "\n" + prompt += f"### Assistant:" return prompt diff --git a/matrix_pygmalion_bot/ai/pygmalion_helpers.py b/matrix_pygmalion_bot/ai/pygmalion_helpers.py index f594358..c6d0d09 100644 --- a/matrix_pygmalion_bot/ai/pygmalion_helpers.py +++ b/matrix_pygmalion_bot/ai/pygmalion_helpers.py @@ -26,7 +26,7 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history): dialogue_item = dialogue_item.replace('{{user}}', 'You') dialogue_item = dialogue_item.replace('{{char}}', bot.name) prompt += dialogue_item + "\n\n" - prompt += "" + "\n" + prompt += "" + "\n" #prompt += bot.name + ": " + bot.greeting + "\n" #prompt += "You: " + simple_prompt + "\n" #prompt += bot.name + ":"