llama prompt try 1

matrix_pygmalion_bot/ai/koboldcpp.py

@@ -12,8 +12,8 @@ import io
 import base64
 from PIL import Image, PngImagePlugin
 
-from .pygmalion_helpers import get_full_prompt, num_tokens
-#from .llama_helpers import get_full_prompt, num_tokens
+#from .pygmalion_helpers import get_full_prompt, num_tokens
+from .llama_helpers import get_full_prompt, num_tokens
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +76,7 @@ async def generate_sync(
             complete_reply += partial_reply
             tokens += input_data["max_length"]
             await typing_fn()
-            if not partial_reply or tokens >= max_new_tokens:
+            if not partial_reply or tokens >= max_new_tokens +100: # ToDo: is a hundred past the limit okay?
                 complete = True
                 break
             for t in [f"\nYou:", f"\n### Human:", f"\n{bot.user_name}:", '<|endoftext|>']:

matrix_pygmalion_bot/ai/llama_helpers.py

@@ -31,23 +31,26 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history):
     #prompt += f"{bot.name} is also very curious and will ask the user a lot of questions about themselves and their life, she will also try to make the user like her.\n"
     #prompt += f"\n"
 
-    prompt = f"Write a script for a dialogue between the user {bot.user_name} and the fictional character {bot.name}.\n"
-    prompt += bot.persona + "\n"
-    prompt += "Scenario: " + bot.scenario + "\n"
+    prompt = f"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n"
+    prompt += f"### Instruction:\nGiven the following character description and scenario, write a script for a dialogue between the human user {bot.user_name} and the fictional AI assistant {bot.name}. Play the role of the character {bot.name}\n\n"
+    prompt += "### Input:\n"
+    prompt += bot.name + "'s Persona: " + bot.persona + "\n"
+    prompt += "Scenario: " + bot.scenario + "\n\n"
+    prompt += "### Response:\n"
 
     for dialogue_item in bot.example_dialogue:
-        dialogue_item = dialogue_item.replace('{{user}}', f"{bot.user_name}")
-        dialogue_item = dialogue_item.replace('{{char}}', bot.name)
+        dialogue_item = dialogue_item.replace('{{user}}', f"### Human")
+        dialogue_item = dialogue_item.replace('{{char}}', f"### Assistant")
         prompt += dialogue_item + "\n\n"
     prompt += "<START>" + "\n"
     #prompt += bot.name + ": " + bot.greeting + "\n"
-    #prompt += f"{bot.user_name}: " + simple_prompt + "\n"
-    #prompt += f"{bot.name}:"
+    #prompt += f"### Human: " + simple_prompt + "\n"
+    #prompt += f"### Assistant:"
 
     MAX_TOKENS = 2048
     max_new_tokens = 200
     total_num_tokens = await num_tokens(prompt)
-    total_num_tokens += await num_tokens(f"{bot.user_name}: " + simple_prompt + "\n{bot.name}:")
+    total_num_tokens += await num_tokens(f"### Human: " + simple_prompt + f"\n### Assistant:")
     visible_history = []
     current_message = True
     for key, chat_item in reversed(chat_history.chat_history.items()):
@@ -63,7 +66,7 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history):
         #if chat_item.message["en"] == bot.greeting:
         #    continue
         if chat_item.num_tokens == None:
-            chat_item.num_tokens = await num_tokens("{}: {}".format(chat_item.user_name, chat_item.message["en"]))
+            chat_item.num_tokens = await num_tokens("{}: {}".format("### Human", chat_item.message["en"]))
         # TODO: is it MAX_TOKENS or MAX_TOKENS - max_new_tokens??
         logger.debug(f"History: " + str(chat_item) + " [" + str(chat_item.num_tokens) + "]")
         if total_num_tokens + chat_item.num_tokens < MAX_TOKENS - max_new_tokens:
@@ -76,15 +79,15 @@ async def get_full_prompt(simple_prompt: str, bot, chat_history):
     if not hasattr(bot, "greeting_num_tokens"):
         bot.greeting_num_tokens = await num_tokens(bot.greeting)
     if total_num_tokens + bot.greeting_num_tokens < MAX_TOKENS - max_new_tokens:
-        prompt += bot.name + ": " + bot.greeting + "\n"
+        prompt += "### Assistant: " + bot.greeting + "\n"
 
     for chat_item in visible_history:
         if chat_item.is_own_message:
-            prompt += bot.name + ": " + chat_item.message["en"] + "\n"
+            prompt += "### Assistant: " + chat_item.message["en"] + "\n"
         else:
-            prompt += f"{bot.user_name}: " + chat_item.message["en"] + "\n"
-    prompt += f"{bot.user_name}: " + simple_prompt + "\n"
-    prompt += f"{bot.name}:"
+            prompt += f"### Human: " + chat_item.message["en"] + "\n"
+    prompt += f"### Human: " + simple_prompt + "\n"
+    prompt += f"### Assistant:"
 
     return prompt