open llm

app.py

@@ -3,12 +3,14 @@
 import gradio as gr
 import pandas as pd
 from datetime import datetime, timedelta, timezone
-from config import groq_token, groq_model, QUESTION_PROMPT, init_google_sheets_client, groq_model, default_model_name, user_names, google_sheets_name, AtlasClient
+from config import groq_token, groq_model, QUESTION_PROMPT, init_google_sheets_client, groq_model, default_model_name, user_names, google_sheets_name, AtlasClient, custom_model
 import gspread
 from  groq import Client
 import random, string, json, io
-
+#from trash_folder.alter_app import Local_llm
+from import_model import Local_llm
 import groq
+import torch
 print(groq.__version__)
 # Initialize Google Sheets client
 client = init_google_sheets_client()
@@ -18,7 +20,13 @@ stories_sheet = sheet.worksheet("Stories")
 system_prompts_sheet = sheet.worksheet("System Prompts")
 
 # Combine both model dictionaries
-all_models = {**groq_model}
+all_models = {}
+all_models.update(groq_model)
+if torch.cuda.is_available():
+    all_models.update(custom_model)
+
+#init local modeel as None
+local_model = None
 
 def randomize_key_order(aux):
     keys  = list(aux.keys())
@@ -131,6 +139,24 @@ def save_comment_score(score, comment, story_name, user_name, system_prompt, mod
 
 
 
+
+
+from openai import OpenAI
+client = OpenAI(
+  base_url="https://openrouter.ai/api/v1",
+  api_key="$OPENROUTER_API_KEY",
+)
+def interact_openrouter(context, model_name):
+    completion = client.chat.completions.create(
+      model=model_name,
+      messages=context,
+    )
+    return completion.choices[0].message.content
+
+
+
+
+
 # Function to handle interaction with model
 def interact_groq(context, model_name):
    chat_completion = groq_clinet.chat.completions.create(
@@ -147,6 +173,7 @@ def interact_groq(context, model_name):
 #o=[chatbot_output, chat_history_json, data_table, selected_story_textbox])
 # Function to send selected story and initial message
 def send_selected_story(title, model_name, system_prompt):
+   global local_model
    global chat_history
    global selected_story
    global data  # Ensure data is reset
@@ -163,13 +190,21 @@ Here is the story:
            """
            combined_message = system_prompt.strip()
            if combined_message:
-               chat_history = []  # Reset chat history
-               chat_history.append({"role": "system", "content": combined_message})
-               chat_history.append({"role": "user", "content": QUESTION_PROMPT})
-
-               response = interact_groq(chat_history, model_name)
-               resp = {"role": "assistant", "content": response.strip()}
-               return resp, chat_history, story["story"]
+                chat_history = []  # Reset chat history
+                chat_history.append({"role": "system", "content": combined_message})
+                chat_history.append({"role": "user", "content": QUESTION_PROMPT})
+                if model_name in custom_model:
+                    if local_model is None or local_model.model_name != custom_model[model_name]:
+                        #si hay que cambiar el modelo o levantarlo
+                        del local_model
+                        torch.cuda.empty_cache()
+                        torch.cuda.synchronize() #ver si funciona este
+                        local_model = Local_llm(custom_model[model_name])
+                    response = local_model.interact(chat_history)
+                else:
+                    response = interact_groq(chat_history, model_name)
+                resp = {"role": "assistant", "content": response.strip()}
+                return resp, chat_history, story["story"]
            else:
                print("Combined message is empty.")
        else:
@@ -276,7 +311,10 @@ def multiple_interact(query, models, selected_model, assistant_user_input): #, i
     #no es models es....
     random.shuffle(active_models)
     for index, model in enumerate(active_models):
-        resp = interact_groq(aux_history, model)
+        if model in custom_model:
+            resp = local_model.interact(aux_history)
+        else:
+            resp = interact_groq(aux_history, model)
         resp = {"role": "assistant", "content": resp.strip()}
         chatbot_answser_list[alphabet[index]] = {'response': resp, 'model': model}
         try:
@@ -451,4 +489,5 @@ with gr.Blocks() as demo:
         inputs=[score_input, comment_input, story_dropdown, user_dropdown, system_prompt_dropdown,  model_checkbox],
         outputs=[data_table, comment_input])
 
-demo.launch()
+demo.launch(share=True)
+#demo.launch(share=True)

config.py

@@ -56,6 +56,15 @@ groq_model = {
     "llama3-70b-8192": "llama3-70b-8192",
 }
 
+custom_model = {
+    "rodrisouza/Llama-3-8B-Finetuning-Stories":  "rodrisouza/Llama-3-8B-Finetuning-Stories"
+}
+
+openai_model = {
+    "meta-llama/llama-3.1-70b-instruct:free": "meta-llama/llama-3.1-70b-instruct:free",
+    "meta-llama/llama-3.1-8b-instruct:free": "meta-llama/llama-3.1-8b-instruct:free",
+}
+
 
 # Default model (first in list)
 default_model_name = list(replicate_model.items())[0][0]

import_model.py

@@ -0,0 +1,32 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import torch
+
+class ModelLoader:
+    def __init__(self, model_name, hugging_face_token):
+        self.model_name = model_name
+        # Configure 4-bit quantization
+        self.bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16,
+            llm_int8_enable_fp32_cpu_offload=True
+        )
+
+        # Load tokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_name, 
+            token=hugging_face_token
+        )
+
+        # Load model with memory optimizations
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            quantization_config=self.bnb_config,
+            device_map="auto",
+            low_cpu_mem_usage=True,
+            max_memory={
+                "cpu": "12GiB",
+                "cuda:0": "4GiB",
+            },
+            token=hugging_face_token
+)

requirements.txt

@@ -1,14 +1,15 @@
 huggingface_hub==0.25.1
 minijinja
-#transformers
-#torch
+transformers
+torch
 pandas
 gspread
 oauth2client
-#accelerate
-#bitsandbytes
+accelerate
+bitsandbytes
 replicate
 groq==0.11.0
 gradio
 google-api-python-client
-pymongo==4.6.2
+pymongo==4.6.2
+openai