Update app.py

app.py

@@ -1,52 +1,57 @@
 import gradio as gr
-from mlx_lm import load, generate
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
 # ----------------------------------------------------
-# 1. Load your quantized MLX model from HuggingFace
+# 1. Download GGUF model from HuggingFace
 # ----------------------------------------------------
-MODEL_REPO = "astegaras/my-mlx-llama3"   # <-- change to your repo
 
-print("Loading model...")
-model, tokenizer = load(MODEL_REPO)
-print("Model loaded!")
+REPO_ID = "astegaras/merged_kaggle"   # your GGUF repo
+FILENAME = "model-q4_k_m.gguf"        # your GGUF file
+
+print("Downloading GGUF model...")
+model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
+print("Model downloaded:", model_path)
 
 # ----------------------------------------------------
-# 2. Chat / inference function
+# 2. Load llama.cpp model
 # ----------------------------------------------------
-def respond(user_input, history):
-    """
-    user_input: new user message
-    history: list of [user, assistant] messages from Gradio
-    """
 
-    # Build a conversation prompt (simple version)
-    messages = []
-    for user_msg, assistant_msg in history:
-        messages.append(f"User: {user_msg}\nAssistant: {assistant_msg}")
-    messages.append(f"User: {user_input}\nAssistant:")
+llm = Llama(
+    model_path=model_path,
+    n_ctx=4096,           # context size
+    n_threads=8,          # use HF Space CPU
+)
+
+# ----------------------------------------------------
+# 3. Chat / inference function
+# ----------------------------------------------------
+def respond(message, history):
+    prompt = ""
 
-    prompt = "\n".join(messages)
+    # Build prompt manually
+    for user_msg, bot_msg in history:
+        prompt += f"User: {user_msg}\nAssistant: {bot_msg}\n"
+    prompt += f"User: {message}\nAssistant:"
 
-    # Generate with mlx_lm
-    output = generate(
-        model,
-        tokenizer,
+    # Generate response
+    output = llm(
         prompt,
         max_tokens=256,
         temperature=0.7,
         top_p=0.9,
+        stop=["User:", "Assistant:"]
     )
 
-    # Extract only the assistant's new text
-    assistant_reply = output[len(prompt):].strip()
-
+    assistant_reply = output["choices"][0]["text"].strip()
     return assistant_reply
 
 # ----------------------------------------------------
-# 3. Launch Gradio chat interface
+# 4. Launch Gradio Chat Interface
 # ----------------------------------------------------
 gr.ChatInterface(
     fn=respond,
-    title="My MLX Llama Model",
-    description="Chat with your fine-tuned MLX model!",
+    title="My Llama.cpp GGUF Model",
+    description="Chat with your fine-tuned GGUF model!",
 ).launch()
+