Update app.py

app.py

@@ -1,30 +1,60 @@
-import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
-from peft import PeftModel, PeftConfig
-
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(".")
-
-# Load base model with quantization
-bnb_config = BitsAndBytesConfig(load_in_4bit=True)
-base_model = AutoModelForCausalLM.from_pretrained(
-    "unsloth/Meta-Llama-3.1-8B-bnb-4bit",  # same base you fine-tuned
-    quantization_config=bnb_config,
-    device_map="auto"
-)
-
-# Load LoRA adapters
-model = PeftModel.from_pretrained(base_model, ".")
-
-# Create Gradio Interface
-def generate_response(prompt):
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(**inputs, max_new_tokens=200, do_sample=True, temperature=0.7)
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
-
-gr.Interface(
-    fn=generate_response,
-    inputs=gr.Textbox(label="Enter your instruction"),
-    outputs=gr.Textbox(label="Model response"),
-    title="LLaMA 3 - Fine-tuned Model"
-).launch()
+# import gradio as gr
+# from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+# from peft import PeftModel, PeftConfig
+
+# # Load tokenizer
+# tokenizer = AutoTokenizer.from_pretrained(".")
+
+# # Load base model with quantization
+# bnb_config = BitsAndBytesConfig(load_in_4bit=True)
+# base_model = AutoModelForCausalLM.from_pretrained(
+#     "unsloth/Meta-Llama-3.1-8B-bnb-4bit",  # same base you fine-tuned
+#     quantization_config=bnb_config,
+#     device_map="auto"
+# )
+
+# # Load LoRA adapters
+# model = PeftModel.from_pretrained(base_model, ".")
+
+# # Create Gradio Interface
+# def generate_response(prompt):
+#     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+#     outputs = model.generate(**inputs, max_new_tokens=200, do_sample=True, temperature=0.7)
+#     return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+# gr.Interface(
+#     fn=generate_response,
+#     inputs=gr.Textbox(label="Enter your instruction"),
+#     outputs=gr.Textbox(label="Model response"),
+#     title="LLaMA 3 - Fine-tuned Model"
+# ).launch()
+
+
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(".")
+
+# Load base model normally (no 4-bit quantization)
+base_model = AutoModelForCausalLM.from_pretrained(
+    "unsloth/Meta-Llama-3.1-8B-bnb-4bit",  # same base you fine-tuned
+    device_map="auto"
+)
+
+# Load LoRA adapters
+model = PeftModel.from_pretrained(base_model, ".")
+
+# Create Gradio Interface
+def generate_response(prompt):
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    outputs = model.generate(**inputs, max_new_tokens=200, do_sample=True, temperature=0.7)
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+gr.Interface(
+    fn=generate_response,
+    inputs=gr.Textbox(label="Enter your instruction"),
+    outputs=gr.Textbox(label="Model response"),
+    title="LLaMA 3 - Fine-tuned Model"
+).launch()