modify to cpu

inference.py

@@ -21,13 +21,16 @@ def load_model(
     Returns:
         Tuple[FastLanguageModel, any]: Tuple containing the model and tokenizer
     """
+    device = torch.device("cpu")
+
     model_name = "lora_model"
 
     tokenizer = AutoTokenizer.from_pretrained(model_name)
 
     model = AutoModelForCausalLM.from_pretrained(
-        pretrained_model_name_or_path=model_name
-    )
+        pretrained_model_name_or_path=model_name,
+        torch_dtype=torch.float32 # Use float32 for CPU
+    ).to(device)
 
     model.eval() # Set model to evaluation mode
 
@@ -80,11 +83,16 @@ def generate_response(
     Returns:
         str: Generated response
     """
+
+    device = torch.device("cpu")
+
     # text_streamer = TextStreamer(tokenizer, skip_prompt=skip_prompt)
-    inputs = tokenizer(inputs, return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
+    inputs = tokenizer(inputs, return_tensors="pt").to(device)
     outputs = model.generate(
         **inputs,
-        max_length=2000
+        max_length=2000,
+        # num_return_sequences=1,
+        # do_sample=False  # Deterministic generation
         # streamer=text_streamer,
         # max_new_tokens=max_new_tokens,
         # use_cache=True,