try flash

app.py

@@ -1,3 +1,10 @@
+import subprocess  # 🥲
+
+subprocess.run(
+    "pip install flash-attn --no-build-isolation",
+    env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
+    shell=True,
+)
 import spaces
 import gradio as gr
 
@@ -11,6 +18,7 @@ from typing import Tuple
 
 os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
 
+
 model = Qwen2VLForConditionalGeneration.from_pretrained(
     "Qwen/Qwen2-VL-7B-Instruct",
     torch_dtype=torch.bfloat16,
@@ -93,14 +101,13 @@ def _prep_data_for_input(image):
 
     image_inputs, video_inputs = process_vision_info(messages)
 
-    inputs = processor(
+    return processor(
         text=[text],
         images=image_inputs,
         videos=video_inputs,
         padding=True,
         return_tensors="pt",
     )
-    return inputs
 
 
 @spaces.GPU
@@ -120,8 +127,7 @@ def generate_response(image):
         clean_up_tokenization_spaces=False,
     )
     try:
-        data = json.loads(output_text[0])
-        return data
+        return json.loads(output_text[0])
     except Exception:
         return {}