add parallization

app.py

@@ -12,6 +12,7 @@ import time
 import torch
 import traceback
 import logging
+import asyncio
 
 # Import local modules
 from classifiers import TFIDFClassifier, LLMClassifier
@@ -106,15 +107,11 @@ def process_file(file, text_columns, categories, classifier_type, show_explanati
         if classifier_type == "tfidf":
             classifier = TFIDFClassifier()
             results = classifier.classify(texts, category_list)
-        elif classifier_type == "gpt35":
+        elif classifier_type in ["gpt35", "gpt4"]:
             if client is None:
                 return None, "Erreur : Le client API n'est pas initialisé. Veuillez configurer une clé API valide dans l'onglet 'Setup'."
-            classifier = LLMClassifier(client=client, model="gpt-3.5-turbo")
-            results = classifier.classify(texts, category_list)
-        elif classifier_type == "gpt4":
-            if client is None:
-                return None, "Erreur : Le client API n'est pas initialisé. Veuillez configurer une clé API valide dans l'onglet 'Setup'."
-            classifier = LLMClassifier(client=client, model="gpt-4")
+            model = "gpt-3.5-turbo" if classifier_type == "gpt35" else "gpt-4"
+            classifier = LLMClassifier(client=client, model=model)
             results = classifier.classify(texts, category_list)
         else:  # hybrid
             if client is None:
@@ -126,12 +123,21 @@ def process_file(file, text_columns, categories, classifier_type, show_explanati
             # Second pass with LLM for low confidence results
             llm_classifier = LLMClassifier(client=client, model="gpt-3.5-turbo")
             results = []
+            low_confidence_texts = []
+            low_confidence_indices = []
+            
             for i, (text, tfidf_result) in enumerate(zip(texts, tfidf_results)):
                 if tfidf_result["confidence"] < 70:  # If confidence is below 70%
-                    llm_result = llm_classifier.classify([text], category_list)[0]
-                    results.append(llm_result)
+                    low_confidence_texts.append(text)
+                    low_confidence_indices.append(i)
+                    results.append(None)  # Placeholder
                 else:
                     results.append(tfidf_result)
+            
+            if low_confidence_texts:
+                llm_results = llm_classifier.classify(low_confidence_texts, category_list)
+                for idx, llm_result in zip(low_confidence_indices, llm_results):
+                    results[idx] = llm_result
         
         # Create results dataframe
         result_df = df.copy()
@@ -364,7 +370,7 @@ with gr.Blocks(title="Text Classification System") as demo:
         def show_results(df, validation_report):
             """Show the results after processing"""
             if df is None:
-                return gr.Row(visible=False), gr.File(visible=False), gr.File(visible=False), gr.Dataframe(visible=False), gr.Dataframe(visible=False)
+                return gr.Row(visible=False), gr.File(visible=False), gr.File(visible=False), gr.Dataframe(visible=False)
             
             # Sort by category if it exists
             if "Category" in df.columns:
@@ -374,7 +380,7 @@ with gr.Blocks(title="Text Classification System") as demo:
             csv_path = export_results(df, "csv")
             excel_path = export_results(df, "excel")
             
-            return gr.Row(visible=True), gr.File(value=csv_path, visible=True), gr.File(value=excel_path, visible=True), gr.Dataframe(value=df, visible=True), gr.Dataframe(value=df, visible=True)
+            return gr.Row(visible=True), gr.File(value=csv_path, visible=True), gr.File(value=excel_path, visible=True), gr.Dataframe(value=df, visible=True)
         
         # Function to suggest a new category
         def suggest_new_category(file, current_categories, text_columns):

classifiers.py

@@ -5,6 +5,8 @@ from sklearn.cluster import KMeans
 from sklearn.metrics.pairwise import cosine_similarity
 import random
 import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Optional
 
 class BaseClassifier:
     """Base class for text classifiers"""
@@ -143,21 +145,37 @@ class LLMClassifier(BaseClassifier):
         self.client = client
         self.model = model
     
-    def classify(self, texts, categories=None):
-        """Classify texts using an LLM"""
+    def classify(self, texts: List[str], categories: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        """Classify texts using an LLM with parallel processing"""
         if not categories:
             # First, use LLM to generate appropriate categories
             categories = self._suggest_categories(texts)
         
-        results = []
-        for text in texts:
-            # Classify each text individually
-            result = self._classify_text(text, categories)
-            results.append(result)
+        # Process texts in parallel
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            # Submit all tasks
+            future_to_text = {
+                executor.submit(self._classify_text, text, categories): text 
+                for text in texts
+            }
+            
+            # Collect results as they complete
+            results = []
+            for future in as_completed(future_to_text):
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    print(f"Error processing text: {str(e)}")
+                    results.append({
+                        "category": categories[0],
+                        "confidence": 50,
+                        "explanation": f"Error during classification: {str(e)}"
+                    })
         
         return results
     
-    def _suggest_categories(self, texts, sample_size=20):
+    def _suggest_categories(self, texts: List[str], sample_size: int = 20) -> List[str]:
         """Use LLM to suggest appropriate categories for the dataset"""
         # Take a sample of texts to avoid token limitations
         if len(texts) > sample_size:
@@ -192,7 +210,7 @@ class LLMClassifier(BaseClassifier):
             print(f"Error suggesting categories: {str(e)}")
             return self._generate_default_categories(texts)
     
-    def _classify_text(self, text, categories):
+    def _classify_text(self, text: str, categories: List[str]) -> Dict[str, Any]:
         """Use LLM to classify a single text"""
         categories_str = ", ".join(categories)
         

utils.py

@@ -64,6 +64,16 @@ def visualize_results(df, text_column, category_column="Category"):
     Returns:
         matplotlib.figure.Figure: Visualization figure
     """
+    # Check if category column exists
+    if category_column not in df.columns:
+        # Create a simple figure with a message
+        fig, ax = plt.subplots(figsize=(10, 6))
+        ax.text(0.5, 0.5, "No categories to display", 
+                ha='center', va='center', fontsize=12)
+        ax.set_title('No Classification Results Available')
+        plt.tight_layout()
+        return fig
+    
     # Get categories and their counts
     category_counts = df[category_column].value_counts()