feat: new gpt4o usage

core-model-prediction/prediction.py

@@ -2,7 +2,7 @@ from fastapi import FastAPI, Response, status
 from pydantic import BaseModel
 from hypothesis import BaseModelHypothesis
 from secondary_model_dependencies import SecondaryModelDependencies
-from random_forest_model import RandomForestModel
+from secondary_model import SecondaryModel
 from main_model import PredictMainModel
 import numpy as np
 from typing import List
@@ -17,7 +17,7 @@ class PredictRequest(BaseModel):
     typing_duration: int
     letter_click_counts: dict[str, int]
     gpt35_answer: str
-    gpt4_answer: str
+    gpt4o_answer: str
 
 
 class RequestModel(BaseModel):
@@ -42,7 +42,7 @@ def process_instance(data: PredictRequest):
     typing_duration = data.typing_duration
     letter_click_counts = data.letter_click_counts
     gpt35_answer = data.gpt35_answer
-    gpt4_answer = data.gpt4_answer
+    gpt4o_answer = data.gpt4o_answer
 
     # Data preparation for 1st model
     hypothesis = BaseModelHypothesis()
@@ -56,26 +56,27 @@ def process_instance(data: PredictRequest):
     # Data preparation for 2nd model
     secondary_model_dependencies = SecondaryModelDependencies()
     secondary_model_features = secondary_model_dependencies.calculate_features(
-        question, answer, main_model_probability, backspace_count, typing_duration,
-        letter_click_counts, gpt35_answer, gpt4_answer)
+        answer, main_model_probability, backspace_count, typing_duration,
+        letter_click_counts, gpt35_answer, gpt4o_answer)
 
     # 2nd model prediction
-    secondary_model = RandomForestModel()
-    secondary_model_prediction = secondary_model.predict(
+    secondary_model = SecondaryModel()
+    secondary_model_probability = secondary_model.predict(
         secondary_model_features)
 
     return {
-        "predicted_class": "AI" if secondary_model_prediction == 1 else "HUMAN",
+        "predicted_class": "AI" if secondary_model_probability > 0.57 else "HUMAN",
         "main_model_probability": str(main_model_probability),
-        "secondary_model_prediction": secondary_model_prediction,
-        "confidence": get_confidence(main_model_probability, secondary_model_prediction)
+        "secondary_model_probability": secondary_model_probability,
+        "confidence": get_confidence(main_model_probability, secondary_model_probability)
     }
 
 
 def get_confidence(main_model_output: float, secondary_model_output: int):
-    if (main_model_output >= 0.8 and secondary_model_output == 1) or (main_model_output <= 0.2 and secondary_model_output == 0):
+    threshold = 0.57
+    if (main_model_output >= 0.8 and secondary_model_output >= threshold) or (main_model_output <= 0.2 and secondary_model_output <= 1 - threshold):
         return 'High Confidence'
-    elif (0.5 < main_model_output < 0.8 and secondary_model_output == 1) or (0.2 < main_model_output <= 0.5 and secondary_model_output == 0):
+    elif (0.5 < main_model_output < 0.8 and secondary_model_output >= threshold) or (0.2 < main_model_output <= 0.5 and secondary_model_output < threshold):
         return 'Partially Confident'
     else:
         return 'Low Confidence'

core-model-prediction/{random_forest_model.py → secondary_model.py}

@@ -4,13 +4,13 @@ import pandas as pd
 from typing import List
 
 
-class RandomForestModel:
+class SecondaryModel:
     def __init__(self):
         self.scaler = joblib.load("scalers/secondary_scaler.joblib")
-        self.model = joblib.load("models/rf_weights.joblib")
+        self.model = joblib.load("models/secondary_weights.joblib")
         self.secondary_model_features = [
             "machine_probability", "backspace_count_normalized", "typing_duration_normalized",
-            "letter_discrepancy_normalized", "cosine_sim_gpt35", "cosine_sim_gpt4"
+            "letter_discrepancy_normalized", "cosine_sim_gpt35", "cosine_sim_gpt4o"
         ]
 
     def preprocess_input(self, secondary_model_features: List[float]) -> pd.DataFrame:

core-model-prediction/secondary_model_dependencies.py

@@ -7,8 +7,8 @@ class SecondaryModelDependencies:
         self.text_similarity_model = SentenceTransformer(
             'sentence-transformers/all-mpnet-base-v2')
 
-    def calculate_features(self, question: str, answer: str, probability: float, backspace_count: int, typing_duration: int,
-                           letter_click_counts: dict[str, int], gpt35_answer: str, gpt4_answer: str):
+    def calculate_features(self, answer: str, probability: float, backspace_count: int, typing_duration: int,
+                           letter_click_counts: dict[str, int], gpt35_answer: str, gpt4o_answer: str):
         backspace_count_normalized = backspace_count / len(answer)
         typing_duration_normalized = typing_duration / len(answer)
         letter_discrepancy = self.calculate_letter_discrepancy(
@@ -16,11 +16,12 @@ class SecondaryModelDependencies:
 
         cosine_sim_gpt35 = self.calculate_similarity_gpt35(
             answer, gpt35_answer)
-        cosine_sim_gpt4 = self.calculate_similarity_gpt4(answer, gpt4_answer)
+        cosine_sim_gpt4o = self.calculate_similarity_gpt4o(
+            answer, gpt4o_answer)
 
         return [
             probability, backspace_count_normalized, typing_duration_normalized,
-            letter_discrepancy, cosine_sim_gpt35, cosine_sim_gpt4
+            letter_discrepancy, cosine_sim_gpt35, cosine_sim_gpt4o
         ]
 
     def calculate_letter_discrepancy(self, text: str, letter_click_counts: dict[str, int]):
@@ -46,10 +47,10 @@ class SecondaryModelDependencies:
         cosine_scores = util.cos_sim(embedding1, embedding2)
         return cosine_scores.item()
 
-    def calculate_similarity_gpt4(self, answer: str, gpt4_answer: str) -> float:
+    def calculate_similarity_gpt4o(self, answer: str, gpt4o_answer: str) -> float:
         embedding1 = self.text_similarity_model.encode(
             [answer], convert_to_tensor=True)
         embedding2 = self.text_similarity_model.encode(
-            [gpt4_answer], convert_to_tensor=True)
+            [gpt4o_answer], convert_to_tensor=True)
         cosine_scores = util.cos_sim(embedding1, embedding2)
         return cosine_scores.item()