version 1 metric

nl2bash_m.py

@@ -94,38 +94,66 @@ class nl2bash_m(evaluate.Metric):
             reference_urls=[],
         )
 
+    def get_score(self, pred, ref):
+        if not pred and not ref: return 1
+        cor = 0
+        for i in range(min(len(pred), len(ref))):
+            if (pred[i] == ref[i]):
+                cor += 1
+        
+        return cor/max(len(pred), len(ref))
+
     def _compute(
         self,
         predictions,
-        references,
-        regexes_to_ignore=None,
+        references, 
+        cmd_weight = 0.65,
+        opt_weight = 0.25,
+        arg_weight = 0.15,
         ignore_case=False,
-        ignore_punctuation=False,
         ignore_numbers=False,
     ):
 
-        if regexes_to_ignore is not None:
-            for s in regexes_to_ignore:
-                predictions = np.array([re.sub(s, "", x) for x in predictions])
-                references = np.array([re.sub(s, "", x) for x in references])
-        else:
-            predictions = np.asarray(predictions)
-            references = np.asarray(references)
+        predictions = np.asarray(predictions)
+        references = np.asarray(references)
 
         if ignore_case:
             predictions = np.char.lower(predictions)
             references = np.char.lower(references)
 
-        if ignore_punctuation:
-            repl_table = string.punctuation.maketrans("", "", string.punctuation)
-            predictions = np.char.translate(predictions, table=repl_table)
-            references = np.char.translate(references, table=repl_table)
-
         if ignore_numbers:
             repl_table = string.digits.maketrans("", "", string.digits)
             predictions = np.char.translate(predictions, table=repl_table)
             references = np.char.translate(references, table=repl_table)
 
-        score_list = predictions == references
 
-        return {"exact_match": np.mean(score_list)}
+        final_score = 0
+
+        for pred, ref in zip(predictions, references):
+            print(pred, ref)
+            pred_words, ref_words = pred[0].split(), ref[0].split()
+            # Get the cmd of predicted and ref 
+            cmd_corr = 1 if pred_words.pop(0)==ref_words.pop(0) else 0
+
+            # Get the option of predicted and ref
+            pred_option = [ x for x in pred_words if x[0] == '-']
+            ref_option = [ x for x in ref_words if x[0] == '-']
+            
+            # Get the arguments of predicted and ref
+            pred_args = [ x for x in pred_words if x[0] != '-']
+            ref_args = [ x for x in ref_words if x[0] != '-']
+
+            # Calculate scores
+            cmd_score = cmd_weight * cmd_corr
+            opt_score = opt_weight * self.get_score(pred_option, ref_option)
+            arg_score = arg_weight * self.get_score(pred_args, ref_args)
+
+            score = cmd_score + opt_score + arg_score
+            final_score += score
+            print(score)
+
+        final_score = final_score/len(self.preds)
+        print("f_s: ", final_score)
+ 
+
+        return {"nl2bash_m": (final_score)}