omit eval long texts columns (comments)

bangalore_score.py

@@ -85,32 +85,32 @@ class Bangalore_Score(evaluate.Metric):
 
     def _download_and_prepare(self, dl_manager):
         """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
         import nltk
         nltk.download('punkt')
 
+        import evaluate
         import pandas as pd
-        from datasets import Dataset, DatasetDict
+        from datasets import Dataset
 
+        ### metrics ###
         # https://huggingface.co/evaluate-metric
-        import evaluate
-        evaluate.load('evaluate-metric/meteor')
-        # evaluate.load('evaluate-metric/meteor')
-        # evaluate.load('evaluate-metric/wer')
-        # evaluate.load('evaluate-metric/exact_match')
-        # evaluate.load('evaluate-metric/character')
-        # evaluate.load('evaluate-metric/ter')
-        # evaluate.load('bleu')
-        # evaluate.load('rouge')
+        ###############
+        # evaluate.load('evaluate-metric/meteor')       # respect order (machine translation)
+        # evaluate.load('evaluate-metric/wer')          # word error rate
+        # evaluate.load('evaluate-metric/exact_match')  # exact match
+        # evaluate.load('evaluate-metric/character')    # character error rate
+        # evaluate.load('evaluate-metric/ter')          # translation error rate
+        # evaluate.load('bleu')                         # no respect order (machine translation)
+        # evaluate.load('rouge')                        # no respect order (machine translation)
         # evaluate.load('ncoop57/levenshtein_distance')
         pass
 
-
     def normalize_fn(
             self,
             example,
             text_field='text',
-            unk_token='Not Disclosed'
+            unk_token='Not Disclosed',
+            return_df=False
     ):
         """
         parse output text into headers, rows, and records
@@ -126,30 +126,38 @@ class Bangalore_Score(evaluate.Metric):
         | ... | ... | ... |
         """
         import pandas as pd
-        headers_text, records, rows_text = unk_token, [], unk_token
-        rows = dict(example)[text_field].strip().split('\n')
 
-        # parse headers
+        records = []
+        rows_text = unk_token
+        omit_columns = example.get('omit_columns', [])
+
+        text = dict(example)[text_field]
+        text = text.strip()
+        rows = text.split('\n')
+
+        # headers
         if len(rows) > 0:
-            # trailing pipes
-            headers_row = rows[0].strip().strip('|')
-            # split on pipes and remove whitespace
-            headers_list = [x.strip() for x in headers_row.split('|')]
-            headers_text = ' '.join(sorted(headers_list))                   # join headers
+            headers_row = rows[0]
+            headers_row = headers_row.strip()
+            headers_row = headers_row.strip('|')
+            headers_list = headers_row.split('|')
+            headers_list = [c.strip() for c in headers_list]
 
-        # try parse records
+        # records / rows
         if len(rows) > 2:
-            data_rows = rows[2:]
-            # trailing pipes
-            data_rows = [x.strip('|').split('|') for x in data_rows]
+            data_rows = []
+            for row_text in rows[2:]:
+                row_text = row_text.strip()
+                row_text = row_text.strip('|')
+                row_values = row_text.split('|')
+                row_values = [v.strip() for v in row_values]
+                data_rows.append(row_values)
+
             for row in data_rows:
                 cleaned_row = []
                 for cell in row:
-                    # Remove leading and trailing whitespace
-                    cell = cell.strip()
-                    # Check if cell is empty or contains only whitespace
                     if not cell or cell.isspace():
-                        cell = 'Not Disclosed'
+                        cell = unk_token
                     cleaned_row.append(cell)
                 try:
                     if len(cleaned_row) == len(headers_list):
@@ -158,22 +166,71 @@ class Bangalore_Score(evaluate.Metric):
                 except Exception as e:
                     print(e)
 
-            # normalize rows (set column order with sorted headers)
+            # normalize
             sorted_headers = sorted(set(list(headers_list)))
-            df = pd.DataFrame(records, columns=sorted_headers)  # create dataframe
-            df.fillna(unk_token, inplace=True)
-            df = df.sort_values(by=sorted_headers)              # sort rows
+            df = pd.DataFrame(records, columns=sorted_headers)  # normalize headers
+            df.fillna(unk_token, inplace=True)                  # fill NaNs
+            df = df.sort_values(by=sorted_headers)              # normalize rows
+
+            # omit columns
+            if len(omit_columns) > 0:
+                omit_columns = [c.strip() for c in omit_columns]
+                omit_columns = [c for c in omit_columns if c in df.columns]
+                df = df.drop(columns=omit_columns)
+
+            # return df only
+            if return_df:
+                return df
+
             # csv
             csv_norm = df.to_csv(index=False, sep='|')
             csv_norm  = csv_norm.replace('|', ' | ')    # add spaces around pipes
             csv_norm = csv_norm.replace('\r', '')       # remove carriage returns
-
-            # rows text
+            # only rows text
             rows_text = csv_norm.split('\n')[1:-1]
             rows_text = '\n'.join(rows_text).strip()
-        return {'rtext': rows_text}
 
-    def msr_fn(self, reference, predictions, metric, metric_key, max_score=True):
+        if return_df:
+            return None
+        else:
+            return {'rtext': rows_text}
+
+    def ref_omit_columns(
+            self,
+            example,
+            max_chars=50,
+    ):
+        """
+        Detect columns to omit from eval
+        1. columns with text > max_chars are likely to be comments
+            * to be skipped in evaluation
+            * screws up metrics
+        :param example:
+        :param max_chars:
+        :param omit_columns:
+        :param text_field:
+        :return:
+            {
+                'omit_columns': <list of text columns>,
+            }
+        """
+        comments = []
+        df = self.normalize_fn(example,text_field='ref', return_df=True,)
+        if df is not None:
+            for colname in df.columns:
+                c_lens = [len(str(x)) for x in df[colname]]
+                if max(c_lens) > max_chars:
+                    comments.append(colname)
+        return {'omit_columns': comments}
+
+    def msr_fn(
+            self,
+            reference,
+            predictions,
+            metric,
+            metric_key,
+            max_score=True,
+    ):
         """
         MSR (Most Similar Row / Record)
         * computes metric for predictions
@@ -187,12 +244,20 @@ class Bangalore_Score(evaluate.Metric):
         """
         scores_list = []
         for ref, pred in zip([reference]*len(predictions), predictions):
-            score = metric.compute(references=[ref], predictions=[pred])[metric_key]
+            score_dict = metric.compute(references=[ref], predictions=[pred])
+            if isinstance(score_dict, dict):
+                score = score_dict.get(metric_key, 0)
+            elif isinstance(score_dict, float):
+                score = score_dict
+            else:
+                score = 0
             scores_list.append(score)
+
         if max_score:
             best_score = max(scores_list)
         else:
             best_score = min(scores_list)
+
         best_pred = predictions[scores_list.index(best_score)]
         _predictions = []
         for pred in predictions:
@@ -200,18 +265,27 @@ class Bangalore_Score(evaluate.Metric):
                 _predictions.append(pred)
         return best_score, best_pred, _predictions
 
-    def amsr_fn(self, example, **kwargs):
+    def amsr_fn(
+            self,
+            example,
+            **kwargs
+    ):
+        """
+        Aggregate MSR (Most Similar Row / Record)
+        :param example:
+        :param kwargs:
+        :return:
+        """
         ref_text, pred_text = example['ref'].strip(), example['pred'].strip()
         ref_rows, pred_rows = ref_text.split('\n'), pred_text.split('\n')
-
-        # test msr
+        ### test msr
         msr_list = []
         for ref in ref_rows:
             if len(pred_rows) == 0:
                 msr_list.append(0)
                 continue
             score, best_pred, pred_rows = self.msr_fn(reference=ref, predictions=pred_rows, **kwargs)
-            # meteor STILL too flexible...
+            ### meteor STILL too flexible...
             if False:
                 print(
                     '\n\n\n---'
@@ -225,13 +299,14 @@ class Bangalore_Score(evaluate.Metric):
             msr_list.append(score)
 
         aggregate_score = sum(msr_list) / len(msr_list)
-        # print('ref_rows:')
-        # for ref in ref_rows:
-        #     print(f'\t* {ref}')
-        # print('\n\npred_rows:')
-        # for pred in pred_text.split('\n'):
-        #     print(f'\t* {pred}')
-        # print(f'\n\naggregate_score: {aggregate_score}')
+        if False:
+            print('ref_rows:')
+            for ref in ref_rows:
+                print(f'\t* {ref}')
+            print('\n\npred_rows:')
+            for pred in pred_text.split('\n'):
+                print(f'\t* {pred}')
+            print(f'\n\naggregate_score: {aggregate_score}')
         return {'amsr': aggregate_score}
 
     def _compute(
@@ -240,28 +315,37 @@ class Bangalore_Score(evaluate.Metric):
             references,
             metric,
             metric_key,
-            max_score=True,
-            num_proc=None
+            best='max',
     ):
         """Returns the scores"""
         import json
         import evaluate
         import pandas as pd
-        from datasets import Dataset, DatasetDict
-
-        ### normalize ###
-        pred_ds = Dataset.from_dict({'text': predictions})
-        refs_ds = Dataset.from_dict({'text': references})
-        proc_ds = DatasetDict({'predictions': pred_ds, 'references': refs_ds})
-        proc_ds = proc_ds.map(self.normalize_fn, num_proc=num_proc, desc='normalizing')
-        predictions = proc_ds['predictions']['rtext']
-        references = proc_ds['references']['rtext']
+        from datasets import Dataset, DatasetDict, disable_caching
+
+        disable_caching()
+
+        # 1. parse predictions and references
         proc_ds = Dataset.from_dict({'pred': predictions, 'ref': references})
 
-        ### compute amsr ###
+        # 2. detect columns to omit from eval calculations (eg: comments)
+        proc_ds = proc_ds.map(self.ref_omit_columns, desc='omit_columns (ref)')
+
+        # 3. normalize predictions and references
+        predictions_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='pred'))
+        references_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='ref'))
+        eval_data = {'pred': predictions_ds['rtext'], 'ref': references_ds['rtext']}
+
+        # 4. compute amsr for given metric
+        proc_ds = Dataset.from_dict(eval_data)
         proc_ds = proc_ds.map(
-            lambda x: self.amsr_fn(x, metric=metric, metric_key=metric_key, max_score=max_score),
-            num_proc=num_proc, desc='computing amsr'
+            lambda x: self.amsr_fn(
+                example=x,
+                metric=metric,
+                metric_key=metric_key,
+                max_score=True if best == 'max' else False
+            ),
+            desc=f'amsr ({metric_key})'
         )
 
         amsr_mean = sum(proc_ds['amsr']) / len(proc_ds['amsr'])