fall back to GPT2TokenizerFast for models which don't have a fast tokenizer (like OPT)

syntaxgym.py

@@ -14,16 +14,21 @@
 """TODO: Add a description here."""
 
 from collections import defaultdict
+import logging
 from typing import List, Dict, Tuple, NamedTuple
 
 import datasets
 import evaluate
 import numpy as np
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM, \
+    PreTrainedTokenizer, PreTrainedTokenizerFast, \
+    GPT2TokenizerFast
 
 from .prediction import Prediction
 
+L = logging.getLogger(__name__)
+
 
 _CITATION = """\
 @inproceedings{Hu:et-al:2020,
@@ -108,7 +113,15 @@ def prepare_tokenizer(model, batch_size, add_start_token=True) -> Tuple[PreTrain
         tokenizer:
         tokenizer_kwargs: suggested kwargs for any tokenizer calls
     """
+    
     tokenizer = AutoTokenizer.from_pretrained(model.name_or_path)
+    if not isinstance(tokenizer, PreTrainedTokenizerFast):
+        # We need a fast tokenizer because these are the only tokenizers that support
+        # return_offsets_mapping. Try to use GPT2 tokenizer -- this is sufficient for
+        # OPT.
+        L.warning(f"The model {model.name_or_path} does not have a fast tokenizer, "
+                  f"which is required for this metric. Running with GPT2 tokenizer.")
+        tokenizer = GPT2TokenizerFast.from_pretrained(model.name_or_path)
 
     # if batch_size > 1 (which generally leads to padding being required), and
     # if there is not an already assigned pad_token, assign an existing