Use the new tokenizers

allennlp/common/util.py

@@ -519,9 +519,9 @@ def sanitize_wordpiece(wordpiece: str) -> str:
     if wordpiece.startswith("##"):
         return wordpiece[2:]
     elif wordpiece.startswith("Ġ"):
-        return wordpiece[1:]
+        return wordpiece.replace("Ġ", " ")
     elif wordpiece.startswith("▁"):
-        return wordpiece[1:]
+        return wordpiece.replace("▁", " ")
     else:
         return wordpiece
 

allennlp/data/token_indexers/pretrained_transformer_indexer.py

@@ -49,13 +49,14 @@ def __init__(
         self._tokenizer = self._allennlp_tokenizer.tokenizer
         self._added_to_vocabulary = False
 
-        self._num_added_start_tokens = self._allennlp_tokenizer.num_added_start_tokens
-        self._num_added_end_tokens = self._allennlp_tokenizer.num_added_end_tokens
+        self._num_added_start_tokens = len(self._allennlp_tokenizer.single_sequence_start_tokens)
+        self._num_added_end_tokens = len(self._allennlp_tokenizer.single_sequence_end_tokens)
 
         self._max_length = max_length
         if self._max_length is not None:
+            num_added_tokens = len(self._allennlp_tokenizer.tokenize("a")) - 1
             self._effective_max_length = (  # we need to take into account special tokens
-                self._max_length - self._tokenizer.num_added_tokens()
+                self._max_length - num_added_tokens
             )
             if self._effective_max_length <= 0:
                 raise ValueError(

allennlp/data/tokenizers/tokenizer.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 import logging
 
 from allennlp.common import Registrable
@@ -45,3 +45,37 @@ def tokenize(self, text: str) -> List[Token]:
         tokens : `List[Token]`
         """
         raise NotImplementedError
+
+    def add_special_tokens(
+        self, tokens1: List[Token], tokens2: Optional[List[Token]] = None
+    ) -> List[Token]:
+        """
+        Adds special tokens to tokenized text. These are tokens like [CLS] or [SEP].
+
+        Not all tokenizers do this. The default is to just return the tokens unchanged.
+
+        # Parameters
+
+        tokens1 : `List[Token]`
+            The list of tokens to add special tokens to.
+        tokens2 : `Optional[List[Token]]`
+            An optional second list of tokens. This will be concatenated with `tokens1`. Special tokens will be
+            added as appropriate.
+
+        # Returns
+        tokens : `List[Token]`
+            The combined list of tokens, with special tokens added.
+        """
+        return tokens1 + (tokens2 or [])
+
+    def special_tokens_for_sequence(self) -> int:
+        """
+        Returns the number of special tokens added for a single sequence.
+        """
+        return 0
+
+    def special_tokens_for_pair(self) -> int:
+        """
+        Returns the number of special tokens added for a pair of sequences.
+        """
+        return 0

allennlp/modules/token_embedders/pretrained_transformer_embedder.py

@@ -41,8 +41,8 @@ def __init__(self, model_name: str, max_length: int = None) -> None:
         self.output_dim = self.transformer_model.config.hidden_size
 
         tokenizer = PretrainedTransformerTokenizer(model_name)
-        self._num_added_start_tokens = tokenizer.num_added_start_tokens
-        self._num_added_end_tokens = tokenizer.num_added_end_tokens
+        self._num_added_start_tokens = len(tokenizer.single_sequence_start_tokens)
+        self._num_added_end_tokens = len(tokenizer.single_sequence_end_tokens)
         self._num_added_tokens = self._num_added_start_tokens + self._num_added_end_tokens
 
     @overrides

allennlp/tests/data/token_indexers/pretrained_transformer_indexer_test.py

@@ -35,20 +35,6 @@ def test_as_array_produces_token_sequence_bert_cased(self):
         indexed = indexer.tokens_to_indices(allennlp_tokens, vocab)
         assert indexed["token_ids"] == expected_ids
 
-    def test_as_array_produces_token_sequence_bert_cased_sentence_pair(self):
-        tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
-        allennlp_tokenizer = PretrainedTransformerTokenizer("bert-base-cased")
-        indexer = PretrainedTransformerIndexer(model_name="bert-base-cased")
-        default_format = "[CLS] AllenNLP is great! [SEP] Really it is! [SEP]"
-        tokens = tokenizer.tokenize(default_format)
-        expected_ids = tokenizer.convert_tokens_to_ids(tokens)
-        allennlp_tokens = allennlp_tokenizer.tokenize_sentence_pair(
-            "AllenNLP is great!", "Really it is!"
-        )
-        vocab = Vocabulary()
-        indexed = indexer.tokens_to_indices(allennlp_tokens, vocab)
-        assert indexed["token_ids"] == expected_ids
-
     def test_as_array_produces_token_sequence_roberta(self):
         tokenizer = AutoTokenizer.from_pretrained("roberta-base")
         allennlp_tokenizer = PretrainedTransformerTokenizer("roberta-base")
@@ -63,20 +49,6 @@ def test_as_array_produces_token_sequence_roberta(self):
         indexed = indexer.tokens_to_indices(allennlp_tokens, vocab)
         assert indexed["token_ids"] == expected_ids
 
-    def test_as_array_produces_token_sequence_roberta_sentence_pair(self):
-        tokenizer = AutoTokenizer.from_pretrained("roberta-base")
-        allennlp_tokenizer = PretrainedTransformerTokenizer("roberta-base")
-        indexer = PretrainedTransformerIndexer(model_name="roberta-base")
-        default_format = "<s> AllenNLP is great! </s> </s> Really it is! </s>"
-        tokens = tokenizer.tokenize(default_format)
-        expected_ids = tokenizer.convert_tokens_to_ids(tokens)
-        allennlp_tokens = allennlp_tokenizer.tokenize_sentence_pair(
-            "AllenNLP is great!", "Really it is!"
-        )
-        vocab = Vocabulary()
-        indexed = indexer.tokens_to_indices(allennlp_tokens, vocab)
-        assert indexed["token_ids"] == expected_ids
-
     def test_transformers_vocab_sizes(self):
         def check_vocab_size(model_name: str):
             namespace = "tags"