


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase   :    Optional[Any]     =		logging.get_logger(__name__)


UpperCAmelCase   :    Optional[Any]     =		{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase   :    Optional[int]     =		{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

UpperCAmelCase   :    Optional[Any]     =		{
    "facebook/bart-base": 10_24,
    "facebook/bart-large": 10_24,
    "facebook/bart-large-mnli": 10_24,
    "facebook/bart-large-cnn": 10_24,
    "facebook/bart-large-xsum": 10_24,
    "yjernite/bart_eli5": 10_24,
}






@lru_cache()
def 	__lowerCamelCase						(							):





			'''simple docstring'''




			lowerCamelCase    		=  (
			    list(range(ord("""!"""      )  ,							ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )  ,							ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )  ,							ord("""ÿ"""      ) + 1      )      )
			)
			lowerCamelCase    		=  bs[:]
			lowerCamelCase    		=  0
			for b in range(2**8      ):
						if b not in bs:
									bs.append(lowerCamelCase__      )
									cs.append(2**8 + n      )
									n += 1
			lowerCamelCase    		=  [chr(lowerCamelCase__      ) for n in cs]
			return dict(zip(lowerCamelCase__  ,							lowerCamelCase__      )      )






def 	__lowerCamelCase						(							lowerCamelCase__		:						List[str]      ):





			'''simple docstring'''




			lowerCamelCase    		=  set()
			lowerCamelCase    		=  word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)      )
						lowerCamelCase    		=  char
			return pairs







class        __lowercase       (							a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[Any]			       =		VOCAB_FILES_NAMES
		UpperCamelCase     :					List[str]			       =		PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase     :					Optional[Any]			       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase     :					Dict			       =		["input_ids", "attention_mask"]







		def __init__( self		,      A		,      A		,      A="replace"		,      A="<s>"		,      A="</s>"		,      A="</s>"		,      A="<s>"		,      A="<unk>"		,      A="<pad>"		,      A="<mask>"		,      A=False		,      **A		,      )  -> Union[str, Any]:







					'''simple docstring'''







					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else bos_token
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else eos_token
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else sep_token
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else cls_token
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else unk_token
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					lowerCamelCase    		=  AddedToken(A		,      lstrip=A		,      rstrip=A	) if isinstance(A		,      A	) else mask_token

					super().__init__(
					    errors=A		,      bos_token=A		,      eos_token=A		,      unk_token=A		,      sep_token=A		,      cls_token=A		,      pad_token=A		,      mask_token=A		,      add_prefix_space=A		,      **A		,      )

					with open(A		,      encoding="""utf-8"""	) as vocab_handle:
								lowerCamelCase    		=  json.load(A	)
					lowerCamelCase    		=  {v: k for k, v in self.encoder.items()}
					lowerCamelCase    		=  errors  # how to handle errors in decoding
					lowerCamelCase    		=  bytes_to_unicode()
					lowerCamelCase    		=  {v: k for k, v in self.byte_encoder.items()}
					with open(A		,      encoding="""utf-8"""	) as merges_handle:
								lowerCamelCase    		=  merges_handle.read().split("""\n"""	)[1:-1]
					lowerCamelCase    		=  [tuple(merge.split()	) for merge in bpe_merges]
					lowerCamelCase    		=  dict(zip(A		,      range(len(A	)	)	)	)
					lowerCamelCase    		=  {}
					lowerCamelCase    		=  add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					lowerCamelCase    		=  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""	)







		@property
		def 					__A   ( self	)  -> Tuple:







					'''simple docstring'''







					return len(self.encoder	)







		def 					__A   ( self	)  -> List[str]:







					'''simple docstring'''







					return dict(self.encoder		,      **self.added_tokens_encoder	)







		def 					__A   ( self		,      A	)  -> Dict:







					'''simple docstring'''







					if token in self.cache:
								return self.cache[token]
					lowerCamelCase    		=  tuple(A	)
					lowerCamelCase    		=  get_pairs(A	)

					if not pairs:
								return token

					while True:
								lowerCamelCase    		=  min(A		,      key=lambda A	: self.bpe_ranks.get(A		,      float("""inf"""	)	)	)
								if bigram not in self.bpe_ranks:
											break
								lowerCamelCase		,		lowerCamelCase    		=  bigram
								lowerCamelCase    		=  []
								lowerCamelCase    		=  0
								while i < len(A	):
											try:
														lowerCamelCase    		=  word.index(A		,      A	)
											except ValueError:
														new_word.extend(word[i:]	)
														break
											else:
														new_word.extend(word[i:j]	)
														lowerCamelCase    		=  j

											if word[i] == first and i < len(A	) - 1 and word[i + 1] == second:
														new_word.append(first + second	)
														i += 2
											else:
														new_word.append(word[i]	)
														i += 1
								lowerCamelCase    		=  tuple(A	)
								lowerCamelCase    		=  new_word
								if len(A	) == 1:
											break
								else:
											lowerCamelCase    		=  get_pairs(A	)
					lowerCamelCase    		=  """ """.join(A	)
					lowerCamelCase    		=  word
					return word







		def 					__A   ( self		,      A	)  -> Any:







					'''simple docstring'''







					lowerCamelCase    		=  []
					for token in re.findall(self.pat		,      A	):
								lowerCamelCase    		=  """""".join(
								    self.byte_encoder[b] for b in token.encode("""utf-8"""	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A	).split(""" """	)	)
					return bpe_tokens







		def 					__A   ( self		,      A	)  -> str:







					'''simple docstring'''







					return self.encoder.get(A		,      self.encoder.get(self.unk_token	)	)







		def 					__A   ( self		,      A	)  -> Optional[Any]:







					'''simple docstring'''







					return self.decoder.get(A	)







		def 					__A   ( self		,      A	)  -> Union[str, Any]:







					'''simple docstring'''







					lowerCamelCase    		=  """""".join(A	)
					lowerCamelCase    		=  bytearray([self.byte_decoder[c] for c in text]	).decode("""utf-8"""		,      errors=self.errors	)
					return text







		def 					__A   ( self		,      A		,      A = None	)  -> Tuple[str]:







					'''simple docstring'''







					if not os.path.isdir(A	):
								logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
								return
					lowerCamelCase    		=  os.path.join(
					    A		,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
					lowerCamelCase    		=  os.path.join(
					    A		,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

					with open(A		,      """w"""		,      encoding="""utf-8"""	) as f:
								f.write(json.dumps(self.encoder		,      indent=2		,      sort_keys=A		,      ensure_ascii=A	) + """\n"""	)

					lowerCamelCase    		=  0
					with open(A		,      """w"""		,      encoding="""utf-8"""	) as writer:
								writer.write("""#version: 0.2\n"""	)
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,      key=lambda A	: kv[1]	):
											if index != token_index:
														logger.warning(
														    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
														    """ Please check that the tokenizer is not corrupted!"""	)
														lowerCamelCase    		=  token_index
											writer.write(""" """.join(A	) + """\n"""	)
											index += 1

					return vocab_file, merge_file







		def 					__A   ( self		,      A		,      A = None	)  -> List[int]:







					'''simple docstring'''







					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					lowerCamelCase    		=  [self.cls_token_id]
					lowerCamelCase    		=  [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep







		def 					__A   ( self		,      A		,      A = None		,      A = False	)  -> List[int]:







					'''simple docstring'''







					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=A		,      token_ids_a=A		,      already_has_special_tokens=A	)

					if token_ids_a is None:
								return [1] + ([0] * len(A	)) + [1]
					return [1] + ([0] * len(A	)) + [1, 1] + ([0] * len(A	)) + [1]







		def 					__A   ( self		,      A		,      A = None	)  -> List[int]:







					'''simple docstring'''







					lowerCamelCase    		=  [self.sep_token_id]
					lowerCamelCase    		=  [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep	) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







		def 					__A   ( self		,      A		,      A=False		,      **A	)  -> List[Any]:







					'''simple docstring'''







					lowerCamelCase    		=  kwargs.pop("""add_prefix_space"""		,      self.add_prefix_space	)
					if (is_split_into_words or add_prefix_space) and (len(A	) > 0 and not text[0].isspace()):
								lowerCamelCase    		=  """ """ + text
					return (text, kwargs)





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase   :    Optional[Any]     =		get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class        __lowercase       (							a_			,							unittest.TestCase  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[Any]			       =		ReformerTokenizer
		UpperCamelCase     :					Optional[int]			       =		ReformerTokenizerFast
		UpperCamelCase     :					Union[str, Any]			       =		True
		UpperCamelCase     :					Dict			       =		False
		UpperCamelCase     :					Dict			       =		True







		def 					__A   ( self	)  -> str:







					'''simple docstring'''







					super().setUp()

					lowerCamelCase    		=  ReformerTokenizer(A		,      keep_accents=A	)
					tokenizer.save_pretrained(self.tmpdirname	)







		def 					__A   ( self	)  -> Optional[Any]:







					'''simple docstring'''







					lowerCamelCase    		=  """<s>"""
					lowerCamelCase    		=  1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(A	)		,      A	)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(A	)		,      A	)







		def 					__A   ( self	)  -> Optional[Any]:







					'''simple docstring'''







					lowerCamelCase    		=  list(self.get_tokenizer().get_vocab().keys()	)

					self.assertEqual(vocab_keys[0]		,      """<unk>"""	)
					self.assertEqual(vocab_keys[1]		,      """<s>"""	)
					self.assertEqual(vocab_keys[-1]		,      """j"""	)
					self.assertEqual(len(A	)		,      10_00	)







		def 					__A   ( self	)  -> int:







					'''simple docstring'''







					self.assertEqual(self.get_tokenizer().vocab_size		,      10_00	)







		def 					__A   ( self	)  -> Tuple:







					'''simple docstring'''







					if not self.test_rust_tokenizer:
								return

					lowerCamelCase    		=  self.get_tokenizer()
					lowerCamelCase    		=  self.get_rust_tokenizer()

					lowerCamelCase    		=  """I was born in 92000, and this is falsé."""

					lowerCamelCase    		=  tokenizer.tokenize(A	)
					lowerCamelCase    		=  rust_tokenizer.tokenize(A	)
					self.assertListEqual(A		,      A	)

					lowerCamelCase    		=  tokenizer.encode(A		,      add_special_tokens=A	)
					lowerCamelCase    		=  rust_tokenizer.encode(A		,      add_special_tokens=A	)
					self.assertListEqual(A		,      A	)

					lowerCamelCase    		=  self.get_rust_tokenizer()
					lowerCamelCase    		=  tokenizer.encode(A	)
					lowerCamelCase    		=  rust_tokenizer.encode(A	)
					self.assertListEqual(A		,      A	)







		def 					__A   ( self		,      A=15	)  -> Tuple:







					'''simple docstring'''







					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
											lowerCamelCase    		=  self.rust_tokenizer_class.from_pretrained(A		,      **A	)

											# Simple input
											lowerCamelCase    		=  """This is a simple input"""
											lowerCamelCase    		=  ["""This is a simple input 1""", """This is a simple input 2"""]
											lowerCamelCase    		=  ("""This is a simple input""", """This is a pair""")
											lowerCamelCase    		=  [
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											self.assertRaises(A		,      tokenizer_r.encode		,      A		,      max_length=A		,      padding="""max_length"""	)

											# Simple input
											self.assertRaises(A		,      tokenizer_r.encode_plus		,      A		,      max_length=A		,      padding="""max_length"""	)

											# Simple input
											self.assertRaises(
											    A		,      tokenizer_r.batch_encode_plus		,      A		,      max_length=A		,      padding="""max_length"""		,      )

											# Pair input
											self.assertRaises(A		,      tokenizer_r.encode		,      A		,      max_length=A		,      padding="""max_length"""	)

											# Pair input
											self.assertRaises(A		,      tokenizer_r.encode_plus		,      A		,      max_length=A		,      padding="""max_length"""	)

											# Pair input
											self.assertRaises(
											    A		,      tokenizer_r.batch_encode_plus		,      A		,      max_length=A		,      padding="""max_length"""		,      )







		def 					__A   ( self	)  -> List[Any]:







					'''simple docstring'''







					pass







		def 					__A   ( self	)  -> List[Any]:







					'''simple docstring'''







					lowerCamelCase    		=  ReformerTokenizer(A		,      keep_accents=A	)

					lowerCamelCase    		=  tokenizer.tokenize("""This is a test"""	)
					self.assertListEqual(A		,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(A	)		,      [2_85, 46, 10, 1_70, 3_82]		,      )

					lowerCamelCase    		=  tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
					self.assertListEqual(
					    A		,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]		,      )
					lowerCamelCase    		=  tokenizer.convert_tokens_to_ids(A	)
					self.assertListEqual(
					    A		,      [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,      )

					lowerCamelCase    		=  tokenizer.convert_ids_to_tokens(A	)
					self.assertListEqual(
					    A		,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]		,      )







		@cached_property
		def 					__A   ( self	)  -> List[Any]:







					'''simple docstring'''







					return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""	)







		@slow
		def 					__A   ( self	)  -> Any:







					'''simple docstring'''







					lowerCamelCase    		=  """Hello World!"""
					lowerCamelCase    		=  [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

					self.assertListEqual(A		,      self.big_tokenizer.encode(A	)	)







		@slow
		def 					__A   ( self	)  -> int:







					'''simple docstring'''







					lowerCamelCase    		=  (
					    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
					    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
					)
					lowerCamelCase    		=  [
					    1_08,
					    2_65,
					    24,
					    1_11,
					    4,
					    2_58,
					    1_56,
					    35,
					    28,
					    2_75,
					    3,
					    2_59,
					    2_97,
					    2_60,
					    84,
					    4,
					    35,
					    1_10,
					    44,
					    8,
					    2_59,
					    91,
					    2_68,
					    21,
					    11,
					    2_09,
					    2_74,
					    1_09,
					    2_66,
					    2_77,
					    1_17,
					    86,
					    93,
					    3_15,
					    2_58,
					    2_78,
					    2_58,
					    2_77,
					    2_58,
					    0,
					    2_58,
					    2_88,
					    2_58,
					    3_19,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    2_87,
					    2_58,
					    3_15,
					    2_58,
					    2_89,
					    2_58,
					    2_78,
					    99,
					    2_69,
					    2_66,
					    2_62,
					    8,
					    2_59,
					    2_41,
					    4,
					    2_17,
					    2_30,
					    2_68,
					    2_66,
					    55,
					    1_68,
					    1_06,
					    75,
					    1_93,
					    2_66,
					    2_23,
					    27,
					    49,
					    26,
					    2_82,
					    25,
					    2_64,
					    2_99,
					    19,
					    26,
					    0,
					    2_58,
					    2_77,
					    1_17,
					    86,
					    93,
					    1_76,
					    1_83,
					    2_70,
					    11,
					    2_62,
					    42,
					    61,
					    2_65,
					]

					self.assertListEqual(A		,      self.big_tokenizer.encode(A	)	)







		@require_torch
		@slow
		def 					__A   ( self	)  -> Tuple:







					'''simple docstring'''







					import torch

					from transformers import ReformerConfig, ReformerModel

					# Build sequence
					lowerCamelCase    		=  list(self.big_tokenizer.get_vocab().keys()	)[:10]
					lowerCamelCase    		=  """ """.join(A	)
					lowerCamelCase    		=  self.big_tokenizer.encode_plus(A		,      return_tensors="""pt"""	)
					lowerCamelCase    		=  self.big_tokenizer.batch_encode_plus([sequence, sequence]		,      return_tensors="""pt"""	)

					lowerCamelCase    		=  ReformerConfig()
					# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
					lowerCamelCase    		=  encoded_sequence["""input_ids"""].shape
					lowerCamelCase    		=  ReformerModel(A	)

					# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
					assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

					with torch.no_grad():
								model(**A	)
								model(**A	)







		@slow
		def 					__A   ( self	)  -> int:







					'''simple docstring'''







					lowerCamelCase    		=  {"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					# This tokenizer does not know some characters like ")".
					# That is the reason why we use very simple texts here.
					# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
					lowerCamelCase    		=  [
					    """This is a very simple sentence.""",
					    """The quick brown fox jumps over the lazy dog.""",
					]

					self.tokenizer_integration_test_util(
					    expected_encoding=A		,      model_name="""google/reformer-crime-and-punishment"""		,      revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""		,      padding=A		,      sequences=A		,      )

