






"""simple docstring"""




import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 	__UpperCAmelCase		(				_lowerCamelCase,  unittest.TestCase	):


	'''simple docstring'''





	lowercase		:	Dict							     =				LongformerTokenizer
	lowercase		:	Tuple							     =				True
	lowercase		:	Any							     =				LongformerTokenizerFast
	lowercase		:	List[Any]							     =				True


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		super().setUp()

		# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
		_SCREAMING_SNAKE_CASE						=[
		    "l",
		    "o",
		    "w",
		    "e",
		    "r",
		    "s",
		    "t",
		    "i",
		    "d",
		    "n",
		    "\u0120",
		    "\u0120l",
		    "\u0120n",
		    "\u0120lo",
		    "\u0120low",
		    "er",
		    "\u0120lowest",
		    "\u0120newer",
		    "\u0120wider",
		    "<unk>",
		]
		_SCREAMING_SNAKE_CASE						=dict(zip(_A      ,      range(len(_A					)					)					)					)
		_SCREAMING_SNAKE_CASE						=["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
		_SCREAMING_SNAKE_CASE						={"unk_token": "<unk>"}

		_SCREAMING_SNAKE_CASE						=os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['''vocab_file''']					)
		_SCREAMING_SNAKE_CASE						=os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['''merges_file''']					)
		with open(self.vocab_file      ,      '''w'''      ,      encoding='''utf-8'''					) as fp:
			fp.write(json.dumps(_A					) + '''\n'''					)
		with open(self.merges_file      ,      '''w'''      ,      encoding='''utf-8'''					) as fp:
			fp.write('''\n'''.join(_A					)					)


	def 				UpperCamelCase_  (     self      ,      **_A					):



		'''simple docstring'''

		kwargs.update(self.special_tokens_map					)
		return self.tokenizer_class.from_pretrained(self.tmpdirname      ,      **_A					)


	def 				UpperCamelCase_  (     self      ,      **_A					):



		'''simple docstring'''

		kwargs.update(self.special_tokens_map					)
		return self.rust_tokenizer_class.from_pretrained(self.tmpdirname      ,      **_A					)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						="lower newer"
		_SCREAMING_SNAKE_CASE						="lower newer"
		return input_text, output_text


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.tokenizer_class(self.vocab_file      ,      self.merges_file      ,      **self.special_tokens_map					)
		_SCREAMING_SNAKE_CASE						="lower newer"
		_SCREAMING_SNAKE_CASE						=["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
		_SCREAMING_SNAKE_CASE						=tokenizer.tokenize(_A					)  # , add_prefix_space=True)
		self.assertListEqual(_A      ,      _A					)

		_SCREAMING_SNAKE_CASE						=tokens + [tokenizer.unk_token]
		_SCREAMING_SNAKE_CASE						=[0, 1, 2, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
		self.assertListEqual(tokenizer.convert_tokens_to_ids(_A					)      ,      _A					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.get_tokenizer()

		self.assertListEqual(tokenizer.encode('''Hello world!'''      ,      add_special_tokens=_A					)      ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 2]					)
		self.assertListEqual(
		    tokenizer.encode('''Hello world! cécé herlolip 418'''      ,      add_special_tokens=_A					)      ,      [0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]      ,      )


	@slow
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.tokenizer_class.from_pretrained('''allenai/longformer-base-4096'''					)

		_SCREAMING_SNAKE_CASE						=tokenizer.encode('''sequence builders'''      ,      add_special_tokens=_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.encode('''multi-sequence build'''      ,      add_special_tokens=_A					)

		_SCREAMING_SNAKE_CASE						=tokenizer.encode(
		    '''sequence builders'''      ,      add_special_tokens=_A      ,      add_prefix_space=_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.encode(
		    '''sequence builders'''      ,      '''multi-sequence build'''      ,      add_special_tokens=_A      ,      add_prefix_space=_A					)

		_SCREAMING_SNAKE_CASE						=tokenizer.build_inputs_with_special_tokens(_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.build_inputs_with_special_tokens(_A      ,      _A					)

		assert encoded_sentence == encoded_text_from_decode
		assert encoded_pair == encoded_pair_from_decode


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.get_tokenizer()

		_SCREAMING_SNAKE_CASE						="Encode this sequence."
		_SCREAMING_SNAKE_CASE						=tokenizer.byte_encoder[" ".encode('''utf-8'''					)[0]]

		# Testing encoder arguments
		_SCREAMING_SNAKE_CASE						=tokenizer.encode(_A      ,      add_special_tokens=_A      ,      add_prefix_space=_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
		self.assertNotEqual(_A      ,      _A					)

		_SCREAMING_SNAKE_CASE						=tokenizer.encode(_A      ,      add_special_tokens=_A      ,      add_prefix_space=_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
		self.assertEqual(_A      ,      _A					)

		tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}					)
		_SCREAMING_SNAKE_CASE						=tokenizer.encode(_A      ,      add_special_tokens=_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(encoded[1]					)[0]
		self.assertNotEqual(_A      ,      _A					)

		# Testing spaces after special tokens
		_SCREAMING_SNAKE_CASE						="<mask>"
		tokenizer.add_special_tokens(
		    {'''mask_token''': AddedToken(_A      ,      lstrip=_A      ,      rstrip=_A					)}					)  # mask token has a left space
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_tokens_to_ids(_A					)

		_SCREAMING_SNAKE_CASE						="Encode <mask> sequence"
		_SCREAMING_SNAKE_CASE						="Encode <mask>sequence"

		_SCREAMING_SNAKE_CASE						=tokenizer.encode(_A					)
		_SCREAMING_SNAKE_CASE						=encoded.index(_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
		self.assertEqual(_A      ,      _A					)

		_SCREAMING_SNAKE_CASE						=tokenizer.encode(_A					)
		_SCREAMING_SNAKE_CASE						=encoded.index(_A					)
		_SCREAMING_SNAKE_CASE						=tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
		self.assertNotEqual(_A      ,      _A					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(_A      ,      **_A					)
				_SCREAMING_SNAKE_CASE						=self.tokenizer_class.from_pretrained(_A      ,      **_A					)
				_SCREAMING_SNAKE_CASE						="A, <mask> AllenNLP sentence."
				_SCREAMING_SNAKE_CASE						=tokenizer_r.encode_plus(_A      ,      add_special_tokens=_A      ,      return_token_type_ids=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_p.encode_plus(_A      ,      add_special_tokens=_A      ,      return_token_type_ids=_A					)

				# token_type_ids should put 0 everywhere
				self.assertEqual(sum(tokens_r['''token_type_ids''']					)      ,      sum(tokens_p['''token_type_ids''']					)					)

				# attention_mask should put 1 everywhere, so sum over length should be 1
				self.assertEqual(
				    sum(tokens_r['''attention_mask''']					) / len(tokens_r['''attention_mask''']					)      ,      sum(tokens_p['''attention_mask''']					) / len(tokens_p['''attention_mask''']					)      ,      )

				_SCREAMING_SNAKE_CASE						=tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']					)
				_SCREAMING_SNAKE_CASE						=tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']					)

				# Rust correctly handles the space before the mask while python doesnt
				self.assertSequenceEqual(tokens_p['''input_ids''']      ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]					)
				self.assertSequenceEqual(tokens_r['''input_ids''']      ,      [0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]					)

				self.assertSequenceEqual(
				    _A      ,      ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)
				self.assertSequenceEqual(
				    _A      ,      ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		for trim_offsets, add_prefix_space in itertools.product([True, False]      ,      repeat=2					):
			_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
			    self.tmpdirname      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)

			_SCREAMING_SNAKE_CASE						=json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()					)
			_SCREAMING_SNAKE_CASE						=json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()					)

			self.assertEqual(pre_tokenizer_state['''add_prefix_space''']      ,      _A					)

			self.assertEqual(post_processor_state['''add_prefix_space''']      ,      _A					)
			self.assertEqual(post_processor_state['''trim_offsets''']      ,      _A					)







	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
				_SCREAMING_SNAKE_CASE						="hello"  # `hello` is a token in the vocabulary of `pretrained_name`
				_SCREAMING_SNAKE_CASE						=f"""{text_of_1_token} {text_of_1_token}"""

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (len(_A					) + 1, len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (len(_A					) + 1, len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (len(_A					), len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (len(_A					), len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=f""" {text}"""

				# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
				#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
				# )
				# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
				# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
				# self.assertEqual(
				#     encoding.offset_mapping[1],
				#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
				# )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (1, 1 + len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (1 + len(_A					) + 1, 1 + len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, 1 + len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (1 + len(_A					), 1 + len(_A					) + 1 + len(_A					))      ,      )

				_SCREAMING_SNAKE_CASE						=self.rust_tokenizer_class.from_pretrained(
				    _A      ,      use_fast=_A      ,      add_prefix_space=_A      ,      trim_offsets=_A					)
				_SCREAMING_SNAKE_CASE						=tokenizer_r(_A      ,      return_offsets_mapping=_A      ,      add_special_tokens=_A					)
				self.assertEqual(encoding.offset_mapping[0]      ,      (0, 1 + len(_A					))					)
				self.assertEqual(
				    encoding.offset_mapping[1]      ,      (1 + len(_A					), 1 + len(_A					) + 1 + len(_A					))      ,      )








'''simple docstring'''


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCamelCase 							=  logging.get_logger(__name__)


_UpperCamelCase 							=  {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
          raise ValueError(f"Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()							)}."							)

     if tokenizer_name is None:
          __A	:       Tuple      =							TOKENIZER_CLASSES
     else:
          __A	:       Dict      =							{tokenizer_name: getattr(SCREAMING_SNAKE_CASE       ,					tokenizer_name + "Fast"							)}

     logger.info(f"Loading tokenizer classes: {tokenizer_names}"							)

     for tokenizer_name in tokenizer_names:
          __A	:       int      =							TOKENIZER_CLASSES[tokenizer_name]

          __A	:       Union[str, Any]      =							True
          if checkpoint_name is None:
               __A	:       Dict      =							list(tokenizer_class.max_model_input_sizes.keys()							)
          else:
               __A	:       str      =							[checkpoint_name]

          logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"							)

          for checkpoint in checkpoint_names:
               logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}"							)

               # Load tokenizer
               __A	:       List[str]      =							tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE       ,					force_download=SCREAMING_SNAKE_CASE							)

               # Save fast tokenizer
               logger.info(f"Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"							)

               # For organization names we create sub-directories
               if "/" in checkpoint:
                    __A  ,__A	:       Optional[Any]      =							checkpoint.split("/"							)
                    __A	:       str      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
               elif add_prefix:
                    __A	:       Optional[int]      =							checkpoint
                    __A	:       Any      =							dump_path
               else:
                    __A	:       int      =							None
                    __A	:       Optional[int]      =							dump_path

               logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()							)[0]:
                    __A	:       Any      =							list(tokenizer.pretrained_vocab_files_map.values()							)[0][checkpoint]
                    __A	:       Optional[int]      =							file_path.split(SCREAMING_SNAKE_CASE							)[-1][0]
                    if next_char == "/":
                         __A	:       Union[str, Any]      =							os.path.join(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)
                         __A	:       Union[str, Any]      =							None

                    logger.info(f"=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"							)

               __A	:       Tuple      =							tokenizer.save_pretrained(
                   SCREAMING_SNAKE_CASE       ,					legacy_format=SCREAMING_SNAKE_CASE       ,					filename_prefix=SCREAMING_SNAKE_CASE							)
               logger.info(f"=> File names {file_names}"							)

               for file_name in file_names:
                    if not file_name.endswith("tokenizer.json"							):
                         os.remove(SCREAMING_SNAKE_CASE							)
                         logger.info(f"=> removing {file_name}"							)


if __name__ == "__main__":
      _UpperCamelCase 							=  argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      _UpperCamelCase 							=  parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

