

"""simple docstring"""





import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 							snake_case							(			_SCREAMING_SNAKE_CASE      ,  unittest.TestCase ):

      '''simple docstring'''



      _A     :       Optional[Any]         =       RobertaTokenizer
      _A     :       Union[str, Any]         =       RobertaTokenizerFast
      _A     :       Tuple         =       True
      _A     :       Any         =       {"cls_token": "<s>"}

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        __UpperCAmelCase    :						Union[str, Any]       =						[
            '''l''',
            '''o''',
            '''w''',
            '''e''',
            '''r''',
            '''s''',
            '''t''',
            '''i''',
            '''d''',
            '''n''',
            '''\u0120''',
            '''\u0120l''',
            '''\u0120n''',
            '''\u0120lo''',
            '''\u0120low''',
            '''er''',
            '''\u0120lowest''',
            '''\u0120newer''',
            '''\u0120wider''',
            '''<unk>''',
        ]
        __UpperCAmelCase    :						Optional[int]       =						dict(zip(A_						,				range(len(A_ ) ) ) )
        __UpperCAmelCase    :						Dict       =						['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
        __UpperCAmelCase    :						Tuple       =						{'''unk_token''': '''<unk>'''}

        __UpperCAmelCase    :						Optional[Any]       =						os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['''vocab_file'''] )
        __UpperCAmelCase    :						Union[str, Any]       =						os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['''merges_file'''] )
        with open(self.vocab_file						,				'''w'''						,				encoding='''utf-8''' ) as fp:
          fp.write(json.dumps(A_ ) + '''\n''' )
        with open(self.merges_file						,				'''w'''						,				encoding='''utf-8''' ) as fp:
          fp.write('''\n'''.join(A_ ) )

      def       A_					(							self       :      str						,				**__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        kwargs.update(self.special_tokens_map )
        return self.tokenizer_class.from_pretrained(self.tmpdirname						,				**A_ )

      def       A_					(							self       :      Dict						,				**__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        kwargs.update(self.special_tokens_map )
        return RobertaTokenizerFast.from_pretrained(self.tmpdirname						,				**A_ )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						'''lower newer'''
        __UpperCAmelCase    :						Any       =						'''lower newer'''
        return input_text, output_text

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self.tokenizer_class(self.vocab_file						,				self.merges_file						,				**self.special_tokens_map )
        __UpperCAmelCase    :						Union[str, Any]       =						'''lower newer'''
        __UpperCAmelCase    :						Union[str, Any]       =						['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
        __UpperCAmelCase    :						str       =						tokenizer.tokenize(A_ )  # , add_prefix_space=True)
        self.assertListEqual(A_						,				A_ )

        __UpperCAmelCase    :						Dict       =						tokens + [tokenizer.unk_token]
        __UpperCAmelCase    :						Tuple       =						[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A_ )						,				A_ )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.get_tokenizer()

        self.assertListEqual(tokenizer.encode('''Hello world!'''						,				add_special_tokens=A_ )						,				[0, 31_414, 232, 328, 2] )
        self.assertListEqual(
            tokenizer.encode('''Hello world! cécé herlolip 418'''						,				add_special_tokens=A_ )						,				[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]						,				)

      @slow
      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.tokenizer_class.from_pretrained('''roberta-base''' )

        __UpperCAmelCase    :						List[Any]       =						tokenizer.encode('''sequence builders'''						,				add_special_tokens=A_ )
        __UpperCAmelCase    :						Tuple       =						tokenizer.encode('''multi-sequence build'''						,				add_special_tokens=A_ )

        __UpperCAmelCase    :						List[str]       =						tokenizer.encode(
            '''sequence builders'''						,				add_special_tokens=A_						,				add_prefix_space=A_ )
        __UpperCAmelCase    :						List[Any]       =						tokenizer.encode(
            '''sequence builders'''						,				'''multi-sequence build'''						,				add_special_tokens=A_						,				add_prefix_space=A_ )

        __UpperCAmelCase    :						List[Any]       =						tokenizer.build_inputs_with_special_tokens(A_ )
        __UpperCAmelCase    :						Any       =						tokenizer.build_inputs_with_special_tokens(A_						,				A_ )

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						self.get_tokenizer()

        __UpperCAmelCase    :						Optional[int]       =						'''Encode this sequence.'''
        __UpperCAmelCase    :						Dict       =						tokenizer.byte_encoder[''' '''.encode('''utf-8''' )[0]]

        # Testing encoder arguments
        __UpperCAmelCase    :						int       =						tokenizer.encode(A_						,				add_special_tokens=A_						,				add_prefix_space=A_ )
        __UpperCAmelCase    :						Optional[int]       =						tokenizer.convert_ids_to_tokens(encoded[0] )[0]
        self.assertNotEqual(A_						,				A_ )

        __UpperCAmelCase    :						Optional[int]       =						tokenizer.encode(A_						,				add_special_tokens=A_						,				add_prefix_space=A_ )
        __UpperCAmelCase    :						int       =						tokenizer.convert_ids_to_tokens(encoded[0] )[0]
        self.assertEqual(A_						,				A_ )

        tokenizer.add_special_tokens({'''bos_token''': '''<s>'''} )
        __UpperCAmelCase    :						int       =						tokenizer.encode(A_						,				add_special_tokens=A_ )
        __UpperCAmelCase    :						Any       =						tokenizer.convert_ids_to_tokens(encoded[1] )[0]
        self.assertNotEqual(A_						,				A_ )

        # Testing spaces after special tokens
        __UpperCAmelCase    :						Union[str, Any]       =						'''<mask>'''
        tokenizer.add_special_tokens(
            {'''mask_token''': AddedToken(A_						,				lstrip=A_						,				rstrip=A_ )} )  # mask token has a left space
        __UpperCAmelCase    :						Optional[int]       =						tokenizer.convert_tokens_to_ids(A_ )

        __UpperCAmelCase    :						int       =						'''Encode <mask> sequence'''
        __UpperCAmelCase    :						Optional[int]       =						'''Encode <mask>sequence'''

        __UpperCAmelCase    :						Tuple       =						tokenizer.encode(A_ )
        __UpperCAmelCase    :						Optional[Any]       =						encoded.index(A_ )
        __UpperCAmelCase    :						int       =						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1] )[0]
        self.assertEqual(A_						,				A_ )

        __UpperCAmelCase    :						List[str]       =						tokenizer.encode(A_ )
        __UpperCAmelCase    :						List[str]       =						encoded.index(A_ )
        __UpperCAmelCase    :						Any       =						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1] )[0]
        self.assertNotEqual(A_						,				A_ )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        pass

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
            __UpperCAmelCase    :						int       =						self.rust_tokenizer_class.from_pretrained(A_						,				**A_ )
            __UpperCAmelCase    :						Optional[Any]       =						self.tokenizer_class.from_pretrained(A_						,				**A_ )
            __UpperCAmelCase    :						Tuple       =						'''A, <mask> AllenNLP sentence.'''
            __UpperCAmelCase    :						Dict       =						tokenizer_r.encode_plus(A_						,				add_special_tokens=A_						,				return_token_type_ids=A_ )
            __UpperCAmelCase    :						str       =						tokenizer_p.encode_plus(A_						,				add_special_tokens=A_						,				return_token_type_ids=A_ )

            # token_type_ids should put 0 everywhere
            self.assertEqual(sum(tokens_r['''token_type_ids'''] )						,				sum(tokens_p['''token_type_ids'''] ) )

            # attention_mask should put 1 everywhere, so sum over length should be 1
            self.assertEqual(
                sum(tokens_r['''attention_mask'''] ) / len(tokens_r['''attention_mask'''] )						,				sum(tokens_p['''attention_mask'''] ) / len(tokens_p['''attention_mask'''] )						,				)

            __UpperCAmelCase    :						str       =						tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids'''] )
            __UpperCAmelCase    :						Optional[Any]       =						tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids'''] )

            # Rust correctly handles the space before the mask while python doesnt
            self.assertSequenceEqual(tokens_p['''input_ids''']						,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2] )
            self.assertSequenceEqual(tokens_r['''input_ids''']						,				[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2] )

            self.assertSequenceEqual(
                A_						,				['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>'''] )
            self.assertSequenceEqual(
                A_						,				['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>'''] )

      def       A_					(							self       :      int ):


        '''simple docstring'''




        for trim_offsets, add_prefix_space in itertools.product([True, False]						,				repeat=2 ):
          __UpperCAmelCase    :						Dict       =						self.rust_tokenizer_class.from_pretrained(
              self.tmpdirname						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )

          __UpperCAmelCase    :						List[Any]       =						json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__() )
          __UpperCAmelCase    :						Union[str, Any]       =						json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__() )

          self.assertEqual(pre_tokenizer_state['''add_prefix_space''']						,				A_ )

          self.assertEqual(post_processor_state['''add_prefix_space''']						,				A_ )
          self.assertEqual(post_processor_state['''trim_offsets''']						,				A_ )







      def       A_					(							self       :      str ):


        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
            __UpperCAmelCase    :						Union[str, Any]       =						'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
            __UpperCAmelCase    :						Any       =						f'''{text_of_1_token} {text_of_1_token}'''

            __UpperCAmelCase    :						List[Any]       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						int       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(len(A_ ) + 1, len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						Tuple       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						str       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(len(A_ ) + 1, len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						List[Any]       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						List[str]       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(len(A_ ), len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						Optional[Any]       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						Optional[int]       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(len(A_ ), len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						Optional[int]       =						f''' {text}'''

            # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
            #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
            # )
            # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
            # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
            # self.assertEqual(
            #     encoding.offset_mapping[1],
            #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
            # )

            __UpperCAmelCase    :						Tuple       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						Optional[Any]       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(1, 1 + len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(1 + len(A_ ) + 1, 1 + len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						Union[str, Any]       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						Any       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, 1 + len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(1 + len(A_ ), 1 + len(A_ ) + 1 + len(A_ ))						,				)

            __UpperCAmelCase    :						Dict       =						self.rust_tokenizer_class.from_pretrained(
                A_						,				use_fast=A_						,				add_prefix_space=A_						,				trim_offsets=A_ )
            __UpperCAmelCase    :						List[str]       =						tokenizer_r(A_						,				return_offsets_mapping=A_						,				add_special_tokens=A_ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, 1 + len(A_ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(1 + len(A_ ), 1 + len(A_ ) + 1 + len(A_ ))						,				)






"""simple docstring"""





def       lowerCamelCase_       (   )						->str:





  """simple docstring"""


  for n in range(1  ,			1_00_00_00  ):
    yield n * (n + 1) // 2


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[str, Any]:





  """simple docstring"""


  __UpperCAmelCase    :						Dict       =						1
  __UpperCAmelCase    :						Any       =						2
  while i * i <= n:
    __UpperCAmelCase    :						Tuple       =						0
    while n % i == 0:
      n //= i
      multiplicity += 1
    divisors_count *= multiplicity + 1
    i += 1
  if n > 1:
    divisors_count *= 2
  return divisors_count


def       lowerCamelCase_       (   )						->Optional[int]:





  """simple docstring"""


  return next(i for i in triangle_number_generator() if count_divisors(UpperCAmelCase_  ) > 5_00  )


if __name__ == "__main__":
  print(solution())