


'''simple docstring'''







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class  __lowerCamelCase (       a_				,					unittest.TestCase       ):
    """simple docstring"""
    a													=    PreTrainedTokenizerFast
    a													=    False
    a													=    True
    a													=    "tokenizer_file"


    def    A    (  self			:							Optional[int]):
           _A     :	List[Any]      =					False  # because we don't have pretrained_vocab_files_map
           super().setUp()
           _A     :	Optional[Any]      =					True

           _A     :	str      =					['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
           _A     :	Dict      =					'SaulLu/dummy-tokenizer-bytelevel-bpe'

           # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
           _A     :	List[str]      =					[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

           _A     :	Optional[Any]      =					PreTrainedTokenizerFast.from_pretrained(model_paths[0])
           tokenizer.save_pretrained(self.tmpdirname)


    def    A    (  self			:							Any):
           # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
           # model
           pass


    def    A    (  self			:							Union[str, Any]):
           # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
           # model
           pass


    def    A    (  self			:							List[str]):
           # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
           # model
           pass


    def    A    (  self			:							int):
           # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
           pass


    def    A    (  self			:							int):
           _A     :	int      =					self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                         try:
                                _A     :	Optional[int]      =					tempfile.mkdtemp()
                                _A     :	str      =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE      ,							**SCREAMING_SNAKE_CASE)

                                tokenizer.save_pretrained(self.tmpdirname)
                                super().test_training_new_tokenizer()
                         finally:
                                # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                                # is restored
                                shutil.rmtree(self.tmpdirname)
                                _A     :	int      =					tmpdirname_orig


    def    A    (  self			:							List[str]):
           _A     :	int      =					self.tmpdirname
           # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
                         try:
                                _A     :	str      =					tempfile.mkdtemp()
                                _A     :	str      =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE      ,							**SCREAMING_SNAKE_CASE)

                                tokenizer.save_pretrained(self.tmpdirname)
                                super().test_training_new_tokenizer_with_special_tokens_change()
                         finally:
                                # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                                # is restored
                                shutil.rmtree(self.tmpdirname)
                                _A     :	List[str]      =					tmpdirname_orig


    def    A    (  self			:							int):
           _A     :	Tuple      =					self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

           _A     :	Optional[int]      =					('a' for _ in range(1000))
           _A     :	int      =					tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE      ,							length=1000      ,							vocab_size=50)

           _A     :	Optional[int]      =					new_tokenizer.encode('a🤗')
           self.assertEqual(SCREAMING_SNAKE_CASE      ,							[64, 172, 253, 97, 245])




@require_tokenizers
class  __lowerCamelCase (       unittest.TestCase       ):
    """simple docstring"""


    def    A    (  self			:							str):
           _A     :	Any      =					AutoTokenizer.from_pretrained('bert-base-cased')
           _A     :	Any      =					json.loads(tokenizer._tokenizer.to_str())
           _A     :	Dict      =					len(SCREAMING_SNAKE_CASE)

           with tempfile.TemporaryDirectory() as tmp_dir:
                  # Hack to save this in the tokenizer_config.json
                  _A     :	Dict      =					['tokenizer.4.0.0.json']
                  tokenizer.save_pretrained(SCREAMING_SNAKE_CASE)
                  json.dump(SCREAMING_SNAKE_CASE      ,							open(os.path.join(SCREAMING_SNAKE_CASE      ,							'tokenizer.4.0.0.json')      ,							'w'))

                  # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                  _A     :	Optional[int]      =					AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE)
                  self.assertEqual(len(SCREAMING_SNAKE_CASE)      ,							len(SCREAMING_SNAKE_CASE) + 1)
                  _A     :	Any      =					json.loads(new_tokenizer._tokenizer.to_str())
                  self.assertIn('huggingface'      ,							json_tokenizer['model']['vocab'])

                  # Will need to be adjusted if we reach v42 and this test is still here.
                  # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                  shutil.move(os.path.join(SCREAMING_SNAKE_CASE      ,							'tokenizer.4.0.0.json')      ,							os.path.join(SCREAMING_SNAKE_CASE      ,							'tokenizer.42.0.0.json'))
                  _A     :	Union[str, Any]      =					['tokenizer.42.0.0.json']
                  tokenizer.save_pretrained(SCREAMING_SNAKE_CASE)
                  _A     :	int      =					AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE)
                  self.assertEqual(len(SCREAMING_SNAKE_CASE)      ,							len(SCREAMING_SNAKE_CASE))
                  _A     :	Optional[Any]      =					json.loads(new_tokenizer._tokenizer.to_str())
                  self.assertNotIn('huggingface'      ,							json_tokenizer['model']['vocab'])


    def    A    (  self			:							List[str]):
           # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
           _A     :	Dict      =					'hf-internal-testing/test-two-tokenizers'

           # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
           _A     :	Tuple      =					AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE)
           self.assertEqual(len(SCREAMING_SNAKE_CASE)      ,							28997)
           _A     :	str      =					json.loads(tokenizer._tokenizer.to_str())
           self.assertIn('huggingface'      ,							json_tokenizer['model']['vocab'])

           # Testing an older version by monkey-patching the version in the module it's used.
           import transformers as old_transformers

           _A     :	Optional[int]      =					'3.0.0'
           _A     :	List[Any]      =					old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE)
           self.assertEqual(len(SCREAMING_SNAKE_CASE)      ,							28996)
           _A     :	Optional[Any]      =					json.loads(old_tokenizer._tokenizer.to_str())
           self.assertNotIn('huggingface'      ,							json_tokenizer['model']['vocab'])




@require_tokenizers
class  __lowerCamelCase (       unittest.TestCase       ):
    """simple docstring"""


    def    A    (  self			:							Tuple):
           # See https://github.com/huggingface/transformers/pull/12550
           # and https://github.com/huggingface/tokenizers/issues/537
           _A     :	Tuple      =					PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel')
           _A     :	str      =					'The Matrix is a 1999 science fiction action film.'

           with concurrent.futures.ThreadPoolExecutor() as executor:
                  _A     :	int      =					[executor.submit(self.fetch      ,							SCREAMING_SNAKE_CASE      ,							SCREAMING_SNAKE_CASE) for i in range(10)]
                  _A     :	str      =					[future.result() for future in futures]
                  self.assertEqual(SCREAMING_SNAKE_CASE      ,							[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10)])


    def    A    (  self			:							Tuple      ,							SCREAMING_SNAKE_CASE			:							Optional[Any]      ,							SCREAMING_SNAKE_CASE			:							Optional[int]):
           return tokenizer.encode(SCREAMING_SNAKE_CASE      ,							truncation='longest_first'      ,							padding='longest')










'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
   A      :     Optional[int]										= None


A      :     Optional[Any]										= logging.get_logger(__name__)

A      :     Dict										= {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

A      :     Optional[int]										= {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'''
        ),
    },
}

A      :     List[str]										= {
    '''xlm-roberta-base''': 512,
    '''xlm-roberta-large''': 512,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 512,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 512,
    '''xlm-roberta-large-finetuned-conll03-english''': 512,
    '''xlm-roberta-large-finetuned-conll03-german''': 512,
}


class  __lowerCamelCase (       a_       ):
    """simple docstring"""
    a													=    VOCAB_FILES_NAMES
    a													=    PRETRAINED_VOCAB_FILES_MAP
    a													=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a													=    ["input_ids", "attention_mask"]
    a													=    XLMRobertaTokenizer


    def __init__(  self			:							Union[str, Any]      ,							SCREAMING_SNAKE_CASE			:							Dict=None      ,							SCREAMING_SNAKE_CASE			:							List[Any]=None      ,							SCREAMING_SNAKE_CASE			:							Any="<s>"      ,							SCREAMING_SNAKE_CASE			:							Tuple="</s>"      ,							SCREAMING_SNAKE_CASE			:							str="</s>"      ,							SCREAMING_SNAKE_CASE			:							Optional[Any]="<s>"      ,							SCREAMING_SNAKE_CASE			:							Any="<unk>"      ,							SCREAMING_SNAKE_CASE			:							Union[str, Any]="<pad>"      ,							SCREAMING_SNAKE_CASE			:							List[Any]="<mask>"      ,							**SCREAMING_SNAKE_CASE			:							int      ,							):
           # Mask token behave like a normal word, i.e. include the space before it
           _A     :	str      =					AddedToken(SCREAMING_SNAKE_CASE      ,							lstrip=SCREAMING_SNAKE_CASE      ,							rstrip=SCREAMING_SNAKE_CASE) if isinstance(SCREAMING_SNAKE_CASE      ,							SCREAMING_SNAKE_CASE) else mask_token

           super().__init__(
               SCREAMING_SNAKE_CASE      ,							tokenizer_file=SCREAMING_SNAKE_CASE      ,							bos_token=SCREAMING_SNAKE_CASE      ,							eos_token=SCREAMING_SNAKE_CASE      ,							sep_token=SCREAMING_SNAKE_CASE      ,							cls_token=SCREAMING_SNAKE_CASE      ,							unk_token=SCREAMING_SNAKE_CASE      ,							pad_token=SCREAMING_SNAKE_CASE      ,							mask_token=SCREAMING_SNAKE_CASE      ,							**SCREAMING_SNAKE_CASE      ,							)

           _A     :	Dict      =					vocab_file
           _A     :	Optional[Any]      =					False if not self.vocab_file else True


    def    A    (  self			:							int      ,							SCREAMING_SNAKE_CASE			:							List[int]      ,							SCREAMING_SNAKE_CASE			:							Optional[List[int]] = None):

           if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           _A     :	Optional[Any]      =					[self.cls_token_id]
           _A     :	Optional[int]      =					[self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep


    def    A    (  self			:							List[str]      ,							SCREAMING_SNAKE_CASE			:							List[int]      ,							SCREAMING_SNAKE_CASE			:							Optional[List[int]] = None):
           _A     :	Any      =					[self.sep_token_id]
           _A     :	Optional[Any]      =					[self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


    def    A    (  self			:							Dict      ,							SCREAMING_SNAKE_CASE			:							str      ,							SCREAMING_SNAKE_CASE			:							Optional[str] = None):
           if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                      'tokenizer.')

           if not os.path.isdir(SCREAMING_SNAKE_CASE):
                  logger.error(F'Vocabulary path ({save_directory}) should be a directory.')
                  return
           _A     :	Any      =					os.path.join(
               SCREAMING_SNAKE_CASE      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

           if os.path.abspath(self.vocab_file) != os.path.abspath(SCREAMING_SNAKE_CASE):
                  copyfile(self.vocab_file      ,							SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)






