


'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__UpperCAmelCase       							=				logging.get_logger(__name__)

__UpperCAmelCase       							=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__UpperCAmelCase       							=				[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

__UpperCAmelCase       							=				{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
__UpperCAmelCase       							=				{f"""funnel-transformer/{name}""": 512 for name in _model_names}
__UpperCAmelCase       							=				{f"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}




class 						a__							(				a__    ):





							'''simple docstring'''






							lowercase__     :  Union[str, Any]      		= VOCAB_FILES_NAMES
							lowercase__     :  List[str]      		= PRETRAINED_VOCAB_FILES_MAP
							lowercase__     :  str      		= PRETRAINED_INIT_CONFIGURATION
							lowercase__     :  str      		= FunnelTokenizer
							lowercase__     :  Any      		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__     :  Any      		= 2







							def __init__(						self   ,       lowerCamelCase_=None   ,       lowerCamelCase_=None   ,       lowerCamelCase_=True   ,       lowerCamelCase_="<unk>"   ,       lowerCamelCase_="<sep>"   ,       lowerCamelCase_="<pad>"   ,       lowerCamelCase_="<cls>"   ,       lowerCamelCase_="<mask>"   ,       lowerCamelCase_="<s>"   ,       lowerCamelCase_="</s>"   ,       lowerCamelCase_=True   ,       lowerCamelCase_=True   ,       lowerCamelCase_=None   ,       lowerCamelCase_="##"   ,       **lowerCamelCase_   ,       )			->				Optional[Any]:
														super().__init__(
														    snake_case__   ,       tokenizer_file=snake_case__   ,       do_lower_case=snake_case__   ,       unk_token=snake_case__   ,       sep_token=snake_case__   ,       pad_token=snake_case__   ,       cls_token=snake_case__   ,       mask_token=snake_case__   ,       bos_token=snake_case__   ,       eos_token=snake_case__   ,       clean_text=snake_case__   ,       tokenize_chinese_chars=snake_case__   ,       strip_accents=snake_case__   ,       wordpieces_prefix=snake_case__   ,       **snake_case__   ,       )

														lowerCAmelCase__			  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
														if (
														    normalizer_state.get('''lowercase'''   ,       snake_case__  ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''   ,       snake_case__  ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''   ,       snake_case__  ) != tokenize_chinese_chars
														):
																					lowerCAmelCase__			  =      getattr(snake_case__   ,       normalizer_state.pop('''type'''  )  )
																					lowerCAmelCase__			  =      do_lower_case
																					lowerCAmelCase__			  =      strip_accents
																					lowerCAmelCase__			  =      tokenize_chinese_chars
																					lowerCAmelCase__			  =      normalizer_class(**snake_case__  )

														lowerCAmelCase__			  =      do_lower_case







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_=None  )			->				int:
														lowerCAmelCase__			  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				List[str]:
														lowerCAmelCase__			  =      [self.sep_token_id]
														lowerCAmelCase__			  =      [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0]
														return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_ = None  )			->				List[Any]:
														lowerCAmelCase__			  =      self._tokenizer.model.save(snake_case__   ,       name=snake_case__  )
														return tuple(snake_case__  )







"""simple docstring"""



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
 import torch

 from transformers import XLMRobertaModel




@require_sentencepiece
@require_tokenizers
@require_torch
class 			lowerCAmelCase__			(    unittest.TestCase   ):
   @slow
   def       __a							(		self      :	Tuple							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[str]							=							XLMRobertaModel.from_pretrained("xlm-roberta-base"							)
        UpperCAmelCase__						:    str							=							torch.tensor([[0, 5_8_1, 1_0_2_6_9, 8_3, 9_9_9_4_2, 1_3_6, 6_0_7_4_2, 2_3, 7_0, 8_0_5_8_3, 1_8_2_7_6, 2]]							)
        # The dog is cute and lives in the garden house

        UpperCAmelCase__						:    Tuple							=							torch.Size((1, 1_2, 7_6_8)							)  # batch_size, sequence_length, embedding_vector_dim
        UpperCAmelCase__						:    Optional[int]							=							torch.tensor(
            [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]							)
        #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
        #  xlmr.eval()
        #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
        with torch.no_grad():
             UpperCAmelCase__						:    int							=							model(snake_case__							)["last_hidden_state"].detach()
        self.assertEqual(output.shape							,			snake_case__							)
        # compare the actual values for a slice of last dim
        self.assertTrue(torch.allclose(output[:, :, -1]							,			snake_case__							,			atol=1e-3							)							)




   @slow
   def       __a							(		self      :	Optional[Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Any							=							XLMRobertaModel.from_pretrained("xlm-roberta-large"							)
        UpperCAmelCase__						:    Tuple							=							torch.tensor([[0, 5_8_1, 1_0_2_6_9, 8_3, 9_9_9_4_2, 1_3_6, 6_0_7_4_2, 2_3, 7_0, 8_0_5_8_3, 1_8_2_7_6, 2]]							)
        # The dog is cute and lives in the garden house

        UpperCAmelCase__						:    Union[str, Any]							=							torch.Size((1, 1_2, 1_0_2_4)							)  # batch_size, sequence_length, embedding_vector_dim
        UpperCAmelCase__						:    Optional[Any]							=							torch.tensor(
            [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]							)
        #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
        #  xlmr.eval()
        #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
        with torch.no_grad():
             UpperCAmelCase__						:    Optional[Any]							=							model(snake_case__							)["last_hidden_state"].detach()
        self.assertEqual(output.shape							,			snake_case__							)
        # compare the actual values for a slice of last dim
        self.assertTrue(torch.allclose(output[:, :, -1]							,			snake_case__							,			atol=1e-3							)							)


