


'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_      =			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
SCREAMING_SNAKE_CASE_      =			{f"""funnel-transformer/{name}""": 5_12 for name in _model_names}
SCREAMING_SNAKE_CASE_      =			{f"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :Any												=VOCAB_FILES_NAMES
  a_       :Optional[int]												=PRETRAINED_VOCAB_FILES_MAP
  a_       :Optional[Any]												=PRETRAINED_INIT_CONFIGURATION
  a_       :Union[str, Any]												=FunnelTokenizer
  a_       :Tuple												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_       :int												=2

  def __init__(					self     :							Optional[int]				,							SCREAMING_SNAKE_CASE__     :							List[str]=None				,							SCREAMING_SNAKE_CASE__     :							Any=None				,							SCREAMING_SNAKE_CASE__     :							Dict=True				,							SCREAMING_SNAKE_CASE__     :							Any="<unk>"				,							SCREAMING_SNAKE_CASE__     :							str="<sep>"				,							SCREAMING_SNAKE_CASE__     :							Dict="<pad>"				,							SCREAMING_SNAKE_CASE__     :							Any="<cls>"				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]="<mask>"				,							SCREAMING_SNAKE_CASE__     :							int="<s>"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="</s>"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=True				,							SCREAMING_SNAKE_CASE__     :							Any=True				,							SCREAMING_SNAKE_CASE__     :							Any=None				,							SCREAMING_SNAKE_CASE__     :							List[Any]="##"				,							**SCREAMING_SNAKE_CASE__     :							Any				,							):





       '''simple docstring'''

       super().__init__(
           SCREAMING_SNAKE_CASE__				,							tokenizer_file=SCREAMING_SNAKE_CASE__				,							do_lower_case=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							bos_token=SCREAMING_SNAKE_CASE__				,							eos_token=SCREAMING_SNAKE_CASE__				,							clean_text=SCREAMING_SNAKE_CASE__				,							tokenize_chinese_chars=SCREAMING_SNAKE_CASE__				,							strip_accents=SCREAMING_SNAKE_CASE__				,							wordpieces_prefix=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
       if (
           normalizer_state.get("""lowercase"""				,							SCREAMING_SNAKE_CASE__						) != do_lower_case
           or normalizer_state.get("""strip_accents"""				,							SCREAMING_SNAKE_CASE__						) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""				,							SCREAMING_SNAKE_CASE__						) != tokenize_chinese_chars
       ):
            __a       =   getattr(SCREAMING_SNAKE_CASE__				,							normalizer_state.pop("""type"""						)						)
            __a       =   do_lower_case
            __a       =   strip_accents
            __a       =   tokenize_chinese_chars
            __a       =   normalizer_class(**SCREAMING_SNAKE_CASE__						)

       __a       =   do_lower_case

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							Any=None						):





       '''simple docstring'''

       __a       =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]
       if token_ids_a is None:
            return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
       return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def __a						(					self     :							Dict				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       __a       =   self._tokenizer.model.save(SCREAMING_SNAKE_CASE__				,							name=SCREAMING_SNAKE_CASE__						)
       return tuple(SCREAMING_SNAKE_CASE__						)










'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
 import torch

 from transformers import XLMRobertaModel


@require_sentencepiece
@require_tokenizers
@require_torch
class  lowerCAmelCase_						(       unittest.TestCase      ):





  """simple docstring"""



  @slow
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   XLMRobertaModel.from_pretrained("""xlm-roberta-base"""						)
       __a       =   torch.tensor([[0, 5_8_1, 1_0_2_6_9, 8_3, 9_9_9_4_2, 1_3_6, 6_0_7_4_2, 2_3, 7_0, 8_0_5_8_3, 1_8_2_7_6, 2]]						)
       # The dog is cute and lives in the garden house

       __a       =   torch.Size((1, 1_2, 7_6_8)						)  # batch_size, sequence_length, embedding_vector_dim
       __a       =   torch.tensor(
           [[-0.0_1_0_1, 0.1_2_1_8, -0.0_8_0_3, 0.0_8_0_1, 0.1_3_2_7, 0.0_7_7_6, -0.1_2_1_5, 0.2_3_8_3, 0.3_3_3_8, 0.3_1_0_6, 0.0_3_0_0, 0.0_2_5_2]]						)
       #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
       #  xlmr.eval()
       #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
       with torch.no_grad():
            __a       =   model(SCREAMING_SNAKE_CASE__						)["""last_hidden_state"""].detach()
       self.assertEqual(output.shape				,							SCREAMING_SNAKE_CASE__						)
       # compare the actual values for a slice of last dim
       self.assertTrue(torch.allclose(output[:, :, -1]				,							SCREAMING_SNAKE_CASE__				,							atol=1E-3						)						)


  @slow
  def __a						(					self     :							int						):





       '''simple docstring'''

       __a       =   XLMRobertaModel.from_pretrained("""xlm-roberta-large"""						)
       __a       =   torch.tensor([[0, 5_8_1, 1_0_2_6_9, 8_3, 9_9_9_4_2, 1_3_6, 6_0_7_4_2, 2_3, 7_0, 8_0_5_8_3, 1_8_2_7_6, 2]]						)
       # The dog is cute and lives in the garden house

       __a       =   torch.Size((1, 1_2, 1_0_2_4)						)  # batch_size, sequence_length, embedding_vector_dim
       __a       =   torch.tensor(
           [[-0.0_6_9_9, -0.0_3_1_8, 0.0_7_0_5, -0.1_2_4_1, 0.0_9_9_9, -0.0_5_2_0, 0.1_0_0_4, -0.1_8_3_8, -0.4_7_0_4, 0.1_4_3_7, 0.0_8_2_1, 0.0_1_2_6]]						)
       #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
       #  xlmr.eval()
       #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
       with torch.no_grad():
            __a       =   model(SCREAMING_SNAKE_CASE__						)["""last_hidden_state"""].detach()
       self.assertEqual(output.shape				,							SCREAMING_SNAKE_CASE__						)
       # compare the actual values for a slice of last dim
       self.assertTrue(torch.allclose(output[:, :, -1]				,							SCREAMING_SNAKE_CASE__				,							atol=1E-3						)						)






