





import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class       A_			(a_	,				unittest.TestCase     ):
	UpperCAmelCase__   =     PhobertTokenizer
	UpperCAmelCase__   =     False
	def 	_lowercase							(       self						):






				'''simple docstring'''







				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				UpperCAmelCase  						=							['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
				UpperCAmelCase  						=							dict(zip(lowerCamelCase__				,   range(len(lowerCamelCase__						)						)						)						)
				UpperCAmelCase  						=							['''#version: 0.2''', '''l à</w>''']
				UpperCAmelCase  						=							{'''unk_token''': '''<unk>'''}

				UpperCAmelCase  						=							os.path.join(self.tmpdirname				,   VOCAB_FILES_NAMES['''vocab_file''']						)
				UpperCAmelCase  						=							os.path.join(self.tmpdirname				,   VOCAB_FILES_NAMES['''merges_file''']						)

				with open(self.vocab_file				,   '''w'''				,   encoding='''utf-8'''						) as fp:
							for token in vocab_tokens:
										fp.write(F"""{token} {vocab_tokens[token]}\n"""						)
				with open(self.merges_file				,   '''w'''				,   encoding='''utf-8'''						) as fp:
							fp.write('''\n'''.join(lowerCamelCase__						)						)
	def 	_lowercase							(       self				,   **_A						):






				'''simple docstring'''







				kwargs.update(self.special_tokens_map						)
				return PhobertTokenizer.from_pretrained(self.tmpdirname				,   **lowerCamelCase__						)
	def 	_lowercase							(       self				,   _A						):






				'''simple docstring'''







				UpperCAmelCase  						=							'''Tôi là VinAI Research'''
				UpperCAmelCase  						=							'''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
				return input_text, output_text







	def 	_lowercase							(       self						):






				'''simple docstring'''







				UpperCAmelCase  						=							PhobertTokenizer(self.vocab_file				,   self.merges_file				,   **self.special_tokens_map						)
				UpperCAmelCase  						=							'''Tôi là VinAI Research'''
				UpperCAmelCase  						=							'''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
				UpperCAmelCase  						=							tokenizer.tokenize(lowerCamelCase__						)
				print(lowerCamelCase__						)
				self.assertListEqual(lowerCamelCase__				,   lowerCamelCase__						)

				UpperCAmelCase  						=							tokens + [tokenizer.unk_token]

				UpperCAmelCase  						=							[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__						)				,   lowerCamelCase__						)









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A								=       logging.get_logger(__name__)

__A								=       {
    "facebook/xlm-roberta-xl": "https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json",
    "facebook/xlm-roberta-xxl": "https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json",
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			'''xlm-roberta-xl'''


			def __init__( self     ,		lowerCamelCase__=250_880     ,		lowerCamelCase__=2_560     ,		lowerCamelCase__=36     ,		lowerCamelCase__=32     ,		lowerCamelCase__=10_240     ,		lowerCamelCase__="gelu"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=514     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=1e-05     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0     ,		lowerCamelCase__=2     ,		lowerCamelCase__="absolute"     ,		lowerCamelCase__=True     ,		lowerCamelCase__=None     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							super().__init__(pad_token_id=lowerCamelCase__     ,		bos_token_id=lowerCamelCase__     ,		eos_token_id=lowerCamelCase__     ,		**lowerCamelCase__				)
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       hidden_size
							__lowerCamelCase									=       num_hidden_layers
							__lowerCamelCase									=       num_attention_heads
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       intermediate_size
							__lowerCamelCase									=       hidden_dropout_prob
							__lowerCamelCase									=       attention_probs_dropout_prob
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       initializer_range
							__lowerCamelCase									=       layer_norm_eps
							__lowerCamelCase									=       position_embedding_type
							__lowerCamelCase									=       use_cache
							__lowerCamelCase									=       classifier_dropout







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			@property
			def  lowercase_       ( self				)       -> Mapping[str, Mapping[int, str]]:



							'''simple docstring'''
							if self.task == "multiple-choice":
											__lowerCamelCase									=       {0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
											__lowerCamelCase									=       {0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]				)


