
"""simple docstring"""


import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer







class 		lowerCamelCase__					(					tf.keras.layers.Layer				):





			'''simple docstring'''



			def __init__(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase = None     ,     __UpperCAmelCase = None     ,     __UpperCAmelCase = None     ,     __UpperCAmelCase = "longest"     ,     __UpperCAmelCase = True     ,     __UpperCAmelCase = 5_12     ,     __UpperCAmelCase = None     ,     __UpperCAmelCase = True     ,     __UpperCAmelCase = True     ,     __UpperCAmelCase = True     ,     **__UpperCAmelCase     ,     ) ->     Tuple:
				super().__init__()
				if use_fast_bert_tokenizer:
					_lowerCAmelCase								=FastBertTokenizer(
					    __UpperCAmelCase     ,     token_out_type=tf.intaa     ,     lower_case_nfd_strip_accents=__UpperCAmelCase     ,     **__UpperCAmelCase  )
				else:
					_lowerCAmelCase								=tf.lookup.StaticVocabularyTable(
					    tf.lookup.KeyValueTensorInitializer(
					        keys=__UpperCAmelCase     ,     key_dtype=tf.string     ,     values=tf.range(tf.size(__UpperCAmelCase     ,     out_type=tf.intaa  )     ,     dtype=tf.intaa  )     ,     value_dtype=tf.intaa     ,     )     ,     num_oov_buckets=1     ,     )
					_lowerCAmelCase								=BertTokenizerLayer(
					    __UpperCAmelCase     ,     token_out_type=tf.intaa     ,     lower_case=__UpperCAmelCase     ,     **__UpperCAmelCase  )

				_lowerCAmelCase								=vocab_list
				_lowerCAmelCase								=do_lower_case
				_lowerCAmelCase								=cls_token_id or vocab_list.index("""[CLS]"""  )
				_lowerCAmelCase								=sep_token_id or vocab_list.index("""[SEP]"""  )
				_lowerCAmelCase								=pad_token_id or vocab_list.index("""[PAD]"""  )
				_lowerCAmelCase								=ShrinkLongestTrimmer(max_length - 3     ,     axis=1  )  # Allow room for special tokens
				_lowerCAmelCase								=max_length
				_lowerCAmelCase								=padding
				_lowerCAmelCase								=truncation
				_lowerCAmelCase								=pad_to_multiple_of
				_lowerCAmelCase								=return_token_type_ids
				_lowerCAmelCase								=return_attention_mask



			@classmethod
			def 					_lowerCAmelCase		(							cls     ,     __UpperCAmelCase     ,     **__UpperCAmelCase  ) ->     Dict:  # noqa: F821
				_lowerCAmelCase								=kwargs.pop("""do_lower_case"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=tokenizer.do_lower_case if do_lower_case is None else do_lower_case
				_lowerCAmelCase								=kwargs.pop("""cls_token_id"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=tokenizer.cls_token_id if cls_token_id is None else cls_token_id
				_lowerCAmelCase								=kwargs.pop("""sep_token_id"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=tokenizer.sep_token_id if sep_token_id is None else sep_token_id
				_lowerCAmelCase								=kwargs.pop("""pad_token_id"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=tokenizer.pad_token_id if pad_token_id is None else pad_token_id

				_lowerCAmelCase								=tokenizer.get_vocab()
				_lowerCAmelCase								=sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]     ,     key=lambda __UpperCAmelCase  : x[1]  )
				_lowerCAmelCase								=[entry[0] for entry in vocab]
				return cls(
				    vocab_list=__UpperCAmelCase     ,     do_lower_case=__UpperCAmelCase     ,     cls_token_id=__UpperCAmelCase     ,     sep_token_id=__UpperCAmelCase     ,     pad_token_id=__UpperCAmelCase     ,     **__UpperCAmelCase     ,     )



			@classmethod
			def 					_lowerCAmelCase		(							cls     ,     __UpperCAmelCase     ,     *__UpperCAmelCase     ,     **__UpperCAmelCase  ) ->     Union[str, Any]:
				try:
					_lowerCAmelCase								=BertTokenizer.from_pretrained(__UpperCAmelCase     ,     *__UpperCAmelCase     ,     **__UpperCAmelCase  )
				except:  # noqa: E722
					from .tokenization_bert_fast import BertTokenizerFast

					_lowerCAmelCase								=BertTokenizerFast.from_pretrained(__UpperCAmelCase     ,     *__UpperCAmelCase     ,     **__UpperCAmelCase  )
				return cls.from_tokenizer(__UpperCAmelCase     ,     **__UpperCAmelCase  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Dict:
				if self.do_lower_case:
					_lowerCAmelCase								=case_fold_utfa(__UpperCAmelCase  )
				_lowerCAmelCase								=self.tf_tokenizer.tokenize(__UpperCAmelCase  )
				return tokens.merge_dims(1     ,     -1  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     ) ->     Union[str, Any]:
				if padding is None:
					_lowerCAmelCase								=self.padding
				if padding not in ("longest", "max_length"):
					raise ValueError("""Padding must be either 'longest' or 'max_length'!"""  )
				if max_length is not None and text_pair is not None:
					# Because we have to instantiate a Trimmer to do it properly
					raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""  )
				if max_length is None:
					_lowerCAmelCase								=self.max_length
				if truncation is None:
					_lowerCAmelCase								=self.truncation
				if pad_to_multiple_of is None:
					_lowerCAmelCase								=self.pad_to_multiple_of
				if return_token_type_ids is None:
					_lowerCAmelCase								=self.return_token_type_ids
				if return_attention_mask is None:
					_lowerCAmelCase								=self.return_attention_mask
				if not isinstance(__UpperCAmelCase     ,     tf.Tensor  ):
					_lowerCAmelCase								=tf.convert_to_tensor(__UpperCAmelCase  )
				if text_pair is not None and not isinstance(__UpperCAmelCase     ,     tf.Tensor  ):
					_lowerCAmelCase								=tf.convert_to_tensor(__UpperCAmelCase  )
				if text_pair is not None:
					if text.shape.rank > 1:
						raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""  )
					if text_pair.shape.rank > 1:
						raise ValueError("""text_pair should not be multidimensional!"""  )
				if text.shape.rank == 2:
					_lowerCAmelCase						,							_lowerCAmelCase								=text[:, 0], text[:, 1]
				_lowerCAmelCase								=self.unpaired_tokenize(__UpperCAmelCase  )
				if text_pair is None:  # Unpaired text
					if truncation:
						_lowerCAmelCase								=text[:, : max_length - 2]  # Allow room for special tokens
					_lowerCAmelCase						,							_lowerCAmelCase								=combine_segments(
					    (text,)     ,     start_of_sequence_id=self.cls_token_id     ,     end_of_segment_id=self.sep_token_id  )
				else:  # Paired text
					_lowerCAmelCase								=self.unpaired_tokenize(__UpperCAmelCase  )
					if truncation:
						_lowerCAmelCase						,							_lowerCAmelCase								=self.paired_trimmer.trim([text, text_pair]  )
					_lowerCAmelCase						,							_lowerCAmelCase								=combine_segments(
					    (text, text_pair)     ,     start_of_sequence_id=self.cls_token_id     ,     end_of_segment_id=self.sep_token_id  )
				if padding == "longest":
					_lowerCAmelCase								=input_ids.bounding_shape(axis=1  )
					if pad_to_multiple_of is not None:
						# No ceiling division in tensorflow, so we negate floordiv instead
						_lowerCAmelCase								=pad_to_multiple_of * (-tf.math.floordiv(-pad_length     ,     __UpperCAmelCase  ))
				else:
					_lowerCAmelCase								=max_length

				_lowerCAmelCase						,							_lowerCAmelCase								=pad_model_inputs(__UpperCAmelCase     ,     max_seq_length=__UpperCAmelCase     ,     pad_value=self.pad_token_id  )
				_lowerCAmelCase								={"""input_ids""": input_ids}
				if return_attention_mask:
					_lowerCAmelCase								=attention_mask
				if return_token_type_ids:
					_lowerCAmelCase						,							_lowerCAmelCase								=pad_model_inputs(
					    __UpperCAmelCase     ,     max_seq_length=__UpperCAmelCase     ,     pad_value=self.pad_token_id  )
					_lowerCAmelCase								=token_type_ids
				return output



			def 					_lowerCAmelCase		(							self  ) ->     List[Any]:
				return {
				    "vocab_list": self.vocab_list,
				    "do_lower_case": self.do_lower_case,
				    "cls_token_id": self.cls_token_id,
				    "sep_token_id": self.sep_token_id,
				    "pad_token_id": self.pad_token_id,
				}





"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
			__A       						=       None


__A       						=       logging.get_logger(__name__)

__A       						=       {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__A       						=       {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

__A       						=       {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}







class 		lowerCamelCase__					(					__magic_name__				):





			'''simple docstring'''
			lowerCamelCase  					=		VOCAB_FILES_NAMES
			lowerCamelCase  					=		PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase  					=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase  					=		['''input_ids''', '''attention_mask''']
			lowerCamelCase  					=		XLMRobertaTokenizer



			def __init__(							self     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase="<s>"     ,     __UpperCAmelCase="</s>"     ,     __UpperCAmelCase="</s>"     ,     __UpperCAmelCase="<s>"     ,     __UpperCAmelCase="<unk>"     ,     __UpperCAmelCase="<pad>"     ,     __UpperCAmelCase="<mask>"     ,     **__UpperCAmelCase     ,     ) ->     str:
				# Mask token behave like a normal word, i.e. include the space before it
				_lowerCAmelCase								=AddedToken(__UpperCAmelCase     ,     lstrip=__UpperCAmelCase     ,     rstrip=__UpperCAmelCase  ) if isinstance(__UpperCAmelCase     ,     __UpperCAmelCase  ) else mask_token

				super().__init__(
				    __UpperCAmelCase     ,     tokenizer_file=__UpperCAmelCase     ,     bos_token=__UpperCAmelCase     ,     eos_token=__UpperCAmelCase     ,     sep_token=__UpperCAmelCase     ,     cls_token=__UpperCAmelCase     ,     unk_token=__UpperCAmelCase     ,     pad_token=__UpperCAmelCase     ,     mask_token=__UpperCAmelCase     ,     **__UpperCAmelCase     ,     )

				_lowerCAmelCase								=vocab_file
				_lowerCAmelCase								=False if not self.vocab_file else True



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase = None  ) ->     List[int]:

				if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				_lowerCAmelCase								=[self.cls_token_id]
				_lowerCAmelCase								=[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase = None  ) ->     List[int]:
				_lowerCAmelCase								=[self.sep_token_id]
				_lowerCAmelCase								=[self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep  ) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase = None  ) ->     Tuple[str]:
				if not self.can_save_slow_tokenizer:
					raise ValueError(
					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
					    """tokenizer."""  )

				if not os.path.isdir(__UpperCAmelCase  ):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''  )
					return
				_lowerCAmelCase								=os.path.join(
				    __UpperCAmelCase     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

				if os.path.abspath(self.vocab_file  ) != os.path.abspath(__UpperCAmelCase  ):
					copyfile(self.vocab_file     ,     __UpperCAmelCase  )

				return (out_vocab_file,)



