





"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A 			=  logging.get_logger(__name__)

A 			=  '''▁'''

A 			=  {'''vocab_file''': '''sentencepiece.bpe.model'''}

A 			=  {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

A 			=  {
    '''facebook/mbart-large-en-ro''': 1_024,
    '''facebook/mbart-large-cc25''': 1_024,
}

# fmt: off
A 			=  ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']




class 				__lowercase			(			_UpperCamelCase							):







     '''simple docstring'''




     __lowerCAmelCase  =  VOCAB_FILES_NAMES
     __lowerCAmelCase  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCAmelCase  =  PRETRAINED_VOCAB_FILES_MAP
     __lowerCAmelCase  =  ['''input_ids''', '''attention_mask''']

     __lowerCAmelCase  =  []
     __lowerCAmelCase  =  []







     def __init__(						self	,			_UpperCAmelCase	,			_UpperCAmelCase="<s>"	,			_UpperCAmelCase="</s>"	,			_UpperCAmelCase="</s>"	,			_UpperCAmelCase="<s>"	,			_UpperCAmelCase="<unk>"	,			_UpperCAmelCase="<pad>"	,			_UpperCAmelCase="<mask>"	,			_UpperCAmelCase=None	,			_UpperCAmelCase=None	,			_UpperCAmelCase=None	,			_UpperCAmelCase = None	,			_UpperCAmelCase=None	,			**_UpperCAmelCase	,			):
         # Mask token behave like a normal word, i.e. include the space before it
         __a :						List[str]			       =  AddedToken(_UpperCAmelCase	,			lstrip=_UpperCAmelCase	,			rstrip=_UpperCAmelCase     ) if isinstance(_UpperCAmelCase	,			_UpperCAmelCase     ) else mask_token

         __a :						Tuple			       =  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=_UpperCAmelCase	,			eos_token=_UpperCAmelCase	,			unk_token=_UpperCAmelCase	,			sep_token=_UpperCAmelCase	,			cls_token=_UpperCAmelCase	,			pad_token=_UpperCAmelCase	,			mask_token=_UpperCAmelCase	,			tokenizer_file=_UpperCAmelCase	,			src_lang=_UpperCAmelCase	,			tgt_lang=_UpperCAmelCase	,			additional_special_tokens=_UpperCAmelCase	,			sp_model_kwargs=self.sp_model_kwargs	,			**_UpperCAmelCase	,			)

         __a :						List[str]			       =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
         self.sp_model.Load(str(_UpperCAmelCase     )     )
         __a :						Tuple			       =  vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         __a :						Optional[Any]			       =  {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         __a :						List[Any]			       =  1

         __a :						Dict			       =  len(self.sp_model     )
         __a :						Any			       =  {
             code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCAmelCase     )
         }
         __a :						Optional[int]			       =  {v: k for k, v in self.lang_code_to_id.items()}
         __a :						Optional[int]			       =  len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

         self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
         __a :						List[Any]			       =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}
         __a :						List[Any]			       =  list(self.lang_code_to_id.keys()     )

         if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             self._additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in self._additional_special_tokens]     )

         __a :						Any			       =  src_lang if src_lang is not None else '''en_XX'''
         __a :						List[Any]			       =  self.lang_code_to_id[self._src_lang]
         __a :						str			       =  tgt_lang
         self.set_src_lang_special_tokens(self._src_lang     )







     def __getstate__(						self     ):
         __a :						Dict			       =  self.__dict__.copy()
         __a :						str			       =  None
         __a :						Tuple			       =  self.sp_model.serialized_model_proto()
         return state







     def __setstate__(						self	,			_UpperCAmelCase     ):
         __a :						Optional[Any]			       =  d

         # for backward compatibility
         if not hasattr(self	,			'''sp_model_kwargs'''     ):
             __a :						str			       =  {}

         __a :						List[str]			       =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto     )







     @property
     def  _lowerCamelCase		(						self     ):
         return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token







     @property
     def  _lowerCamelCase		(						self     ):
         return self._src_lang







     @src_lang.setter
     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         __a :						str			       =  new_src_lang
         self.set_src_lang_special_tokens(self._src_lang     )







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = None	,			_UpperCAmelCase = False     ):

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_UpperCAmelCase	,			token_ids_a=_UpperCAmelCase	,			already_has_special_tokens=_UpperCAmelCase     )

         __a :						List[str]			       =  [1] * len(self.prefix_tokens     )
         __a :						Any			       =  [1] * len(self.suffix_tokens     )
         if token_ids_a is None:
             return prefix_ones + ([0] * len(_UpperCAmelCase     )) + suffix_ones
         return prefix_ones + ([0] * len(_UpperCAmelCase     )) + ([0] * len(_UpperCAmelCase     )) + suffix_ones







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = None     ):
         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = None     ):
         __a :						Union[str, Any]			       =  [self.sep_token_id]
         __a :						List[str]			       =  [self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep     ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			**_UpperCAmelCase     ):
         if src_lang is None or tgt_lang is None:
             raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''     )
         __a :						Optional[int]			       =  src_lang
         __a :						List[Any]			       =  self(_UpperCAmelCase	,			add_special_tokens=_UpperCAmelCase	,			return_tensors=_UpperCAmelCase	,			**_UpperCAmelCase     )
         __a :						Any			       =  self.convert_tokens_to_ids(_UpperCAmelCase     )
         __a :						Any			       =  tgt_lang_id
         return inputs







     def  _lowerCamelCase		(						self     ):
         __a :						int			       =  {self.convert_ids_to_tokens(_UpperCAmelCase     ): i for i in range(self.vocab_size     )}
         vocab.update(self.added_tokens_encoder     )
         return vocab







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         return self.sp_model.encode(_UpperCAmelCase	,			out_type=_UpperCAmelCase     )







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
         __a :						Any			       =  self.sp_model.PieceToId(_UpperCAmelCase     )

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset     )







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         __a :						Dict			       =  ''''''.join(_UpperCAmelCase     ).replace(_UpperCAmelCase	,			''' '''     ).strip()
         return out_string







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = None     ):
         if not os.path.isdir(_UpperCAmelCase     ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
             return
         __a :						List[Any]			       =  os.path.join(
             _UpperCAmelCase	,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

         if os.path.abspath(self.vocab_file     ) != os.path.abspath(_UpperCAmelCase     ) and os.path.isfile(self.vocab_file     ):
             copyfile(self.vocab_file	,			_UpperCAmelCase     )
         elif not os.path.isfile(self.vocab_file     ):
             with open(_UpperCAmelCase	,			'''wb'''     ) as fi:
                 __a :						str			       =  self.sp_model.serialized_model_proto()
                 fi.write(_UpperCAmelCase     )

         return (out_vocab_file,)







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase = "en_XX"	,			_UpperCAmelCase = None	,			_UpperCAmelCase = "ro_RO"	,			**_UpperCAmelCase	,			):
         __a :						List[Any]			       =  src_lang
         __a :						Union[str, Any]			       =  tgt_lang
         return super().prepare_seqaseq_batch(_UpperCAmelCase	,			_UpperCAmelCase	,			**_UpperCAmelCase     )







     def  _lowerCamelCase		(						self     ):
         return self.set_src_lang_special_tokens(self.src_lang     )







     def  _lowerCamelCase		(						self     ):
         return self.set_tgt_lang_special_tokens(self.tgt_lang     )







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         __a :						Union[str, Any]			       =  self.lang_code_to_id[src_lang]
         __a :						int			       =  []
         __a :						str			       =  [self.eos_token_id, self.cur_lang_code]







     def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
         __a :						Tuple			       =  self.lang_code_to_id[lang]
         __a :						Optional[Any]			       =  []
         __a :						Dict			       =  [self.eos_token_id, self.cur_lang_code]






"""simple docstring"""






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 				__lowercase			(			_UpperCamelCase   ,      unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  OpenAIGPTTokenizer
					__lowerCAmelCase  =  OpenAIGPTTokenizerFast
					__lowerCAmelCase  =  True
					__lowerCAmelCase  =  False







					def  _lowerCamelCase		(						self     ):
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									__a :						List[str]			       =  [
									    '''l''',
									    '''o''',
									    '''w''',
									    '''e''',
									    '''r''',
									    '''s''',
									    '''t''',
									    '''i''',
									    '''d''',
									    '''n''',
									    '''w</w>''',
									    '''r</w>''',
									    '''t</w>''',
									    '''lo''',
									    '''low''',
									    '''er</w>''',
									    '''low</w>''',
									    '''lowest</w>''',
									    '''newer</w>''',
									    '''wider</w>''',
									    '''<unk>''',
									]
									__a :						str			       =  dict(zip(_UpperCAmelCase	,			range(len(_UpperCAmelCase     )     )     )     )
									__a :						List[Any]			       =  ['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

									__a :						Union[str, Any]			       =  os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''vocab_file''']     )
									__a :						Dict			       =  os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''merges_file''']     )
									with open(self.vocab_file	,			'''w'''     ) as fp:
													fp.write(json.dumps(_UpperCAmelCase     )     )
									with open(self.merges_file	,			'''w'''     ) as fp:
													fp.write('''\n'''.join(_UpperCAmelCase     )     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									return "lower newer", "lower newer"







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  OpenAIGPTTokenizer(self.vocab_file	,			self.merges_file     )

									__a :						Tuple			       =  '''lower'''
									__a :						Union[str, Any]			       =  ['''low''', '''er</w>''']
									__a :						str			       =  tokenizer.tokenize(_UpperCAmelCase     )
									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )

									__a :						List[Any]			       =  tokens + ['''<unk>''']
									__a :						Tuple			       =  [14, 15, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase     )	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase=15     ):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																	__a :						Any			       =  self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase	,			**_UpperCAmelCase     )

																	# Simple input
																	__a :						str			       =  '''This is a simple input'''
																	__a :						Dict			       =  ['''This is a simple input 1''', '''This is a simple input 2''']
																	__a :						List[Any]			       =  ('''This is a simple input''', '''This is a pair''')
																	__a :						Optional[Any]			       =  [
																	    ('''This is a simple input 1''', '''This is a simple input 2'''),
																	    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																	]

																	# Simple input tests
																	self.assertRaises(_UpperCAmelCase	,			tokenizer_r.encode	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''     )

																	# Simple input
																	self.assertRaises(_UpperCAmelCase	,			tokenizer_r.encode_plus	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''     )

																	# Simple input
																	self.assertRaises(
																	    _UpperCAmelCase	,			tokenizer_r.batch_encode_plus	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''	,			)

																	# Pair input
																	self.assertRaises(_UpperCAmelCase	,			tokenizer_r.encode	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''     )

																	# Pair input
																	self.assertRaises(_UpperCAmelCase	,			tokenizer_r.encode_plus	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''     )

																	# Pair input
																	self.assertRaises(
																	    _UpperCAmelCase	,			tokenizer_r.batch_encode_plus	,			_UpperCAmelCase	,			max_length=_UpperCAmelCase	,			padding='''max_length'''	,			)







					def  _lowerCamelCase		(						self     ):
									pass






@require_ftfy
@require_spacy
@require_tokenizers
class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					pass