


'''simple docstring'''



import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase			:			int	   =     logging.get_logger(__name__)

_lowercase			:			Optional[Any]	   =     {
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

_lowercase			:			int	   =     {
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

_lowercase			:			Union[str, Any]	   =     {"""stas/tiny-wmt19-en-de""": 1024}
_lowercase			:			Optional[int]	   =     {
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 1024,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}


def       lowerCamelCase__      (       A   :			Tuple     ):




						'''simple docstring'''






						UpperCAmelCase     				=	set()
						UpperCAmelCase     				=	word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)     )
												UpperCAmelCase     				=	char
						return pairs


def       lowerCamelCase__      (       A   :			List[Any]     ):




						'''simple docstring'''






						UpperCAmelCase     				=	text.replace('''，'''      ,   ''','''     )
						UpperCAmelCase     				=	re.sub(R'''。\s*'''      ,   '''. '''      ,   lowercase_     )
						UpperCAmelCase     				=	text.replace('''、'''      ,   ''','''     )
						UpperCAmelCase     				=	text.replace('''”'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''“'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''∶'''      ,   ''':'''     )
						UpperCAmelCase     				=	text.replace('''：'''      ,   ''':'''     )
						UpperCAmelCase     				=	text.replace('''？'''      ,   '''?'''     )
						UpperCAmelCase     				=	text.replace('''《'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''》'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''）'''      ,   ''')'''     )
						UpperCAmelCase     				=	text.replace('''！'''      ,   '''!'''     )
						UpperCAmelCase     				=	text.replace('''（'''      ,   '''('''     )
						UpperCAmelCase     				=	text.replace('''；'''      ,   ''';'''     )
						UpperCAmelCase     				=	text.replace('''１'''      ,   '''1'''     )
						UpperCAmelCase     				=	text.replace('''」'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''「'''      ,   '''\"'''     )
						UpperCAmelCase     				=	text.replace('''０'''      ,   '''0'''     )
						UpperCAmelCase     				=	text.replace('''３'''      ,   '''3'''     )
						UpperCAmelCase     				=	text.replace('''２'''      ,   '''2'''     )
						UpperCAmelCase     				=	text.replace('''５'''      ,   '''5'''     )
						UpperCAmelCase     				=	text.replace('''６'''      ,   '''6'''     )
						UpperCAmelCase     				=	text.replace('''９'''      ,   '''9'''     )
						UpperCAmelCase     				=	text.replace('''７'''      ,   '''7'''     )
						UpperCAmelCase     				=	text.replace('''８'''      ,   '''8'''     )
						UpperCAmelCase     				=	text.replace('''４'''      ,   '''4'''     )
						UpperCAmelCase     				=	re.sub(R'''．\s*'''      ,   '''. '''      ,   lowercase_     )
						UpperCAmelCase     				=	text.replace('''～'''      ,   '''~'''     )
						UpperCAmelCase     				=	text.replace('''’'''      ,   '''\''''     )
						UpperCAmelCase     				=	text.replace('''…'''      ,   '''...'''     )
						UpperCAmelCase     				=	text.replace('''━'''      ,   '''-'''     )
						UpperCAmelCase     				=	text.replace('''〈'''      ,   '''<'''     )
						UpperCAmelCase     				=	text.replace('''〉'''      ,   '''>'''     )
						UpperCAmelCase     				=	text.replace('''【'''      ,   '''['''     )
						UpperCAmelCase     				=	text.replace('''】'''      ,   ''']'''     )
						UpperCAmelCase     				=	text.replace('''％'''      ,   '''%'''     )
						return text


def       lowerCamelCase__      (       A   :			List[Any]     ):




						'''simple docstring'''






						UpperCAmelCase     				=	[]
						for char in text:
												UpperCAmelCase     				=	unicodedata.category(lowercase_     )
												if cat.startswith('''C'''     ):
																		continue
												output.append(lowercase_     )
						return "".join(lowercase_     )






class        UpperCamelCase__(   _UpperCamelCase       ):
				__magic_name__				:      List[Any]  		=       VOCAB_FILES_NAMES
				__magic_name__				:      Any  		=       PRETRAINED_VOCAB_FILES_MAP
				__magic_name__				:      int  		=       PRETRAINED_INIT_CONFIGURATION
				__magic_name__				:      Tuple  		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__				:      List[str]  		=       ["input_ids", "attention_mask"]




				def __init__(		self			: Dict      ,    lowerCAmelCase			: Union[str, Any]=None      ,    lowerCAmelCase			: Optional[int]=None      ,    lowerCAmelCase			: Optional[int]=None      ,    lowerCAmelCase			: List[str]=None      ,    lowerCAmelCase			: Optional[Any]=False      ,    lowerCAmelCase			: Dict="<unk>"      ,    lowerCAmelCase			: Union[str, Any]="<s>"      ,    lowerCAmelCase			: Optional[Any]="</s>"      ,    lowerCAmelCase			: Tuple="<pad>"      ,    **lowerCAmelCase			: List[Any]      ,    )->     Dict:

										"""simple docstring"""




										super().__init__(
										    langs=__a      ,    src_vocab_file=__a      ,    tgt_vocab_file=__a      ,    merges_file=__a      ,    do_lower_case=__a      ,    unk_token=__a      ,    bos_token=__a      ,    sep_token=__a      ,    pad_token=__a      ,    **__a      ,    )

										try:
																import sacremoses
										except ImportError:
																raise ImportError(
																    '''You need to install sacremoses to use XLMTokenizer. '''
																    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

										UpperCAmelCase     				=	sacremoses

										UpperCAmelCase     				=	src_vocab_file
										UpperCAmelCase     				=	tgt_vocab_file
										UpperCAmelCase     				=	merges_file
										UpperCAmelCase     				=	do_lower_case

										# cache of sm.MosesPunctNormalizer instance
										UpperCAmelCase     				=	{}
										# cache of sm.MosesTokenizer instance
										UpperCAmelCase     				=	{}
										UpperCAmelCase     				=	{}

										if langs and len(__a      ) == 2:
																UpperCAmelCase     				=	langs
										else:
																raise ValueError(
																    F"""arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. """
																    '''Usually that means that tokenizer can\'t find a mapping for the given model path '''
																    '''in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'''      )

										with open(__a      ,    encoding='''utf-8'''      ) as src_vocab_handle:
																UpperCAmelCase     				=	json.load(__a      )
										with open(__a      ,    encoding='''utf-8'''      ) as tgt_vocab_handle:
																UpperCAmelCase     				=	json.load(__a      )
																UpperCAmelCase     				=	{v: k for k, v in tgt_vocab.items()}
										with open(__a      ,    encoding='''utf-8'''      ) as merges_handle:
																UpperCAmelCase     				=	merges_handle.read().split('''\n'''      )[:-1]
										UpperCAmelCase     				=	[tuple(merge.split()[:2]      ) for merge in merges]
										UpperCAmelCase     				=	dict(zip(__a      ,    range(len(__a      )      )      )      )
										UpperCAmelCase     				=	{}




				def      a__(		self			: int      )->     Dict[str, int]:

										"""simple docstring"""




										return self.get_src_vocab()




				@property
				def      a__(		self			: str      )->     int:

										"""simple docstring"""




										return self.src_vocab_size




				def      a__(		self			: List[str]      ,    lowerCAmelCase			: Tuple      ,    lowerCAmelCase			: str      )->     Optional[Any]:

										"""simple docstring"""




										if lang not in self.cache_moses_punct_normalizer:
																UpperCAmelCase     				=	self.sm.MosesPunctNormalizer(lang=__a      )
																UpperCAmelCase     				=	punct_normalizer
										return self.cache_moses_punct_normalizer[lang].normalize(__a      )




				def      a__(		self			: Tuple      ,    lowerCAmelCase			: Union[str, Any]      ,    lowerCAmelCase			: Tuple      )->     Tuple:

										"""simple docstring"""




										if lang not in self.cache_moses_tokenizer:
																UpperCAmelCase     				=	self.sm.MosesTokenizer(lang=__a      )
																UpperCAmelCase     				=	moses_tokenizer
										return self.cache_moses_tokenizer[lang].tokenize(
										    __a      ,    aggressive_dash_splits=__a      ,    return_str=__a      ,    escape=__a      )




				def      a__(		self			: str      ,    lowerCAmelCase			: str      ,    lowerCAmelCase			: Union[str, Any]      )->     Union[str, Any]:

										"""simple docstring"""




										if lang not in self.cache_moses_detokenizer:
																UpperCAmelCase     				=	self.sm.MosesDetokenizer(lang=__a      )
																UpperCAmelCase     				=	moses_detokenizer
										return self.cache_moses_detokenizer[lang].detokenize(__a      )




				def      a__(		self			: Tuple      ,    lowerCAmelCase			: int      ,    lowerCAmelCase			: str      )->     int:

										"""simple docstring"""




										UpperCAmelCase     				=	replace_unicode_punct(__a      )
										UpperCAmelCase     				=	self.moses_punct_norm(__a      ,    __a      )
										UpperCAmelCase     				=	remove_non_printing_char(__a      )
										return text




				@property
				def      a__(		self			: Optional[Any]      )->     Optional[Any]:

										"""simple docstring"""




										return len(self.encoder      )




				@property
				def      a__(		self			: Optional[Any]      )->     str:

										"""simple docstring"""




										return len(self.decoder      )




				def      a__(		self			: Union[str, Any]      )->     Optional[Any]:

										"""simple docstring"""




										return dict(self.encoder      ,    **self.added_tokens_encoder      )




				def      a__(		self			: str      )->     List[Any]:

										"""simple docstring"""




										return dict(self.decoder      ,    **self.added_tokens_decoder      )




				def      a__(		self			: Optional[int]      ,    lowerCAmelCase			: Dict      )->     Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	tuple(token[:-1]      ) + (token[-1] + "</w>",)
										if token in self.cache:
																return self.cache[token]
										UpperCAmelCase     				=	get_pairs(__a      )

										if not pairs:
																return token + "</w>"

										while True:
																UpperCAmelCase     				=	min(__a      ,    key=lambda lowerCAmelCase      : self.bpe_ranks.get(__a      ,    float('''inf'''      )      )      )
																if bigram not in self.bpe_ranks:
																						break
																UpperCAmelCase     				=	bigram
																UpperCAmelCase     				=	[]
																UpperCAmelCase     				=	0
																while i < len(__a      ):
																						try:
																												UpperCAmelCase     				=	word.index(__a      ,    __a      )
																						except ValueError:
																												new_word.extend(word[i:]      )
																												break
																						else:
																												new_word.extend(word[i:j]      )
																												UpperCAmelCase     				=	j

																						if word[i] == first and i < len(__a      ) - 1 and word[i + 1] == second:
																												new_word.append(first + second      )
																												i += 2
																						else:
																												new_word.append(word[i]      )
																												i += 1
																UpperCAmelCase     				=	tuple(__a      )
																UpperCAmelCase     				=	new_word
																if len(__a      ) == 1:
																						break
																else:
																						UpperCAmelCase     				=	get_pairs(__a      )
										UpperCAmelCase     				=	" ".join(__a      )
										if word == "\n  </w>":
																UpperCAmelCase     				=	"\n</w>"
										UpperCAmelCase     				=	word
										return word




				def      a__(		self			: List[Any]      ,    lowerCAmelCase			: Union[str, Any]      ,    lowerCAmelCase			: Optional[int]="en"      ,    lowerCAmelCase			: int=False      )->     Dict:

										"""simple docstring"""




										UpperCAmelCase     				=	self.src_lang

										if self.do_lower_case:
																UpperCAmelCase     				=	text.lower()

										if bypass_tokenizer:
																UpperCAmelCase     				=	text.split()
										else:
																UpperCAmelCase     				=	self.moses_pipeline(__a      ,    lang=__a      )
																UpperCAmelCase     				=	self.moses_tokenize(__a      ,    lang=__a      )

										UpperCAmelCase     				=	[]
										for token in text:
																if token:
																						split_tokens.extend(list(self.bpe(__a      ).split(''' '''      )      )      )

										return split_tokens




				def      a__(		self			: Optional[Any]      ,    lowerCAmelCase			: List[Any]      )->     List[Any]:

										"""simple docstring"""




										return self.encoder.get(__a      ,    self.encoder.get(self.unk_token      )      )




				def      a__(		self			: str      ,    lowerCAmelCase			: Dict      )->     Optional[Any]:

										"""simple docstring"""




										return self.decoder.get(__a      ,    self.unk_token      )




				def      a__(		self			: Dict      ,    lowerCAmelCase			: Union[str, Any]      )->     Optional[int]:

										"""simple docstring"""




										UpperCAmelCase     				=	[t.replace(''' '''      ,    ''''''      ).replace('''</w>'''      ,    ''' '''      ) for t in tokens]
										UpperCAmelCase     				=	"".join(__a      ).split()
										# detokenize
										UpperCAmelCase     				=	self.moses_detokenize(__a      ,    self.tgt_lang      )
										return text




				def      a__(		self			: Union[str, Any]      ,    lowerCAmelCase			: List[int]      ,    lowerCAmelCase			: Optional[List[int]] = None      )->     List[int]:

										"""simple docstring"""




										UpperCAmelCase     				=	[self.sep_token_id]

										# no bos used in fairseq
										if token_ids_a is None:
																return token_ids_a + sep
										return token_ids_a + sep + token_ids_a + sep




				def      a__(		self			: Union[str, Any]      ,    lowerCAmelCase			: List[int]      ,    lowerCAmelCase			: Optional[List[int]] = None      ,    lowerCAmelCase			: bool = False      )->     List[int]:

										"""simple docstring"""




										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=__a      ,    token_ids_a=__a      ,    already_has_special_tokens=__a      )
										# no bos used in fairseq
										if token_ids_a is not None:
																return ([0] * len(__a      )) + [1] + ([0] * len(__a      )) + [1]
										return ([0] * len(__a      )) + [1]




				def      a__(		self			: int      ,    lowerCAmelCase			: List[int]      ,    lowerCAmelCase			: Optional[List[int]] = None      )->     List[int]:

										"""simple docstring"""




										UpperCAmelCase     				=	[self.sep_token_id]

										# no bos used in fairseq
										if token_ids_a is None:
																return len(token_ids_a + sep      ) * [0]
										return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




				def      a__(		self			: Optional[int]      ,    lowerCAmelCase			: str      ,    lowerCAmelCase			: Optional[str] = None      )->     Tuple[str]:

										"""simple docstring"""




										if not os.path.isdir(__a      ):
																logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
																return

										UpperCAmelCase     				=	os.path.join(
										    __a      ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''src_vocab_file''']      )
										UpperCAmelCase     				=	os.path.join(
										    __a      ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''tgt_vocab_file''']      )
										UpperCAmelCase     				=	os.path.join(
										    __a      ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

										with open(__a      ,    '''w'''      ,    encoding='''utf-8'''      ) as f:
																f.write(json.dumps(self.encoder      ,    indent=2      ,    sort_keys=__a      ,    ensure_ascii=__a      ) + '''\n'''      )

										with open(__a      ,    '''w'''      ,    encoding='''utf-8'''      ) as f:
																UpperCAmelCase     				=	{v: k for k, v in self.decoder.items()}
																f.write(json.dumps(__a      ,    indent=2      ,    sort_keys=__a      ,    ensure_ascii=__a      ) + '''\n'''      )

										UpperCAmelCase     				=	0
										with open(__a      ,    '''w'''      ,    encoding='''utf-8'''      ) as writer:
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,    key=lambda lowerCAmelCase      : kv[1]      ):
																						if index != token_index:
																												logger.warning(
																												    F"""Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."""
																												    ''' Please check that the tokenizer is not corrupted!'''      )
																												UpperCAmelCase     				=	token_index
																						writer.write(''' '''.join(__a      ) + '''\n'''      )
																						index += 1

										return src_vocab_file, tgt_vocab_file, merges_file




				def __getstate__(		self			: List[Any]      )->     Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	self.__dict__.copy()
										UpperCAmelCase     				=	None
										return state





				def __setstate__(		self			: Optional[int]      ,    lowerCAmelCase			: int      )->     List[Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	d

										try:
																import sacremoses
										except ImportError:
																raise ImportError(
																    '''You need to install sacremoses to use XLMTokenizer. '''
																    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

										UpperCAmelCase     				=	sacremoses












"""simple docstring"""


import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_reformer import ReformerTokenizer
else:
  lowerCamelCase__      =      None


lowerCamelCase__      =      logging.get_logger(__name__)


lowerCamelCase__      =      "▁"

lowerCamelCase__      =      {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase__      =      {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    },
    "tokenizer_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json"
        )
    },
}

lowerCamelCase__      =      {
    "google/reformer-crime-and-punishment": 52_4288,
}



class    __SCREAMING_SNAKE_CASE			(  _UpperCamelCase      ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__							:List[str]		   =    VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__							:Any		   =    PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__							:List[Any]		   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__							:List[Any]		   =    ["input_ids", "attention_mask"]
  SCREAMING_SNAKE_CASE__							:str		   =    ReformerTokenizer

  def __init__(						self      :				List[str]    ,      __a      :				str=None    ,      __a      :				List[Any]=None    ,      __a      :				Optional[int]="</s>"    ,      __a      :				str="<unk>"    ,      __a      :				Dict=[]    ,      **__a      :				Tuple    ,      )						-> Optional[int]:
        super().__init__(
            __a    ,      tokenizer_file=__a    ,      eos_token=__a    ,      unk_token=__a    ,      additional_special_tokens=__a    ,      **__a    ,      )

        _UpperCamelCase					:		Dict  =				vocab_file
        _UpperCamelCase					:		Optional[Any]  =				False if not self.vocab_file else True


  def       __SCREAMING_SNAKE_CASE (						self      :				Dict    ,      __a      :				str    ,      __a      :				Optional[str] = None		)						-> Tuple[str]:
        if not self.can_save_slow_tokenizer:
              raise ValueError(
                  "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                  "tokenizer."		)

        if not os.path.isdir(__a		):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
              return
        _UpperCamelCase					:		Optional[Any]  =				os.path.join(
            __a    ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

        if os.path.abspath(self.vocab_file		) != os.path.abspath(__a		):
              copyfile(self.vocab_file    ,      __a		)

        return (out_vocab_file,)
