






import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case			:			Dict			   =   logging.get_logger(__name__)

__snake_case			:			str			   =   {
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

__snake_case			:			List[str]			   =   {
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

__snake_case			:			Optional[Any]			   =   {"""stas/tiny-wmt19-en-de""": 10_24}
__snake_case			:			str			   =   {
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 10_24,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}







def 		_UpperCamelCase      (			UpperCamelCase_				:    str						)     ->      List[str]:

						"""simple docstring"""



						lowerCAmelCase__			  =  set()
						lowerCAmelCase__			  =  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)						)
												lowerCAmelCase__			  =  char
						return pairs







def 		_UpperCamelCase      (			UpperCamelCase_				:    Tuple						)     ->      Union[str, Any]:

						"""simple docstring"""



						lowerCAmelCase__			  =  text.replace('，'			,				','						)
						lowerCAmelCase__			  =  re.sub(r'。\s*'			,				'. '			,				UpperCamelCase_						)
						lowerCAmelCase__			  =  text.replace('、'			,				','						)
						lowerCAmelCase__			  =  text.replace('”'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('“'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('∶'			,				':'						)
						lowerCAmelCase__			  =  text.replace('：'			,				':'						)
						lowerCAmelCase__			  =  text.replace('？'			,				'?'						)
						lowerCAmelCase__			  =  text.replace('《'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('》'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('）'			,				')'						)
						lowerCAmelCase__			  =  text.replace('！'			,				'!'						)
						lowerCAmelCase__			  =  text.replace('（'			,				'('						)
						lowerCAmelCase__			  =  text.replace('；'			,				';'						)
						lowerCAmelCase__			  =  text.replace('１'			,				'1'						)
						lowerCAmelCase__			  =  text.replace('」'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('「'			,				'\"'						)
						lowerCAmelCase__			  =  text.replace('０'			,				'0'						)
						lowerCAmelCase__			  =  text.replace('３'			,				'3'						)
						lowerCAmelCase__			  =  text.replace('２'			,				'2'						)
						lowerCAmelCase__			  =  text.replace('５'			,				'5'						)
						lowerCAmelCase__			  =  text.replace('６'			,				'6'						)
						lowerCAmelCase__			  =  text.replace('９'			,				'9'						)
						lowerCAmelCase__			  =  text.replace('７'			,				'7'						)
						lowerCAmelCase__			  =  text.replace('８'			,				'8'						)
						lowerCAmelCase__			  =  text.replace('４'			,				'4'						)
						lowerCAmelCase__			  =  re.sub(r'．\s*'			,				'. '			,				UpperCamelCase_						)
						lowerCAmelCase__			  =  text.replace('～'			,				'~'						)
						lowerCAmelCase__			  =  text.replace('’'			,				'\''						)
						lowerCAmelCase__			  =  text.replace('…'			,				'...'						)
						lowerCAmelCase__			  =  text.replace('━'			,				'-'						)
						lowerCAmelCase__			  =  text.replace('〈'			,				'<'						)
						lowerCAmelCase__			  =  text.replace('〉'			,				'>'						)
						lowerCAmelCase__			  =  text.replace('【'			,				'['						)
						lowerCAmelCase__			  =  text.replace('】'			,				']'						)
						lowerCAmelCase__			  =  text.replace('％'			,				'%'						)
						return text







def 		_UpperCamelCase      (			UpperCamelCase_				:    Dict						)     ->      Tuple:

						"""simple docstring"""



						lowerCAmelCase__			  =  []
						for char in text:
												lowerCAmelCase__			  =  unicodedata.category(UpperCamelCase_						)
												if cat.startswith('C'						):
																		continue
												output.append(UpperCamelCase_						)
						return "".join(UpperCamelCase_						)


class 			__SCREAMING_SNAKE_CASE    (      lowercase_):
						_SCREAMING_SNAKE_CASE		:				str            =    VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE		:				List[Any]            =    PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE		:				Any            =    PRETRAINED_INIT_CONFIGURATION
						_SCREAMING_SNAKE_CASE		:				Optional[Any]            =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE		:				str            =    ['''input_ids''', '''attention_mask''']






						def __init__(		self   ,  _UpperCamelCase=None   ,  _UpperCamelCase=None   ,  _UpperCamelCase=None   ,  _UpperCamelCase=None   ,  _UpperCamelCase=False   ,  _UpperCamelCase="<unk>"   ,  _UpperCamelCase="<s>"   ,  _UpperCamelCase="</s>"   ,  _UpperCamelCase="<pad>"   ,  **_UpperCamelCase   ,  ):






												"""simple docstring"""
												super().__init__(
												    langs=_lowercase   ,  src_vocab_file=_lowercase   ,  tgt_vocab_file=_lowercase   ,  merges_file=_lowercase   ,  do_lower_case=_lowercase   ,  unk_token=_lowercase   ,  bos_token=_lowercase   ,  sep_token=_lowercase   ,  pad_token=_lowercase   ,  **_lowercase   ,  )

												try:
																		import sacremoses
												except ImportError:
																		raise ImportError(
																		    'You need to install sacremoses to use XLMTokenizer. '
																		    'See https://pypi.org/project/sacremoses/ for installation.'      )

												lowerCAmelCase__			  =  sacremoses

												lowerCAmelCase__			  =  src_vocab_file
												lowerCAmelCase__			  =  tgt_vocab_file
												lowerCAmelCase__			  =  merges_file
												lowerCAmelCase__			  =  do_lower_case

												# cache of sm.MosesPunctNormalizer instance
												lowerCAmelCase__			  =  {}
												# cache of sm.MosesTokenizer instance
												lowerCAmelCase__			  =  {}
												lowerCAmelCase__			  =  {}

												if langs and len(_lowercase      ) == 2:
																		lowerCAmelCase__  ,				lowerCAmelCase__			  =  langs
												else:
																		raise ValueError(
																		    F"arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. "
																		    'Usually that means that tokenizer can\'t find a mapping for the given model path '
																		    'in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'      )

												with open(_lowercase   ,  encoding='utf-8'      ) as src_vocab_handle:
																		lowerCAmelCase__			  =  json.load(_lowercase      )
												with open(_lowercase   ,  encoding='utf-8'      ) as tgt_vocab_handle:
																		lowerCAmelCase__			  =  json.load(_lowercase      )
																		lowerCAmelCase__			  =  {v: k for k, v in tgt_vocab.items()}
												with open(_lowercase   ,  encoding='utf-8'      ) as merges_handle:
																		lowerCAmelCase__			  =  merges_handle.read().split('\n'      )[:-1]
												lowerCAmelCase__			  =  [tuple(merge.split()[:2]      ) for merge in merges]
												lowerCAmelCase__			  =  dict(zip(_lowercase   ,  range(len(_lowercase      )      )      )      )
												lowerCAmelCase__			  =  {}






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return self.get_src_vocab()






						@property
						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return self.src_vocab_size






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase      ):






												"""simple docstring"""





												if lang not in self.cache_moses_punct_normalizer:
																		lowerCAmelCase__			  =  self.sm.MosesPunctNormalizer(lang=_lowercase      )
																		lowerCAmelCase__			  =  punct_normalizer
												return self.cache_moses_punct_normalizer[lang].normalize(_lowercase      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase      ):






												"""simple docstring"""





												if lang not in self.cache_moses_tokenizer:
																		lowerCAmelCase__			  =  self.sm.MosesTokenizer(lang=_lowercase      )
																		lowerCAmelCase__			  =  moses_tokenizer
												return self.cache_moses_tokenizer[lang].tokenize(
												    _lowercase   ,  aggressive_dash_splits=_lowercase   ,  return_str=_lowercase   ,  escape=_lowercase      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase      ):






												"""simple docstring"""





												if lang not in self.cache_moses_detokenizer:
																		lowerCAmelCase__			  =  self.sm.MosesDetokenizer(lang=_lowercase      )
																		lowerCAmelCase__			  =  moses_detokenizer
												return self.cache_moses_detokenizer[lang].detokenize(_lowercase      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  replace_unicode_punct(_lowercase      )
												lowerCAmelCase__			  =  self.moses_punct_norm(_lowercase   ,  _lowercase      )
												lowerCAmelCase__			  =  remove_non_printing_char(_lowercase      )
												return text






						@property
						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return len(self.encoder      )






						@property
						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return len(self.decoder      )






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return dict(self.encoder   ,  **self.added_tokens_encoder      )






						def 					UpperCamelCase__					(		self      ):






												"""simple docstring"""
												return dict(self.decoder   ,  **self.added_tokens_decoder      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  tuple(token[:-1]      ) + (token[-1] + '</w>',)
												if token in self.cache:
																		return self.cache[token]
												lowerCAmelCase__			  =  get_pairs(_lowercase      )

												if not pairs:
																		return token + "</w>"

												while True:
																		lowerCAmelCase__			  =  min(_lowercase   ,  key=lambda _UpperCamelCase      : self.bpe_ranks.get(_lowercase   ,  float('inf'      )      )      )
																		if bigram not in self.bpe_ranks:
																								break
																		lowerCAmelCase__  ,				lowerCAmelCase__			  =  bigram
																		lowerCAmelCase__			  =  []
																		lowerCAmelCase__			  =  0
																		while i < len(_lowercase      ):
																								try:
																														lowerCAmelCase__			  =  word.index(_lowercase   ,  _lowercase      )
																								except ValueError:
																														new_word.extend(word[i:]      )
																														break
																								else:
																														new_word.extend(word[i:j]      )
																														lowerCAmelCase__			  =  j

																								if word[i] == first and i < len(_lowercase      ) - 1 and word[i + 1] == second:
																														new_word.append(first + second      )
																														i += 2
																								else:
																														new_word.append(word[i]      )
																														i += 1
																		lowerCAmelCase__			  =  tuple(_lowercase      )
																		lowerCAmelCase__			  =  new_word
																		if len(_lowercase      ) == 1:
																								break
																		else:
																								lowerCAmelCase__			  =  get_pairs(_lowercase      )
												lowerCAmelCase__			  =  ' '.join(_lowercase      )
												if word == "\n  </w>":
																		lowerCAmelCase__			  =  '\n</w>'
												lowerCAmelCase__			  =  word
												return word






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase="en"   ,  _UpperCamelCase=False      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  self.src_lang

												if self.do_lower_case:
																		lowerCAmelCase__			  =  text.lower()

												if bypass_tokenizer:
																		lowerCAmelCase__			  =  text.split()
												else:
																		lowerCAmelCase__			  =  self.moses_pipeline(_lowercase   ,  lang=_lowercase      )
																		lowerCAmelCase__			  =  self.moses_tokenize(_lowercase   ,  lang=_lowercase      )

												lowerCAmelCase__			  =  []
												for token in text:
																		if token:
																								split_tokens.extend(list(self.bpe(_lowercase      ).split(' '      )      )      )

												return split_tokens






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												return self.encoder.get(_lowercase   ,  self.encoder.get(self.unk_token      )      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												return self.decoder.get(_lowercase   ,  self.unk_token      )






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  [t.replace(' '   ,  ''      ).replace('</w>'   ,  ' '      ) for t in tokens]
												lowerCAmelCase__			  =  ''.join(_lowercase      ).split()
												# detokenize
												lowerCAmelCase__			  =  self.moses_detokenize(_lowercase   ,  self.tgt_lang      )
												return text






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase = None      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  [self.sep_token_id]

												# no bos used in fairseq
												if token_ids_a is None:
																		return token_ids_a + sep
												return token_ids_a + sep + token_ids_a + sep






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase = None   ,  _UpperCamelCase = False      ):






												"""simple docstring"""





												if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=_lowercase   ,  token_ids_a=_lowercase   ,  already_has_special_tokens=_lowercase      )
												# no bos used in fairseq
												if token_ids_a is not None:
																		return ([0] * len(_lowercase      )) + [1] + ([0] * len(_lowercase      )) + [1]
												return ([0] * len(_lowercase      )) + [1]






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase = None      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  [self.sep_token_id]

												# no bos used in fairseq
												if token_ids_a is None:
																		return len(token_ids_a + sep      ) * [0]
												return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






						def 					UpperCamelCase__					(		self   ,  _UpperCamelCase   ,  _UpperCamelCase = None      ):






												"""simple docstring"""





												if not os.path.isdir(_lowercase      ):
																		logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
																		return

												lowerCAmelCase__			  =  os.path.join(
												    _lowercase   ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['src_vocab_file']      )
												lowerCAmelCase__			  =  os.path.join(
												    _lowercase   ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['tgt_vocab_file']      )
												lowerCAmelCase__			  =  os.path.join(
												    _lowercase   ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

												with open(_lowercase   ,  'w'   ,  encoding='utf-8'      ) as f:
																		f.write(json.dumps(self.encoder   ,  indent=2   ,  sort_keys=_lowercase   ,  ensure_ascii=_lowercase      ) + '\n'      )

												with open(_lowercase   ,  'w'   ,  encoding='utf-8'      ) as f:
																		lowerCAmelCase__			  =  {v: k for k, v in self.decoder.items()}
																		f.write(json.dumps(_lowercase   ,  indent=2   ,  sort_keys=_lowercase   ,  ensure_ascii=_lowercase      ) + '\n'      )

												lowerCAmelCase__			  =  0
												with open(_lowercase   ,  'w'   ,  encoding='utf-8'      ) as writer:
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,  key=lambda _UpperCamelCase      : kv[1]      ):
																								if index != token_index:
																														logger.warning(
																														    F"Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."
																														    ' Please check that the tokenizer is not corrupted!'      )
																														lowerCAmelCase__			  =  token_index
																								writer.write(' '.join(_lowercase      ) + '\n'      )
																								index += 1

												return src_vocab_file, tgt_vocab_file, merges_file






						def __getstate__(		self      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  self.__dict__.copy()
												lowerCAmelCase__			  =  None
												return state







						def __setstate__(		self   ,  _UpperCamelCase      ):






												"""simple docstring"""
												lowerCAmelCase__			  =  d

												try:
																		import sacremoses
												except ImportError:
																		raise ImportError(
																		    'You need to install sacremoses to use XLMTokenizer. '
																		    'See https://pypi.org/project/sacremoses/ for installation.'      )

												lowerCAmelCase__			  =  sacremoses












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_snake_case				= {"""configuration_dinat""": ["""DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DinatConfig"""]}


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _snake_case				= [
      """DINAT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """DinatForImageClassification""",
      """DinatModel""",
      """DinatPreTrainedModel""",
      """DinatBackbone""",
  ]

if TYPE_CHECKING:
  from .configuration_dinat import DINAT_PRETRAINED_CONFIG_ARCHIVE_MAP, DinatConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_dinat import (
        DINAT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DinatBackbone,
        DinatForImageClassification,
        DinatModel,
        DinatPreTrainedModel,
    )

else:
  import sys

  _snake_case				= _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






