
"""simple docstring"""

import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			:       Optional[int]									=				logging.get_logger(__name__)

UpperCamelCase			:       Optional[int]									=				{
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

UpperCamelCase			:       Optional[int]									=				{
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

UpperCamelCase			:       List[Any]									=				{
    "vinai/phobert-base": 2_5_6,
    "vinai/phobert-large": 2_5_6,
}






def 							A		(       snake_case   :Union[str, Any]	)   ->			List[str]:
	__UpperCamelCase       						= set()
	__UpperCamelCase       						= word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)	)
		__UpperCamelCase       						= char

	__UpperCamelCase       						= set(snake_case	)
	return pairs





class __lowerCAmelCase	(   __SCREAMING_SNAKE_CASE    ):
					lowercase		     =		VOCAB_FILES_NAMES
					lowercase		     =		PRETRAINED_VOCAB_FILES_MAP
					lowercase		     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





					def __init__(			self			,  __UpperCAmelCase			,  __UpperCAmelCase			,  __UpperCAmelCase="<s>"			,  __UpperCAmelCase="</s>"			,  __UpperCAmelCase="</s>"			,  __UpperCAmelCase="<s>"			,  __UpperCAmelCase="<unk>"			,  __UpperCAmelCase="<pad>"			,  __UpperCAmelCase="<mask>"			,  **__UpperCAmelCase			,  ):

						'''simple docstring'''




						super().__init__(
						    bos_token=__UpperCAmelCase			,  eos_token=__UpperCAmelCase			,  unk_token=__UpperCAmelCase			,  sep_token=__UpperCAmelCase			,  cls_token=__UpperCAmelCase			,  pad_token=__UpperCAmelCase			,  mask_token=__UpperCAmelCase			,  **__UpperCAmelCase			,  )

						__UpperCamelCase       						= vocab_file
						__UpperCamelCase       						= merges_file

						__UpperCamelCase       						= {}
						__UpperCamelCase       						= 0
						__UpperCamelCase       						= 1
						__UpperCamelCase       						= 2
						__UpperCamelCase       						= 3

						self.add_from_file(__UpperCAmelCase   )

						__UpperCamelCase       						= {v: k for k, v in self.encoder.items()}

						with open(__UpperCAmelCase			,  encoding='utf-8'   ) as merges_handle:
							__UpperCamelCase       						= merges_handle.read().split('\n'   )[:-1]
						__UpperCamelCase       						= [tuple(merge.split()[:-1]   ) for merge in merges]
						__UpperCamelCase       						= dict(zip(__UpperCAmelCase			,  range(len(__UpperCAmelCase   )   )   )   )
						__UpperCamelCase       						= {}





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

						'''simple docstring'''




						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__UpperCamelCase       						= [self.cls_token_id]
						__UpperCamelCase       						= [self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None			,  __UpperCAmelCase = False   ):

						'''simple docstring'''




						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__UpperCAmelCase			,  token_ids_a=__UpperCAmelCase			,  already_has_special_tokens=__UpperCAmelCase   )

						if token_ids_a is None:
							return [1] + ([0] * len(__UpperCAmelCase   )) + [1]
						return [1] + ([0] * len(__UpperCAmelCase   )) + [1, 1] + ([0] * len(__UpperCAmelCase   )) + [1]





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

						'''simple docstring'''




						__UpperCamelCase       						= [self.sep_token_id]
						__UpperCamelCase       						= [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep   ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





					@property
					def 		UpperCAmelCase      (			self   ):

						'''simple docstring'''




						return len(self.encoder   )





					def 		UpperCAmelCase      (			self   ):

						'''simple docstring'''




						return dict(self.encoder			,  **self.added_tokens_encoder   )





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						if token in self.cache:
							return self.cache[token]
						__UpperCamelCase       						= tuple(__UpperCAmelCase   )
						__UpperCamelCase       						= tuple(list(word[:-1]   ) + [word[-1] + '</w>']   )
						__UpperCamelCase       						= get_pairs(__UpperCAmelCase   )

						if not pairs:
							return token

						while True:
							__UpperCamelCase       						= min(__UpperCAmelCase			,  key=lambda __UpperCAmelCase   : self.bpe_ranks.get(__UpperCAmelCase			,  float('inf'   )   )   )
							if bigram not in self.bpe_ranks:
								break
							__UpperCamelCase						,			__UpperCamelCase       						= bigram
							__UpperCamelCase       						= []
							__UpperCamelCase       						= 0
							while i < len(__UpperCAmelCase   ):
								try:
									__UpperCamelCase       						= word.index(__UpperCAmelCase			,  __UpperCAmelCase   )
								except ValueError:
									new_word.extend(word[i:]   )
									break
								else:
									new_word.extend(word[i:j]   )
									__UpperCamelCase       						= j

								if word[i] == first and i < len(__UpperCAmelCase   ) - 1 and word[i + 1] == second:
									new_word.append(first + second   )
									i += 2
								else:
									new_word.append(word[i]   )
									i += 1
							__UpperCamelCase       						= tuple(__UpperCAmelCase   )
							__UpperCamelCase       						= new_word
							if len(__UpperCAmelCase   ) == 1:
								break
							else:
								__UpperCamelCase       						= get_pairs(__UpperCAmelCase   )
						__UpperCamelCase       						= '@@ '.join(__UpperCAmelCase   )
						__UpperCamelCase       						= word[:-4]
						__UpperCamelCase       						= word
						return word





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						__UpperCamelCase       						= []

						__UpperCamelCase       						= re.findall(R'\S+\n?'			,  __UpperCAmelCase   )

						for token in words:
							split_tokens.extend(list(self.bpe(__UpperCAmelCase   ).split(' '   )   )   )
						return split_tokens





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						return self.encoder.get(__UpperCAmelCase			,  self.encoder.get(self.unk_token   )   )





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						return self.decoder.get(__UpperCAmelCase			,  self.unk_token   )





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						__UpperCamelCase       						= ' '.join(__UpperCAmelCase   ).replace('@@ '			,  ''   ).strip()
						return out_string





					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

						'''simple docstring'''




						if not os.path.isdir(__UpperCAmelCase   ):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
							return
						__UpperCamelCase       						= os.path.join(
						    __UpperCAmelCase			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
						__UpperCamelCase       						= os.path.join(
						    __UpperCAmelCase			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

						if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCAmelCase   ):
							copyfile(self.vocab_file			,  __UpperCAmelCase   )

						if os.path.abspath(self.merges_file   ) != os.path.abspath(__UpperCAmelCase   ):
							copyfile(self.merges_file			,  __UpperCAmelCase   )

						return out_vocab_file, out_merge_file







					def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

						'''simple docstring'''




						if isinstance(__UpperCAmelCase			,  __UpperCAmelCase   ):
							try:
								with open(__UpperCAmelCase			,  'r'			,  encoding='utf-8'   ) as fd:
									self.add_from_file(__UpperCAmelCase   )
							except FileNotFoundError as fnfe:
								raise fnfe
							except UnicodeError:
								raise Exception(F'Incorrect encoding detected in {f}, please rebuild the dataset'   )
							return

						__UpperCamelCase       						= f.readlines()
						for lineTmp in lines:
							__UpperCamelCase       						= lineTmp.strip()
							__UpperCamelCase       						= line.rfind(' '   )
							if idx == -1:
								raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''   )
							__UpperCamelCase       						= line[:idx]
							__UpperCamelCase       						= len(self.encoder   )





"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase			:       Dict									=				{
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase			:       List[str]									=				["LxmertTokenizerFast"]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase			:       Optional[Any]									=				[
          "LxmertEncoder",
          "LxmertForPreTraining",
          "LxmertForQuestionAnswering",
          "LxmertModel",
          "LxmertPreTrainedModel",
          "LxmertVisualFeatureEncoder",
          "LxmertXLayer",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCamelCase			:       List[Any]									=				[
          "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
          "TFLxmertForPreTraining",
          "TFLxmertMainLayer",
          "TFLxmertModel",
          "TFLxmertPreTrainedModel",
          "TFLxmertVisualFeatureEncoder",
      ]


if TYPE_CHECKING:
      from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
      from .tokenization_lxmert import LxmertTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_lxmert_fast import LxmertTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_lxmert import (
                LxmertEncoder,
                LxmertForPreTraining,
                LxmertForQuestionAnswering,
                LxmertModel,
                LxmertPreTrainedModel,
                LxmertVisualFeatureEncoder,
                LxmertXLayer,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_lxmert import (
                TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFLxmertForPreTraining,
                TFLxmertMainLayer,
                TFLxmertModel,
                TFLxmertPreTrainedModel,
                TFLxmertVisualFeatureEncoder,
            )

else:
      import sys

      UpperCamelCase			:       Any									=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



