





'''simple docstring'''






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_		:  Dict		 =						logging.get_logger(__name__)

A_		:  Union[str, Any]		 =						{
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

A_		:  List[Any]		 =						{
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

A_		:  List[str]		 =						{
    "vinai/phobert-base": 256,
    "vinai/phobert-large": 256,
}



def        UpperCamelCase__		(						__magic_name__  :   str      )				->			str:
 '''simple docstring'''






 snake_case__     :					int    =  set()
 snake_case__     :					Tuple    =  word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)      )
  snake_case__     :					Dict    =  char

 snake_case__     :					int    =  set(__magic_name__      )
 return pairs





class 					__snake_case  (					__SCREAMING_SNAKE_CASE							):






   '''simple docstring'''





   lowerCamelCase__    							=      VOCAB_FILES_NAMES
   lowerCamelCase__    							=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__    							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE="<s>"	,		__SCREAMING_SNAKE_CASE="</s>"	,		__SCREAMING_SNAKE_CASE="</s>"	,		__SCREAMING_SNAKE_CASE="<s>"	,		__SCREAMING_SNAKE_CASE="<unk>"	,		__SCREAMING_SNAKE_CASE="<pad>"	,		__SCREAMING_SNAKE_CASE="<mask>"	,		**__SCREAMING_SNAKE_CASE	,		):
    super().__init__(
        bos_token=__SCREAMING_SNAKE_CASE	,		eos_token=__SCREAMING_SNAKE_CASE	,		unk_token=__SCREAMING_SNAKE_CASE	,		sep_token=__SCREAMING_SNAKE_CASE	,		cls_token=__SCREAMING_SNAKE_CASE	,		pad_token=__SCREAMING_SNAKE_CASE	,		mask_token=__SCREAMING_SNAKE_CASE	,		**__SCREAMING_SNAKE_CASE	,		)

    snake_case__     :					Dict    =  vocab_file
    snake_case__     :					Optional[Any]    =  merges_file

    snake_case__     :					Dict    =  {}
    snake_case__     :					Any    =  0
    snake_case__     :					int    =  1
    snake_case__     :					int    =  2
    snake_case__     :					List[Any]    =  3

    self.add_from_file(__SCREAMING_SNAKE_CASE       )

    snake_case__     :					Optional[Any]    =  {v: k for k, v in self.encoder.items()}

    with open(__SCREAMING_SNAKE_CASE	,		encoding="""utf-8"""       ) as merges_handle:
     snake_case__     :					Any    =  merges_handle.read().split("""\n"""       )[:-1]
    snake_case__     :					int    =  [tuple(merge.split()[:-1]       ) for merge in merges]
    snake_case__     :					List[str]    =  dict(zip(__SCREAMING_SNAKE_CASE	,		range(len(__SCREAMING_SNAKE_CASE       )       )       )       )
    snake_case__     :					List[str]    =  {}

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None       ):

    if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    snake_case__     :					int    =  [self.cls_token_id]
    snake_case__     :					List[str]    =  [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = False       ):

    if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=__SCREAMING_SNAKE_CASE	,		token_ids_a=__SCREAMING_SNAKE_CASE	,		already_has_special_tokens=__SCREAMING_SNAKE_CASE       )

    if token_ids_a is None:
     return [1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1]
    return [1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE       )) + [1]

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None       ):
    snake_case__     :					Any    =  [self.sep_token_id]
    snake_case__     :					List[str]    =  [self.cls_token_id]

    if token_ids_a is None:
     return len(cls + token_ids_a + sep       ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]

   @property
   def 		__UpperCamelCase						(							self       ):
    return len(self.encoder       )

   def 		__UpperCamelCase						(							self       ):
    return dict(self.encoder	,		**self.added_tokens_encoder       )

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    if token in self.cache:
     return self.cache[token]
    snake_case__     :					List[Any]    =  tuple(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					List[Any]    =  tuple(list(word[:-1]       ) + [word[-1] + """</w>"""]       )
    snake_case__     :					Any    =  get_pairs(__SCREAMING_SNAKE_CASE       )

    if not pairs:
     return token

    while True:
     snake_case__     :					Optional[Any]    =  min(__SCREAMING_SNAKE_CASE	,		key=lambda __SCREAMING_SNAKE_CASE       : self.bpe_ranks.get(__SCREAMING_SNAKE_CASE	,		float("""inf"""       )       )       )
     if bigram not in self.bpe_ranks:
      break
     snake_case__  ,							snake_case__     :					Tuple    =  bigram
     snake_case__     :					Dict    =  []
     snake_case__     :					str    =  0
     while i < len(__SCREAMING_SNAKE_CASE       ):
      try:
       snake_case__     :					Tuple    =  word.index(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
      except ValueError:
       new_word.extend(word[i:]       )
       break
      else:
       new_word.extend(word[i:j]       )
       snake_case__     :					List[str]    =  j

      if word[i] == first and i < len(__SCREAMING_SNAKE_CASE       ) - 1 and word[i + 1] == second:
       new_word.append(first + second       )
       i += 2
      else:
       new_word.append(word[i]       )
       i += 1
     snake_case__     :					Dict    =  tuple(__SCREAMING_SNAKE_CASE       )
     snake_case__     :					Tuple    =  new_word
     if len(__SCREAMING_SNAKE_CASE       ) == 1:
      break
     else:
      snake_case__     :					Union[str, Any]    =  get_pairs(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					List[Any]    =  """@@ """.join(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Optional[int]    =  word[:-4]
    snake_case__     :					Dict    =  word
    return word

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Optional[int]    =  []

    snake_case__     :					Any    =  re.findall(R"""\S+\n?"""	,		__SCREAMING_SNAKE_CASE       )

    for token in words:
     split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE       ).split(""" """       )       )       )
    return split_tokens

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    return self.encoder.get(__SCREAMING_SNAKE_CASE	,		self.encoder.get(self.unk_token       )       )

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    return self.decoder.get(__SCREAMING_SNAKE_CASE	,		self.unk_token       )

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Optional[Any]    =  """ """.join(__SCREAMING_SNAKE_CASE       ).replace("""@@ """	,		""""""       ).strip()
    return out_string

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None       ):
    if not os.path.isdir(__SCREAMING_SNAKE_CASE       ):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
     return
    snake_case__     :					Optional[int]    =  os.path.join(
        __SCREAMING_SNAKE_CASE	,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
    snake_case__     :					Any    =  os.path.join(
        __SCREAMING_SNAKE_CASE	,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

    if os.path.abspath(self.vocab_file       ) != os.path.abspath(__SCREAMING_SNAKE_CASE       ):
     copyfile(self.vocab_file	,		__SCREAMING_SNAKE_CASE       )

    if os.path.abspath(self.merges_file       ) != os.path.abspath(__SCREAMING_SNAKE_CASE       ):
     copyfile(self.merges_file	,		__SCREAMING_SNAKE_CASE       )

    return out_vocab_file, out_merge_file


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    if isinstance(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
     try:
      with open(__SCREAMING_SNAKE_CASE	,		"""r"""	,		encoding="""utf-8"""       ) as fd:
       self.add_from_file(__SCREAMING_SNAKE_CASE       )
     except FileNotFoundError as fnfe:
      raise fnfe
     except UnicodeError:
      raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset"       )
     return

    snake_case__     :					Tuple    =  f.readlines()
    for lineTmp in lines:
     snake_case__     :					Any    =  lineTmp.strip()
     snake_case__     :					Optional[Any]    =  line.rfind(""" """       )
     if idx == -1:
      raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""       )
     snake_case__     :					Optional[int]    =  line[:idx]
     snake_case__     :					Union[str, Any]    =  len(self.encoder       )












'''simple docstring'''



def 				lowercase_					(     __A			:							int       ,   __A			:							int      )							->   int:



 """simple docstring"""




 return 1 if input_a == input_a else 0


def 				lowercase_					(     )							->   None:



 """simple docstring"""




 assert xnor_gate(0       ,   0      ) == 1
 assert xnor_gate(0       ,   1      ) == 0
 assert xnor_gate(1       ,   0      ) == 0
 assert xnor_gate(1       ,   1      ) == 1


if __name__ == "__main__":
      print(xnor_gate(0, 0))
      print(xnor_gate(0, 1))
      print(xnor_gate(1, 0))
      print(xnor_gate(1, 1))



