
"""simple docstring"""

import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			:       Optional[int]									=				logging.get_logger(__name__)

UpperCamelCase			:       Optional[int]									=				{
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

UpperCamelCase			:       Optional[int]									=				{
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

UpperCamelCase			:       List[Any]									=				{
    "vinai/phobert-base": 2_5_6,
    "vinai/phobert-large": 2_5_6,
}






def 							A		(       snake_case   :Union[str, Any]	)   ->			List[str]:
 __UpperCamelCase       						= set()
 __UpperCamelCase       						= word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)	)
  __UpperCamelCase       						= char

 __UpperCamelCase       						= set(snake_case	)
 return pairs





class __lowerCAmelCase	(   __SCREAMING_SNAKE_CASE    ):
     lowercase		     =		VOCAB_FILES_NAMES
     lowercase		     =		PRETRAINED_VOCAB_FILES_MAP
     lowercase		     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





     def __init__(			self			,  __UpperCAmelCase			,  __UpperCAmelCase			,  __UpperCAmelCase="<s>"			,  __UpperCAmelCase="</s>"			,  __UpperCAmelCase="</s>"			,  __UpperCAmelCase="<s>"			,  __UpperCAmelCase="<unk>"			,  __UpperCAmelCase="<pad>"			,  __UpperCAmelCase="<mask>"			,  **__UpperCAmelCase			,  ):

      '''simple docstring'''




      super().__init__(
          bos_token=__UpperCAmelCase			,  eos_token=__UpperCAmelCase			,  unk_token=__UpperCAmelCase			,  sep_token=__UpperCAmelCase			,  cls_token=__UpperCAmelCase			,  pad_token=__UpperCAmelCase			,  mask_token=__UpperCAmelCase			,  **__UpperCAmelCase			,  )

      __UpperCamelCase       						= vocab_file
      __UpperCamelCase       						= merges_file

      __UpperCamelCase       						= {}
      __UpperCamelCase       						= 0
      __UpperCamelCase       						= 1
      __UpperCamelCase       						= 2
      __UpperCamelCase       						= 3

      self.add_from_file(__UpperCAmelCase   )

      __UpperCamelCase       						= {v: k for k, v in self.encoder.items()}

      with open(__UpperCAmelCase			,  encoding='utf-8'   ) as merges_handle:
       __UpperCamelCase       						= merges_handle.read().split('\n'   )[:-1]
      __UpperCamelCase       						= [tuple(merge.split()[:-1]   ) for merge in merges]
      __UpperCamelCase       						= dict(zip(__UpperCAmelCase			,  range(len(__UpperCAmelCase   )   )   )   )
      __UpperCamelCase       						= {}





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

      '''simple docstring'''




      if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      __UpperCamelCase       						= [self.cls_token_id]
      __UpperCamelCase       						= [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None			,  __UpperCAmelCase = False   ):

      '''simple docstring'''




      if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__UpperCAmelCase			,  token_ids_a=__UpperCAmelCase			,  already_has_special_tokens=__UpperCAmelCase   )

      if token_ids_a is None:
       return [1] + ([0] * len(__UpperCAmelCase   )) + [1]
      return [1] + ([0] * len(__UpperCAmelCase   )) + [1, 1] + ([0] * len(__UpperCAmelCase   )) + [1]





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

      '''simple docstring'''




      __UpperCamelCase       						= [self.sep_token_id]
      __UpperCamelCase       						= [self.cls_token_id]

      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]





     @property
     def 		UpperCAmelCase      (			self   ):

      '''simple docstring'''




      return len(self.encoder   )





     def 		UpperCAmelCase      (			self   ):

      '''simple docstring'''




      return dict(self.encoder			,  **self.added_tokens_encoder   )





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      if token in self.cache:
       return self.cache[token]
      __UpperCamelCase       						= tuple(__UpperCAmelCase   )
      __UpperCamelCase       						= tuple(list(word[:-1]   ) + [word[-1] + '</w>']   )
      __UpperCamelCase       						= get_pairs(__UpperCAmelCase   )

      if not pairs:
       return token

      while True:
       __UpperCamelCase       						= min(__UpperCAmelCase			,  key=lambda __UpperCAmelCase   : self.bpe_ranks.get(__UpperCAmelCase			,  float('inf'   )   )   )
       if bigram not in self.bpe_ranks:
        break
       __UpperCamelCase						,			__UpperCamelCase       						= bigram
       __UpperCamelCase       						= []
       __UpperCamelCase       						= 0
       while i < len(__UpperCAmelCase   ):
        try:
         __UpperCamelCase       						= word.index(__UpperCAmelCase			,  __UpperCAmelCase   )
        except ValueError:
         new_word.extend(word[i:]   )
         break
        else:
         new_word.extend(word[i:j]   )
         __UpperCamelCase       						= j

        if word[i] == first and i < len(__UpperCAmelCase   ) - 1 and word[i + 1] == second:
         new_word.append(first + second   )
         i += 2
        else:
         new_word.append(word[i]   )
         i += 1
       __UpperCamelCase       						= tuple(__UpperCAmelCase   )
       __UpperCamelCase       						= new_word
       if len(__UpperCAmelCase   ) == 1:
        break
       else:
        __UpperCamelCase       						= get_pairs(__UpperCAmelCase   )
      __UpperCamelCase       						= '@@ '.join(__UpperCAmelCase   )
      __UpperCamelCase       						= word[:-4]
      __UpperCamelCase       						= word
      return word





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      __UpperCamelCase       						= []

      __UpperCamelCase       						= re.findall(R'\S+\n?'			,  __UpperCAmelCase   )

      for token in words:
       split_tokens.extend(list(self.bpe(__UpperCAmelCase   ).split(' '   )   )   )
      return split_tokens





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      return self.encoder.get(__UpperCAmelCase			,  self.encoder.get(self.unk_token   )   )





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      return self.decoder.get(__UpperCAmelCase			,  self.unk_token   )





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      __UpperCamelCase       						= ' '.join(__UpperCAmelCase   ).replace('@@ '			,  ''   ).strip()
      return out_string





     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase			,  __UpperCAmelCase = None   ):

      '''simple docstring'''




      if not os.path.isdir(__UpperCAmelCase   ):
       logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
       return
      __UpperCamelCase       						= os.path.join(
          __UpperCAmelCase			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
      __UpperCamelCase       						= os.path.join(
          __UpperCAmelCase			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCAmelCase   ):
       copyfile(self.vocab_file			,  __UpperCAmelCase   )

      if os.path.abspath(self.merges_file   ) != os.path.abspath(__UpperCAmelCase   ):
       copyfile(self.merges_file			,  __UpperCAmelCase   )

      return out_vocab_file, out_merge_file







     def 		UpperCAmelCase      (			self			,  __UpperCAmelCase   ):

      '''simple docstring'''




      if isinstance(__UpperCAmelCase			,  __UpperCAmelCase   ):
       try:
        with open(__UpperCAmelCase			,  'r'			,  encoding='utf-8'   ) as fd:
         self.add_from_file(__UpperCAmelCase   )
       except FileNotFoundError as fnfe:
        raise fnfe
       except UnicodeError:
        raise Exception(F'Incorrect encoding detected in {f}, please rebuild the dataset'   )
       return

      __UpperCamelCase       						= f.readlines()
      for lineTmp in lines:
       __UpperCamelCase       						= lineTmp.strip()
       __UpperCamelCase       						= line.rfind(' '   )
       if idx == -1:
        raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''   )
       __UpperCamelCase       						= line[:idx]
       __UpperCamelCase       						= len(self.encoder   )





"""simple docstring"""

def 							A		(       snake_case   :int       ,      snake_case   :int	)   ->			int:
 return int(input_a == input_a == 0	)






def 							A		(       )   ->			None:
 print('Truth Table of NOR Gate:'	)
 print('|   Input 1   |  Input 2  | Output |'	)
 print(f'|      0      |    0      |  {nor_gate(0       ,      0	)}     |'	)
 print(f'|      0      |    1      |  {nor_gate(0       ,      1	)}     |'	)
 print(f'|      1      |    0      |  {nor_gate(1       ,      0	)}     |'	)
 print(f'|      1      |    1      |  {nor_gate(1       ,      1	)}     |'	)


if __name__ == "__main__":
      import doctest

      doctest.testmod()
      main()



