




import argparse
import hashlib  # hashlib is only used inside the Test class
import struct

class     __a:






    """simple docstring"""





    def __init__( self  ,_SCREAMING_SNAKE_CASE    )					->						Any:
         UpperCAmelCase_     :				Union[str, Any]  =     data
         UpperCAmelCase_     :				str  =     [0X67_45_23_01, 0Xef_cd_ab_89, 0X98_ba_dc_fe, 0X10_32_54_76, 0Xc3_d2_e1_f0]





    @staticmethod
    def 				a__ ( _SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )					->						Union[str, Any]:
         return ((n << b) | (n >> (32 - b))) & 0Xff_ff_ff_ff





    def 				a__ ( self    )					->						Any:
         UpperCAmelCase_     :				Dict  =     b'''\x80''' + b'''\x00''' * (63 - (len(self.data    ) + 8) % 64)
         UpperCAmelCase_     :				List[str]  =     self.data + padding + struct.pack('''>Q'''  ,8 * len(self.data    )    )
         return padded_data





    def 				a__ ( self    )					->						Any:
         return [
             self.padded_data[i : i + 64] for i in range(0  ,len(self.padded_data    )  ,64    )
         ]





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE    )					->						Tuple:
         UpperCAmelCase_     :				List[Any]  =     list(struct.unpack('''>16L'''  ,_SCREAMING_SNAKE_CASE    )    ) + [0] * 64
         for i in range(16  ,80    ):
              UpperCAmelCase_     :				Dict  =     self.rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16])  ,1    )
         return w





    def 				a__ ( self    )					->						Union[str, Any]:
         UpperCAmelCase_     :				List[Any]  =     self.padding()
         UpperCAmelCase_     :				Tuple  =     self.split_blocks()
         for block in self.blocks:
              UpperCAmelCase_     :				int  =     self.expand_block(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_     :				List[Any]  =     self.h
              for i in range(0  ,80    ):
                   if 0 <= i < 20:
                        UpperCAmelCase_     :				Union[str, Any]  =     (b & c) | ((~b) & d)
                        UpperCAmelCase_     :				Optional[int]  =     0X5a_82_79_99
                   elif 20 <= i < 40:
                        UpperCAmelCase_     :				Any  =     b ^ c ^ d
                        UpperCAmelCase_     :				Union[str, Any]  =     0X6e_d9_eb_a1
                   elif 40 <= i < 60:
                        UpperCAmelCase_     :				List[Any]  =     (b & c) | (b & d) | (c & d)
                        UpperCAmelCase_     :				Tuple  =     0X8f_1b_bc_dc
                   elif 60 <= i < 80:
                        UpperCAmelCase_     :				str  =     b ^ c ^ d
                        UpperCAmelCase_     :				Union[str, Any]  =     0Xca_62_c1_d6
                   UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_     :				Optional[Any]  =     (
                       self.rotate(_SCREAMING_SNAKE_CASE  ,5    ) + f + e + k + expanded_block[i] & 0Xff_ff_ff_ff,
                       a,
                       self.rotate(_SCREAMING_SNAKE_CASE  ,30    ),
                       c,
                       d,
                   )
              UpperCAmelCase_     :				Dict  =     (
                  self.h[0] + a & 0Xff_ff_ff_ff,
                  self.h[1] + b & 0Xff_ff_ff_ff,
                  self.h[2] + c & 0Xff_ff_ff_ff,
                  self.h[3] + d & 0Xff_ff_ff_ff,
                  self.h[4] + e & 0Xff_ff_ff_ff,
              )
         return ("{:08x}" * 5).format(*self.h    )







def lowerCamelCase__  (     ):

     '''simple docstring'''


     UpperCAmelCase_     :				str  =     B'''Test String'''
     assert SHAaHash(SCREAMING_SNAKE_CASE__ ).final_hash() == hashlib.shaa(SCREAMING_SNAKE_CASE__ ).hexdigest()  # noqa: S324







def lowerCamelCase__  (     ):

     '''simple docstring'''


     UpperCAmelCase_     :				Any  =     argparse.ArgumentParser(description='''Process some strings or files''' )
     parser.add_argument(
         '''--string'''					,		dest='''input_string'''					,		default='''Hello World!! Welcome to Cryptography'''					,		help='''Hash the string'''					,		)
     parser.add_argument('''--file'''					,		dest='''input_file'''					,		help='''Hash contents of a file''' )
     UpperCAmelCase_     :				Optional[int]  =     parser.parse_args()
     UpperCAmelCase_     :				str  =     args.input_string
     # In any case hash input should be a bytestring
     if args.input_file:
          with open(args.input_file					,		'''rb''' ) as f:
               UpperCAmelCase_     :				Optional[int]  =     f.read()
     else:
          UpperCAmelCase_     :				Dict  =     bytes(SCREAMING_SNAKE_CASE__					,		'''utf-8''' )
     print(SHAaHash(SCREAMING_SNAKE_CASE__ ).final_hash() )


if __name__ == "__main__":
     main()
     import doctest

     doctest.testmod()







'''simple docstring'''



import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase   :							int         =   logging.get_logger(__name__)

__UpperCamelCase   :							List[str]         =   {
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

__UpperCamelCase   :							List[str]         =   {
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

__UpperCamelCase   :							Union[str, Any]         =   {
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						int     )	->      List[str]:

   """simple docstring"""
   __a					  =     set()
   __a					  =     word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)     )
      __a					  =     char

   __a					  =     set(SCREAMING_SNAKE_CASE__     )
   return pairs



class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase  ):
      __a	       =VOCAB_FILES_NAMES
      __a	       =PRETRAINED_VOCAB_FILES_MAP
      __a	       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




      def __init__(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase="<s>"			, lowerCamelCase="</s>"			, lowerCamelCase="</s>"			, lowerCamelCase="<s>"			, lowerCamelCase="<unk>"			, lowerCamelCase="<pad>"			, lowerCamelCase="<mask>"			, **lowerCamelCase			, )				->str:



         '''simple docstring'''
         super().__init__(
             bos_token=lowerCamelCase			, eos_token=lowerCamelCase			, unk_token=lowerCamelCase			, sep_token=lowerCamelCase			, cls_token=lowerCamelCase			, pad_token=lowerCamelCase			, mask_token=lowerCamelCase			, **lowerCamelCase			, )

         __a					  =     vocab_file
         __a					  =     merges_file

         __a					  =     {}
         __a					  =     0
         __a					  =     1
         __a					  =     2
         __a					  =     3

         self.add_from_file(lowerCamelCase   )

         __a					  =     {v: k for k, v in self.encoder.items()}

         with open(lowerCamelCase			, encoding='utf-8'   ) as merges_handle:
            __a					  =     merges_handle.read().split('\n'   )[:-1]
         __a					  =     [tuple(merge.split()[:-1]   ) for merge in merges]
         __a					  =     dict(zip(lowerCamelCase			, range(len(lowerCamelCase   )   )   )   )
         __a					  =     {}




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None   )				->List[int]:



         '''simple docstring'''

         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __a					  =     [self.cls_token_id]
         __a					  =     [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None			, lowerCamelCase = False   )				->List[int]:



         '''simple docstring'''

         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=lowerCamelCase			, token_ids_a=lowerCamelCase			, already_has_special_tokens=lowerCamelCase   )

         if token_ids_a is None:
            return [1] + ([0] * len(lowerCamelCase   )) + [1]
         return [1] + ([0] * len(lowerCamelCase   )) + [1, 1] + ([0] * len(lowerCamelCase   )) + [1]




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None   )				->List[int]:



         '''simple docstring'''
         __a					  =     [self.sep_token_id]
         __a					  =     [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]




      @property
      def 			__UpperCamelCase				(	self   )				->Union[str, Any]:



         '''simple docstring'''
         return len(self.encoder   )




      def 			__UpperCamelCase				(	self   )				->Optional[int]:



         '''simple docstring'''
         return dict(self.encoder			, **self.added_tokens_encoder   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->List[str]:



         '''simple docstring'''
         if token in self.cache:
            return self.cache[token]
         __a					  =     tuple(lowerCamelCase   )
         __a					  =     tuple(list(word[:-1]   ) + [word[-1] + '</w>']   )
         __a					  =     get_pairs(lowerCamelCase   )

         if not pairs:
            return token

         while True:
            __a					  =     min(lowerCamelCase			, key=lambda lowerCamelCase   : self.bpe_ranks.get(lowerCamelCase			, float('inf'   )   )   )
            if bigram not in self.bpe_ranks:
               break
            __a  ,      __a					  =     bigram
            __a					  =     []
            __a					  =     0
            while i < len(lowerCamelCase   ):
               try:
                  __a					  =     word.index(lowerCamelCase			, lowerCamelCase   )
               except ValueError:
                  new_word.extend(word[i:]   )
                  break
               else:
                  new_word.extend(word[i:j]   )
                  __a					  =     j

               if word[i] == first and i < len(lowerCamelCase   ) - 1 and word[i + 1] == second:
                  new_word.append(first + second   )
                  i += 2
               else:
                  new_word.append(word[i]   )
                  i += 1
            __a					  =     tuple(lowerCamelCase   )
            __a					  =     new_word
            if len(lowerCamelCase   ) == 1:
               break
            else:
               __a					  =     get_pairs(lowerCamelCase   )
         __a					  =     '@@ '.join(lowerCamelCase   )
         __a					  =     word[:-4]
         __a					  =     word
         return word




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->Dict:



         '''simple docstring'''
         __a					  =     []

         __a					  =     re.findall(r'\S+\n?'			, lowerCamelCase   )

         for token in words:
            split_tokens.extend(list(self.bpe(lowerCamelCase   ).split(' '   )   )   )
         return split_tokens




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->List[Any]:



         '''simple docstring'''
         return self.encoder.get(lowerCamelCase			, self.encoder.get(self.unk_token   )   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->List[Any]:



         '''simple docstring'''
         return self.decoder.get(lowerCamelCase			, self.unk_token   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->Dict:



         '''simple docstring'''
         __a					  =     ' '.join(lowerCamelCase   ).replace('@@ '			, ''   ).strip()
         return out_string




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None   )				->Tuple[str]:



         '''simple docstring'''
         if not os.path.isdir(lowerCamelCase   ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
            return
         __a					  =     os.path.join(
             lowerCamelCase			, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
         __a					  =     os.path.join(
             lowerCamelCase			, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase   ):
            copyfile(self.vocab_file			, lowerCamelCase   )

         if os.path.abspath(self.merges_file   ) != os.path.abspath(lowerCamelCase   ):
            copyfile(self.merges_file			, lowerCamelCase   )

         return out_vocab_file, out_merge_file






      def 			__UpperCamelCase				(	self			, lowerCamelCase   )				->str:



         '''simple docstring'''
         if isinstance(lowerCamelCase			, lowerCamelCase   ):
            try:
               with open(lowerCamelCase			, 'r'			, encoding='utf-8'   ) as fd:
                  self.add_from_file(lowerCamelCase   )
            except FileNotFoundError as fnfe:
               raise fnfe
            except UnicodeError:
               raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""   )
            return

         __a					  =     f.readlines()
         for lineTmp in lines:
            __a					  =     lineTmp.strip()
            __a					  =     line.rfind(' '   )
            if idx == -1:
               raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''   )
            __a					  =     line[:idx]
            __a					  =     len(self.encoder   )