


from math import sqrt
def     __lowerCamelCase						(			UpperCAmelCase_						:		int						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          number >= 0
      ), "'number' must been an int and positive"

      a						:str										=	True

      # 0 and 1 are none primes.
      if number <= 1:
            a						:Union[str, Any]										=	False

      for divisor in range(2			,  int(round(sqrt(UpperCAmelCase_						)						)						) + 1						):
            # if 'number' divisible by 'divisor' then sets 'status'
            # of false and break up the loop.
            if number % divisor == 0:
                  a						:List[str]										=	False
                  break

    # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'status' must been from type bool"

      return status
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n > 2), "'N' must been an int and > 2"

      # beginList: contains all natural numbers from 2 up to N
      a						:Optional[int]										=	list(range(2			,  n + 1						)						)

      a						:Union[str, Any]										=	[]  # this list will be returns.

      # actual sieve of erathostenes
      for i in range(len(UpperCAmelCase_						)						):
            for j in range(i + 1			,  len(UpperCAmelCase_						)						):
                  if (begin_list[i] != 0) and (begin_list[j] % begin_list[i] == 0):
                        a						:Optional[Any]										=	0

    # filters actual prime numbers.
      a						:int										=	[x for x in begin_list if x != 0]

      # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'ans' must been from type list"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Dict						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n > 2), "'N' must been an int and > 2"

      a						:List[str]										=	[]

      # iterates over all numbers between 2 up to N+1
      # if a number is prime then appends to list 'ans'
      for number in range(2			,  n + 1						):
            if is_prime(UpperCAmelCase_						):
                  ans.append(UpperCAmelCase_						)

    # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'ans' must been from type list"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and number >= 0, "'number' must been an int and >= 0"

      a						:str										=	[]  # this list will be returns of the function.

      # potential prime number factors.

      a						:Any										=	2

      a						:int										=	number

      if number == 0 or number == 1:
            ans.append(UpperCAmelCase_						)

      # if 'number' not prime then builds the prime factorization of 'number'
      elif not is_prime(UpperCAmelCase_						):
            while quotient != 1:
                  if is_prime(UpperCAmelCase_						) and (quotient % factor == 0):
                        ans.append(UpperCAmelCase_						)
                        quotient /= factor
                  else:
                        factor += 1

      else:
            ans.append(UpperCAmelCase_						)

      # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'ans' must been from type list"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[Any]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          number >= 0
      ), "'number' bust been an int and >= 0"

      a						:Tuple										=	0

      # prime factorization of 'number'
      a						:Union[str, Any]										=	prime_factorization(UpperCAmelCase_						)

      a						:Tuple										=	max(UpperCAmelCase_						)

      # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'ans' must been from type int"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Any						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          number >= 0
      ), "'number' bust been an int and >= 0"

      a						:Any										=	0

      # prime factorization of 'number'
      a						:Any										=	prime_factorization(UpperCAmelCase_						)

      a						:Any										=	min(UpperCAmelCase_						)

      # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'ans' must been from type int"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'number' must been an int"
      assert isinstance(number % 2 == 0			,  UpperCAmelCase_						), "compare bust been from type bool"

      return number % 2 == 0
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[int]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						), "'number' must been an int"
      assert isinstance(number % 2 != 0			,  UpperCAmelCase_						), "compare bust been from type bool"

      return number % 2 != 0
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (number > 2) and is_even(UpperCAmelCase_						)
      ), "'number' must been an int, even and > 2"

      a						:Optional[int]										=	[]  # this list will returned

      # creates a list of prime numbers between 2 up to 'number'
      a						:Optional[Any]										=	get_prime_numbers(UpperCAmelCase_						)
      a						:List[Any]										=	len(UpperCAmelCase_						)

      # run variable for while-loops.
      a						:int										=	0
      a						:Optional[int]										=	None

      # exit variable. for break up the loops
      a						:Optional[Any]										=	True

      while i < len_pn and loop:
            a						:List[Any]										=	i + 1

            while j < len_pn and loop:
                  if prime_numbers[i] + prime_numbers[j] == number:
                        a						:Optional[int]										=	False
                        ans.append(prime_numbers[i]						)
                        ans.append(prime_numbers[j]						)

                  j += 1

            i += 1

      # precondition
      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (len(UpperCAmelCase_						) == 2)
          and (ans[0] + ans[1] == number)
          and is_prime(ans[0]						)
          and is_prime(ans[1]						)
      ), "'ans' must contains two primes. And sum of elements must been eq 'number'"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		str			,  UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (numbera >= 0)
          and (numbera >= 0)
      ), "'number1' and 'number2' must been positive integer."

      a						:Dict										=	0

      while numbera != 0:
            a						:Any										=	numbera % numbera
            a						:str										=	numbera
            a						:Dict										=	rest

      # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          numbera >= 0
      ), "'number' must been from type int and positive"

      return numbera
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (numbera >= 1)
          and (numbera >= 1)
      ), "'number1' and 'number2' must been positive integer."

      a						:List[str]										=	1  # actual answer that will be return.

      # for kgV (x,1)
      if numbera > 1 and numbera > 1:
            # builds the prime factorization of 'number1' and 'number2'
            a						:Tuple										=	prime_factorization(UpperCAmelCase_						)
            a						:Tuple										=	prime_factorization(UpperCAmelCase_						)

      elif numbera == 1 or numbera == 1:
            a						:str										=	[]
            a						:Any										=	[]
            a						:List[Any]										=	max(UpperCAmelCase_			,  UpperCAmelCase_						)

      a						:List[Any]										=	0
      a						:Optional[int]										=	0

      a						:Optional[int]										=	[]  # captured numbers int both 'primeFac1' and 'primeFac2'

      # iterates through primeFac1
      for n in prime_fac_a:
            if n not in done:
                  if n in prime_fac_a:
                        a						:Union[str, Any]										=	prime_fac_a.count(UpperCAmelCase_						)
                        a						:int										=	prime_fac_a.count(UpperCAmelCase_						)

                        for _ in range(max(UpperCAmelCase_			,  UpperCAmelCase_						)						):
                              ans *= n

                  else:
                        a						:Optional[Any]										=	prime_fac_a.count(UpperCAmelCase_						)

                        for _ in range(UpperCAmelCase_						):
                              ans *= n

                  done.append(UpperCAmelCase_						)

    # iterates through primeFac2
      for n in prime_fac_a:
            if n not in done:
                  a						:Dict										=	prime_fac_a.count(UpperCAmelCase_						)

                  for _ in range(UpperCAmelCase_						):
                        ans *= n

                  done.append(UpperCAmelCase_						)

    # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          ans >= 0
      ), "'ans' must been from type int and positive"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[int]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n >= 0), "'number' must been a positive int"

      a						:int										=	0
      a						:Optional[int]										=	2  # this variable holds the answer

      while index < n:
            index += 1

            ans += 1  # counts to the next number

            # if ans not prime then
            # runs to the next prime number.
            while not is_prime(UpperCAmelCase_						):
                  ans += 1

    # precondition
      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and is_prime(
          UpperCAmelCase_						), "'ans' must been a prime number and from type int"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Optional[int]						):




      """simple docstring"""





      assert (
          is_prime(UpperCAmelCase_						) and is_prime(UpperCAmelCase_						) and (p_number_a < p_number_a)
      ), "The arguments must been prime numbers and 'pNumber1' < 'pNumber2'"

      a						:Any										=	p_number_a + 1  # jump to the next number

      a						:Optional[Any]										=	[]  # this list will be returns.

      # if number is not prime then
      # fetch the next prime number.
      while not is_prime(UpperCAmelCase_						):
            number += 1

      while number < p_number_a:
            ans.append(UpperCAmelCase_						)

            number += 1

            # fetch the next prime number.
            while not is_prime(UpperCAmelCase_						):
                  number += 1

    # precondition
      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and ans[0] != p_number_a
          and ans[len(UpperCAmelCase_						) - 1] != p_number_a
      ), "'ans' must been a list without the arguments"

      # 'ans' contains not 'pNumber1' and 'pNumber2' !
      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Dict						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n >= 1), "'n' must been int and >= 1"

      a						:Dict										=	[]  # will be returned.

      for divisor in range(1			,  n + 1						):
            if n % divisor == 0:
                  ans.append(UpperCAmelCase_						)

    # precondition
      assert ans[0] == 1 and ans[len(UpperCAmelCase_						) - 1] == n, "Error in function getDivisiors(...)"

      return ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (
          number > 1
      ), "'number' must been an int and >= 1"

      a						:int										=	get_divisors(UpperCAmelCase_						)

      # precondition
      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (divisors[0] == 1)
          and (divisors[len(UpperCAmelCase_						) - 1] == number)
      ), "Error in help-function getDivisiors(...)"

      # summed all divisors up to 'number' (exclusive), hence [:-1]
      return sum(divisors[:-1]						) == number
def     __lowerCamelCase						(			UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		int						):




      """simple docstring"""





      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (denominator != 0)
      ), "The arguments must been from type int and 'denominator' != 0"

      # build the greatest common divisor of numerator and denominator.
      a						:Tuple										=	gcd(abs(UpperCAmelCase_						)			,  abs(UpperCAmelCase_						)						)

      # precondition
      assert (
          isinstance(UpperCAmelCase_			,  UpperCAmelCase_						)
          and (numerator % gcd_of_fraction == 0)
          and (denominator % gcd_of_fraction == 0)
      ), "Error in function gcd(...,...)"

      return (numerator // gcd_of_fraction, denominator // gcd_of_fraction)
def     __lowerCamelCase						(			UpperCAmelCase_						:		Any						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n >= 0), "'n' must been a int and >= 0"

      a						:str										=	1  # this will be return.

      for factor in range(1			,  n + 1						):
            ans *= factor

      return ans






def     __lowerCamelCase						(			UpperCAmelCase_						:		int						):




      """simple docstring"""





      assert isinstance(UpperCAmelCase_			,  UpperCAmelCase_						) and (n >= 0), "'n' must been an int and >= 0"

      a						:List[str]										=	0
      a						:List[str]										=	1
      a						:int										=	1  # this will be return

      for _ in range(n - 1						):
            a						:Tuple										=	ans
            ans += fiba
            a						:Optional[Any]										=	tmp

      return ans









import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case    :   List[str]	    =					logging.get_logger(__name__)

snake_case    :   Optional[Any]	    =					{
    '''vocab_file''': '''vocab.txt''',
    '''merges_file''': '''bpe.codes''',
}

snake_case    :   str	    =					{
    '''vocab_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt''',
    },
    '''merges_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes''',
    },
}

snake_case    :   List[Any]	    =					{
    '''vinai/phobert-base''': 2_56,
    '''vinai/phobert-large''': 2_56,
}
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[str]						):




      """simple docstring"""





      a						:Union[str, Any]										=	set()
      a						:str										=	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)						)
            a						:Optional[int]										=	char

      a						:Optional[int]										=	set(UpperCAmelCase_						)
      return pairs







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__				=	PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(		self ,  _lowerCamelCase ,  _lowerCamelCase ,  _lowerCamelCase="<s>" ,  _lowerCamelCase="</s>" ,  _lowerCamelCase="</s>" ,  _lowerCamelCase="<s>" ,  _lowerCamelCase="<unk>" ,  _lowerCamelCase="<pad>" ,  _lowerCamelCase="<mask>" ,  **_lowerCamelCase ,  ):
       super().__init__(
           bos_token=_lowerCamelCase ,  eos_token=_lowerCamelCase ,  unk_token=_lowerCamelCase ,  sep_token=_lowerCamelCase ,  cls_token=_lowerCamelCase ,  pad_token=_lowerCamelCase ,  mask_token=_lowerCamelCase ,  **_lowerCamelCase ,  )

       a						:Optional[Any]										=	vocab_file
       a						:Optional[Any]										=	merges_file

       a						:Any										=	{}
       a						:Any										=	0
       a						:int										=	1
       a						:Union[str, Any]										=	2
       a						:List[Any]										=	3

       self.add_from_file(_lowerCamelCase   )

       a						:List[str]										=	{v: k for k, v in self.encoder.items()}

       with open(_lowerCamelCase ,  encoding='''utf-8'''   ) as merges_handle:
             a						:List[str]										=	merges_handle.read().split('''\n'''   )[:-1]
       a						:Any										=	[tuple(merge.split()[:-1]   ) for merge in merges]
       a						:str										=	dict(zip(_lowerCamelCase ,  range(len(_lowerCamelCase   )   )   )   )
       a						:str										=	{}




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):

       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       a						:Union[str, Any]										=	[self.cls_token_id]
       a						:Tuple										=	[self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None ,  _lowerCamelCase = False   ):

       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCamelCase ,  token_ids_a=_lowerCamelCase ,  already_has_special_tokens=_lowerCamelCase   )

       if token_ids_a is None:
             return [1] + ([0] * len(_lowerCamelCase   )) + [1]
       return [1] + ([0] * len(_lowerCamelCase   )) + [1, 1] + ([0] * len(_lowerCamelCase   )) + [1]




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):
       a						:Optional[int]										=	[self.sep_token_id]
       a						:Optional[Any]										=	[self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep   ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]




 @property
 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return len(self.encoder   )




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       return dict(self.encoder ,  **self.added_tokens_encoder   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       if token in self.cache:
             return self.cache[token]
       a						:Optional[int]										=	tuple(_lowerCamelCase   )
       a						:List[str]										=	tuple(list(word[:-1]   ) + [word[-1] + '''</w>''']   )
       a						:Union[str, Any]										=	get_pairs(_lowerCamelCase   )

       if not pairs:
             return token

       while True:
             a						:Optional[Any]										=	min(_lowerCamelCase ,  key=lambda _lowerCamelCase   : self.bpe_ranks.get(_lowerCamelCase ,  float('''inf'''   )   )   )
             if bigram not in self.bpe_ranks:
                   break
             a							,    a						:Dict										=	bigram
             a						:Union[str, Any]										=	[]
             a						:int										=	0
             while i < len(_lowerCamelCase   ):
                   try:
                         a						:Optional[Any]										=	word.index(_lowerCamelCase ,  _lowerCamelCase   )
                   except ValueError:
                         new_word.extend(word[i:]   )
                         break
                   else:
                         new_word.extend(word[i:j]   )
                         a						:Union[str, Any]										=	j

                   if word[i] == first and i < len(_lowerCamelCase   ) - 1 and word[i + 1] == second:
                         new_word.append(first + second   )
                         i += 2
                   else:
                         new_word.append(word[i]   )
                         i += 1
             a						:Union[str, Any]										=	tuple(_lowerCamelCase   )
             a						:int										=	new_word
             if len(_lowerCamelCase   ) == 1:
                   break
             else:
                   a						:List[str]										=	get_pairs(_lowerCamelCase   )
       a						:Union[str, Any]										=	'''@@ '''.join(_lowerCamelCase   )
       a						:Dict										=	word[:-4]
       a						:Any										=	word
       return word




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:Union[str, Any]										=	[]

       a						:str										=	re.findall(R'''\S+\n?''' ,  _lowerCamelCase   )

       for token in words:
             split_tokens.extend(list(self.bpe(_lowerCamelCase   ).split(''' '''   )   )   )
       return split_tokens




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       return self.encoder.get(_lowerCamelCase ,  self.encoder.get(self.unk_token   )   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       return self.decoder.get(_lowerCamelCase ,  self.unk_token   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       a						:Optional[int]										=	''' '''.join(_lowerCamelCase   ).replace('''@@ ''' ,  ''''''   ).strip()
       return out_string




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase = None   ):
       if not os.path.isdir(_lowerCamelCase   ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
             return
       a						:Tuple										=	os.path.join(
           _lowerCamelCase ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
       a						:Optional[int]										=	os.path.join(
           _lowerCamelCase ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

       if os.path.abspath(self.vocab_file   ) != os.path.abspath(_lowerCamelCase   ):
             copyfile(self.vocab_file ,  _lowerCamelCase   )

       if os.path.abspath(self.merges_file   ) != os.path.abspath(_lowerCamelCase   ):
             copyfile(self.merges_file ,  _lowerCamelCase   )

       return out_vocab_file, out_merge_file




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       if isinstance(_lowerCamelCase ,  _lowerCamelCase   ):
             try:
                   with open(_lowerCamelCase ,  '''r''' ,  encoding='''utf-8'''   ) as fd:
                         self.add_from_file(_lowerCamelCase   )
             except FileNotFoundError as fnfe:
                   raise fnfe
             except UnicodeError:
                   raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset'''   )
             return

       a						:str										=	f.readlines()
       for lineTmp in lines:
             a						:Tuple										=	lineTmp.strip()
             a						:int										=	line.rfind(''' '''   )
             if idx == -1:
                   raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''   )
             a						:Tuple										=	line[:idx]
             a						:Tuple										=	len(self.encoder   )





