

__A      		=     '''ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'''



def 				snake_case_(_UpperCamelCase					)							->     bytes:



  """simple docstring"""






  if not isinstance(_UpperCamelCase			,	_UpperCamelCase					):
    _snake_case						=							F"""a bytes-like object is required, not '{data.__class__.__name__}'"""
    raise TypeError(_UpperCamelCase					)

  _snake_case						=							''''''.join(bin(_UpperCamelCase					)[2:].zfill(8					) for byte in data					)

  _snake_case						=							len(_UpperCamelCase					) % 6 != 0

  if padding_needed:
    # The padding that will be added later
    _snake_case						=							b'''=''' * ((6 - len(_UpperCamelCase					) % 6) // 2)

    # Append binary_stream with arbitrary binary digits (0's by default) to make its
    # length a multiple of 6.
    binary_stream += "0" * (6 - len(_UpperCamelCase					) % 6)
  else:
    _snake_case						=							b''''''

  # Encode every 6 binary digits to their corresponding Base64 character
  return (
      "".join(
          B64_CHARSET[int(binary_stream[index : index + 6]			,	2					)]
          for index in range(0			,	len(_UpperCamelCase					)			,	6					)					).encode()
      + padding
  )



def 				snake_case_(_UpperCamelCase					)							->     bytes:



  """simple docstring"""






  if not isinstance(_UpperCamelCase			,	_UpperCamelCase					) and not isinstance(_UpperCamelCase			,	_UpperCamelCase					):
    _snake_case						=							(
        '''argument should be a bytes-like object or ASCII string, '''
        F"""not '{encoded_data.__class__.__name__}'"""
    )
    raise TypeError(_UpperCamelCase					)

  # In case encoded_data is a bytes-like object, make sure it contains only
  # ASCII characters so we convert it to a string object
  if isinstance(_UpperCamelCase			,	_UpperCamelCase					):
    try:
      _snake_case						=							encoded_data.decode('''utf-8'''					)
    except UnicodeDecodeError:
      raise ValueError('''base64 encoded data should only contain ASCII characters'''					)

  _snake_case						=							encoded_data.count('''='''					)

  # Check if the encoded string contains non base64 characters
  if padding:
    assert all(
        char in B64_CHARSET for char in encoded_data[:-padding]					), "Invalid base64 character(s) found."
  else:
    assert all(
        char in B64_CHARSET for char in encoded_data					), "Invalid base64 character(s) found."

  # Check the padding
  assert len(_UpperCamelCase					) % 4 == 0 and padding < 3, "Incorrect padding"

  if padding:
    # Remove padding if there is one
    _snake_case						=							encoded_data[:-padding]

    _snake_case						=							''''''.join(
        bin(B64_CHARSET.index(_UpperCamelCase					)					)[2:].zfill(6					) for char in encoded_data					)[: -padding * 2]
  else:
    _snake_case						=							''''''.join(
        bin(B64_CHARSET.index(_UpperCamelCase					)					)[2:].zfill(6					) for char in encoded_data					)

  _snake_case						=							[
      int(binary_stream[index : index + 8]			,	2					)
      for index in range(0			,	len(_UpperCamelCase					)			,	8					)
  ]

  return bytes(_UpperCamelCase					)


if __name__ == "__main__":
 import doctest

 doctest.testmod()




from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__A      		=     logging.get_logger(__name__)

__A      		=     {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A      		=     {
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

__A      		=     {'''allegro/herbert-base-cased''': 5_14}
__A      		=     {}



class   lowercase_   (  __lowercase					):
      UpperCamelCase_      :		Any										=    VOCAB_FILES_NAMES
      UpperCamelCase_      :		Optional[int]										=    PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_      :		List[str]										=    PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase_      :		Dict										=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_      :		Optional[Any]										=    HerbertTokenizer



      def __init__(      self				:    Tuple      , A__				:    str=None      , A__				:    Optional[Any]=None      , A__				:    Union[str, Any]=None      , A__				:    Optional[int]="<s>"      , A__				:    Optional[int]="<unk>"      , A__				:    str="<pad>"      , A__				:    List[Any]="<mask>"      , A__				:    Dict="</s>"      , **A__				:    Optional[int]      , )   ->					Optional[int]:
        super().__init__(
            A__      , A__      , tokenizer_file=A__      , cls_token=A__      , unk_token=A__      , pad_token=A__      , mask_token=A__      , sep_token=A__      , **A__      , )



      def 						UpperCamelCase_		(      self				:    str      , A__				:    List[int]      , A__				:    Optional[List[int]] = None		)   ->					List[int]:
        _snake_case						=							[self.cls_token_id]
        _snake_case						=							[self.sep_token_id]
        if token_ids_a is None:
          return cls + token_ids_a + sep

        return cls + token_ids_a + sep + token_ids_a + sep



      def 						UpperCamelCase_		(      self				:    Tuple      , A__				:    List[int]      , A__				:    Optional[List[int]] = None      , A__				:    bool = False		)   ->					List[int]:
        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=A__      , token_ids_a=A__      , already_has_special_tokens=A__		)

        if token_ids_a is None:
          return [1] + ([0] * len(A__		)) + [1]
        return [1] + ([0] * len(A__		)) + [1] + ([0] * len(A__		)) + [1]



      def 						UpperCamelCase_		(      self				:    Any      , A__				:    List[int]      , A__				:    Optional[List[int]] = None		)   ->					List[int]:
        _snake_case						=							[self.sep_token_id]
        _snake_case						=							[self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep		) * [0]
        return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







      def 						UpperCamelCase_		(      self				:    Union[str, Any]      , A__				:    str      , A__				:    Optional[str] = None		)   ->					Tuple[str]:
        _snake_case						=							self._tokenizer.model.save(A__      , name=A__		)
        return tuple(A__		)

