


"""simple docstring"""




import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding


class 			__magic_name__       (       __SCREAMING_SNAKE_CASE ):
       UpperCamelCase_   =		['image_processor', 'tokenizer']
       UpperCamelCase_   =		'ChineseCLIPImageProcessor'
       UpperCamelCase_   =		('BertTokenizer', 'BertTokenizerFast')
       def __init__(     self ,				A_=None ,				A_=None ,				**A_       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     None
         if "feature_extractor" in kwargs:
           warnings.warn(
               '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
               ''' instead.''' ,				_a ,				)
           _lowercase:			str       =     kwargs.pop('''feature_extractor'''       )

         _lowercase:			str       =     image_processor if image_processor is not None else feature_extractor
         if image_processor is None:
           raise ValueError('''You need to specify an `image_processor`.'''       )
         if tokenizer is None:
           raise ValueError('''You need to specify a `tokenizer`.'''       )

         super().__init__(_a ,				_a       )
         _lowercase:			Union[str, Any]       =     self.image_processor
       def __call__(     self ,				A_=None ,				A_=None ,				A_=None ,				**A_       )     ->	Dict:





         """simple docstring"""




         if text is None and images is None:
           raise ValueError('''You have to specify either text or images. Both cannot be none.'''       )

         if text is not None:
           _lowercase:			Optional[Any]       =     self.tokenizer(_a ,				return_tensors=_a ,				**_a       )

         if images is not None:
           _lowercase:			Dict       =     self.image_processor(_a ,				return_tensors=_a ,				**_a       )

         if text is not None and images is not None:
           _lowercase:			Tuple       =     image_features.pixel_values
           return encoding
         elif text is not None:
           return encoding
         else:
           return BatchEncoding(data=dict(**_a       ) ,				tensor_type=_a       )
       def 							lowercase_							(     self ,				*A_ ,				**A_       )     ->	Optional[Any]:





         """simple docstring"""




         return self.tokenizer.batch_decode(*_a ,				**_a       )
       def 							lowercase_							(     self ,				*A_ ,				**A_       )     ->	int:





         """simple docstring"""




         return self.tokenizer.decode(*_a ,				**_a       )
       @property
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     self.tokenizer.model_input_names
         _lowercase:			Dict       =     self.image_processor.model_input_names
         return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names       )       )



       @property
       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         warnings.warn(
             '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.''' ,				_a ,				)
         return self.image_processor_class


"""simple docstring"""




import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__   :					str							=   logging.get_logger(__name__)

A__   :					Tuple							=   {
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

A__   :					List[Any]							=   {
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

A__   :					Any							=   {
    'vinai/phobert-base': 2_5_6,
    'vinai/phobert-large': 2_5_6,
}




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  _lowercase:			Optional[int]       =     set()
  _lowercase:			Union[str, Any]       =     word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)						)
    _lowercase:			Tuple       =     char

  _lowercase:			Optional[Any]       =     set(_UpperCamelCase						)
  return pairs







class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		VOCAB_FILES_NAMES
       UpperCamelCase_   =		PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase_   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       def __init__(     self ,				A_ ,				A_ ,				A_="<s>" ,				A_="</s>" ,				A_="</s>" ,				A_="<s>" ,				A_="<unk>" ,				A_="<pad>" ,				A_="<mask>" ,				**A_ ,				)     ->	Dict:





         """simple docstring"""




         super().__init__(
             bos_token=A_ ,				eos_token=A_ ,				unk_token=A_ ,				sep_token=A_ ,				cls_token=A_ ,				pad_token=A_ ,				mask_token=A_ ,				**A_ ,				)

         _lowercase:			List[Any]       =     vocab_file
         _lowercase:			Union[str, Any]       =     merges_file

         _lowercase:			int       =     {}
         _lowercase:			Optional[int]       =     0
         _lowercase:			Optional[Any]       =     1
         _lowercase:			List[str]       =     2
         _lowercase:			List[Any]       =     3

         self.add_from_file(A_       )

         _lowercase:			Tuple       =     {v: k for k, v in self.encoder.items()}

         with open(A_ ,				encoding='''utf-8'''       ) as merges_handle:
           _lowercase:			int       =     merges_handle.read().split('''\n'''       )[:-1]
         _lowercase:			Optional[int]       =     [tuple(merge.split()[:-1]       ) for merge in merges]
         _lowercase:			List[str]       =     dict(zip(A_ ,				range(len(A_       )       )       )       )
         _lowercase:			Union[str, Any]       =     {}
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	List[int]:





         """simple docstring"""




         if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _lowercase:			Optional[Any]       =     [self.cls_token_id]
         _lowercase:			Tuple       =     [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
       def 							lowercase_							(     self ,				A_ ,				A_ = None ,				A_ = False       )     ->	List[int]:





         """simple docstring"""




         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A_ ,				token_ids_a=A_ ,				already_has_special_tokens=A_       )

         if token_ids_a is None:
           return [1] + ([0] * len(A_       )) + [1]
         return [1] + ([0] * len(A_       )) + [1, 1] + ([0] * len(A_       )) + [1]
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	List[int]:





         """simple docstring"""




         _lowercase:			Tuple       =     [self.sep_token_id]
         _lowercase:			List[Any]       =     [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
       @property
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         return len(self.encoder       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         return dict(self.encoder ,				**self.added_tokens_encoder       )
       def 							lowercase_							(     self ,				A_       )     ->	Optional[Any]:





         """simple docstring"""




         if token in self.cache:
           return self.cache[token]
         _lowercase:			List[Any]       =     tuple(A_       )
         _lowercase:			Tuple       =     tuple(list(word[:-1]       ) + [word[-1] + '''</w>''']       )
         _lowercase:			List[Any]       =     get_pairs(A_       )

         if not pairs:
           return token

         while True:
           _lowercase:			List[str]       =     min(A_ ,				key=lambda A_       : self.bpe_ranks.get(A_ ,				float('''inf'''       )       )       )
           if bigram not in self.bpe_ranks:
             break
           _lowercase ,				_lowercase:			Optional[int]       =     bigram
           _lowercase:			Tuple       =     []
           _lowercase:			List[str]       =     0
           while i < len(A_       ):
             try:
               _lowercase:			Tuple       =     word.index(A_ ,				A_       )
             except ValueError:
               new_word.extend(word[i:]       )
               break
             else:
               new_word.extend(word[i:j]       )
               _lowercase:			List[Any]       =     j

             if word[i] == first and i < len(A_       ) - 1 and word[i + 1] == second:
               new_word.append(first + second       )
               i += 2
             else:
               new_word.append(word[i]       )
               i += 1
           _lowercase:			Union[str, Any]       =     tuple(A_       )
           _lowercase:			int       =     new_word
           if len(A_       ) == 1:
             break
           else:
             _lowercase:			List[Any]       =     get_pairs(A_       )
         _lowercase:			Tuple       =     '''@@ '''.join(A_       )
         _lowercase:			Dict       =     word[:-4]
         _lowercase:			Tuple       =     word
         return word
       def 							lowercase_							(     self ,				A_       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			Dict       =     []

         _lowercase:			str       =     re.findall(R'''\S+\n?''' ,				A_       )

         for token in words:
           split_tokens.extend(list(self.bpe(A_       ).split(''' '''       )       )       )
         return split_tokens
       def 							lowercase_							(     self ,				A_       )     ->	List[str]:





         """simple docstring"""




         return self.encoder.get(A_ ,				self.encoder.get(self.unk_token       )       )
       def 							lowercase_							(     self ,				A_       )     ->	Tuple:





         """simple docstring"""




         return self.decoder.get(A_ ,				self.unk_token       )
       def 							lowercase_							(     self ,				A_       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			Union[str, Any]       =     ''' '''.join(A_       ).replace('''@@ ''' ,				''''''       ).strip()
         return out_string
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	Tuple[str]:





         """simple docstring"""




         if not os.path.isdir(A_       ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
           return
         _lowercase:			List[Any]       =     os.path.join(
             A_ ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
         _lowercase:			str       =     os.path.join(
             A_ ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

         if os.path.abspath(self.vocab_file       ) != os.path.abspath(A_       ):
           copyfile(self.vocab_file ,				A_       )

         if os.path.abspath(self.merges_file       ) != os.path.abspath(A_       ):
           copyfile(self.merges_file ,				A_       )

         return out_vocab_file, out_merge_file



       def 							lowercase_							(     self ,				A_       )     ->	str:





         """simple docstring"""




         if isinstance(A_ ,				A_       ):
           try:
             with open(A_ ,				'''r''' ,				encoding='''utf-8'''       ) as fd:
               self.add_from_file(A_       )
           except FileNotFoundError as fnfe:
             raise fnfe
           except UnicodeError:
             raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''       )
           return

         _lowercase:			Dict       =     f.readlines()
         for lineTmp in lines:
           _lowercase:			List[Any]       =     lineTmp.strip()
           _lowercase:			Optional[Any]       =     line.rfind(''' '''       )
           if idx == -1:
             raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''       )
           _lowercase:			List[str]       =     line[:idx]
           _lowercase:			List[str]       =     len(self.encoder       )
