





import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A					:				List[str]				=			logging.get_logger(__name__)

_A					:				str				=			{
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

_A					:				Optional[Any]				=			{
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

_A					:				Optional[int]				=			{
    'vinai/phobert-base': 2_56,
    'vinai/phobert-large': 2_56,
}


def 							__snake_case							(    lowerCAmelCase_	)				->						List[str]:
    SCREAMING_SNAKE_CASE__											=			set()
    SCREAMING_SNAKE_CASE__											=			word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)	)
        SCREAMING_SNAKE_CASE__											=			char

    SCREAMING_SNAKE_CASE__											=			set(__snake_case	)
    return pairs



class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    lowerCamelCase__    :		Optional[Any]				=	VOCAB_FILES_NAMES
    lowerCamelCase__    :		Union[str, Any]				=	PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase__    :		Any				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



    def __init__(       self   ,				A_   ,				A_   ,				A_="<s>"   ,				A_="</s>"   ,				A_="</s>"   ,				A_="<s>"   ,				A_="<unk>"   ,				A_="<pad>"   ,				A_="<mask>"   ,				**A_   ,				):





        '''simple docstring'''

        super().__init__(
            bos_token=_SCREAMING_SNAKE_CASE   ,				eos_token=_SCREAMING_SNAKE_CASE   ,				unk_token=_SCREAMING_SNAKE_CASE   ,				sep_token=_SCREAMING_SNAKE_CASE   ,				cls_token=_SCREAMING_SNAKE_CASE   ,				pad_token=_SCREAMING_SNAKE_CASE   ,				mask_token=_SCREAMING_SNAKE_CASE   ,				**_SCREAMING_SNAKE_CASE   ,				)

        SCREAMING_SNAKE_CASE__											=			vocab_file
        SCREAMING_SNAKE_CASE__											=			merges_file

        SCREAMING_SNAKE_CASE__											=			{}
        SCREAMING_SNAKE_CASE__											=			0
        SCREAMING_SNAKE_CASE__											=			1
        SCREAMING_SNAKE_CASE__											=			2
        SCREAMING_SNAKE_CASE__											=			3

        self.add_from_file(_SCREAMING_SNAKE_CASE     )

        SCREAMING_SNAKE_CASE__											=			{v: k for k, v in self.encoder.items()}

        with open(_SCREAMING_SNAKE_CASE   ,				encoding='''utf-8'''     ) as merges_handle:
            SCREAMING_SNAKE_CASE__											=			merges_handle.read().split('''\n'''     )[:-1]
        SCREAMING_SNAKE_CASE__											=			[tuple(merge.split()[:-1]     ) for merge in merges]
        SCREAMING_SNAKE_CASE__											=			dict(zip(_SCREAMING_SNAKE_CASE   ,				range(len(_SCREAMING_SNAKE_CASE     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			{}



    def  lowercase_					(       self   ,				A_   ,				A_ = None     ):





        '''simple docstring'''

        if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        SCREAMING_SNAKE_CASE__											=			[self.cls_token_id]
        SCREAMING_SNAKE_CASE__											=			[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep



    def  lowercase_					(       self   ,				A_   ,				A_ = None   ,				A_ = False     ):





        '''simple docstring'''

        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_SCREAMING_SNAKE_CASE   ,				token_ids_a=_SCREAMING_SNAKE_CASE   ,				already_has_special_tokens=_SCREAMING_SNAKE_CASE     )

        if token_ids_a is None:
            return [1] + ([0] * len(_SCREAMING_SNAKE_CASE     )) + [1]
        return [1] + ([0] * len(_SCREAMING_SNAKE_CASE     )) + [1, 1] + ([0] * len(_SCREAMING_SNAKE_CASE     )) + [1]



    def  lowercase_					(       self   ,				A_   ,				A_ = None     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			[self.sep_token_id]
        SCREAMING_SNAKE_CASE__											=			[self.cls_token_id]

        if token_ids_a is None:
            return len(cls + token_ids_a + sep     ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]



    @property
    def  lowercase_					(       self     ):





        '''simple docstring'''

        return len(self.encoder     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        return dict(self.encoder   ,				**self.added_tokens_encoder     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if token in self.cache:
            return self.cache[token]
        SCREAMING_SNAKE_CASE__											=			tuple(_SCREAMING_SNAKE_CASE     )
        SCREAMING_SNAKE_CASE__											=			tuple(list(word[:-1]     ) + [word[-1] + '''</w>''']     )
        SCREAMING_SNAKE_CASE__											=			get_pairs(_SCREAMING_SNAKE_CASE     )

        if not pairs:
            return token

        while True:
            SCREAMING_SNAKE_CASE__											=			min(_SCREAMING_SNAKE_CASE   ,				key=lambda A_     : self.bpe_ranks.get(_SCREAMING_SNAKE_CASE   ,				float('''inf'''     )     )     )
            if bigram not in self.bpe_ranks:
                break
            SCREAMING_SNAKE_CASE__											=			bigram
            SCREAMING_SNAKE_CASE__											=			[]
            SCREAMING_SNAKE_CASE__											=			0
            while i < len(_SCREAMING_SNAKE_CASE     ):
                try:
                    SCREAMING_SNAKE_CASE__											=			word.index(_SCREAMING_SNAKE_CASE   ,				_SCREAMING_SNAKE_CASE     )
                except ValueError:
                    new_word.extend(word[i:]     )
                    break
                else:
                    new_word.extend(word[i:j]     )
                    SCREAMING_SNAKE_CASE__											=			j

                if word[i] == first and i < len(_SCREAMING_SNAKE_CASE     ) - 1 and word[i + 1] == second:
                    new_word.append(first + second     )
                    i += 2
                else:
                    new_word.append(word[i]     )
                    i += 1
            SCREAMING_SNAKE_CASE__											=			tuple(_SCREAMING_SNAKE_CASE     )
            SCREAMING_SNAKE_CASE__											=			new_word
            if len(_SCREAMING_SNAKE_CASE     ) == 1:
                break
            else:
                SCREAMING_SNAKE_CASE__											=			get_pairs(_SCREAMING_SNAKE_CASE     )
        SCREAMING_SNAKE_CASE__											=			"@@ ".join(_SCREAMING_SNAKE_CASE     )
        SCREAMING_SNAKE_CASE__											=			word[:-4]
        SCREAMING_SNAKE_CASE__											=			word
        return word



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			[]

        SCREAMING_SNAKE_CASE__											=			re.findall(r'''\S+\n?'''   ,				_SCREAMING_SNAKE_CASE     )

        for token in words:
            split_tokens.extend(list(self.bpe(_SCREAMING_SNAKE_CASE     ).split(''' '''     )     )     )
        return split_tokens



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        return self.encoder.get(_SCREAMING_SNAKE_CASE   ,				self.encoder.get(self.unk_token     )     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        return self.decoder.get(_SCREAMING_SNAKE_CASE   ,				self.unk_token     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			" ".join(_SCREAMING_SNAKE_CASE     ).replace('''@@ '''   ,				''''''     ).strip()
        return out_string



    def  lowercase_					(       self   ,				A_   ,				A_ = None     ):





        '''simple docstring'''

        if not os.path.isdir(_SCREAMING_SNAKE_CASE     ):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
            return
        SCREAMING_SNAKE_CASE__											=			os.path.join(
            _SCREAMING_SNAKE_CASE   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
        SCREAMING_SNAKE_CASE__											=			os.path.join(
            _SCREAMING_SNAKE_CASE   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

        if os.path.abspath(self.vocab_file     ) != os.path.abspath(_SCREAMING_SNAKE_CASE     ):
            copyfile(self.vocab_file   ,				_SCREAMING_SNAKE_CASE     )

        if os.path.abspath(self.merges_file     ) != os.path.abspath(_SCREAMING_SNAKE_CASE     ):
            copyfile(self.merges_file   ,				_SCREAMING_SNAKE_CASE     )

        return out_vocab_file, out_merge_file




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if isinstance(_SCREAMING_SNAKE_CASE   ,				_SCREAMING_SNAKE_CASE     ):
            try:
                with open(_SCREAMING_SNAKE_CASE   ,				'''r'''   ,				encoding='''utf-8'''     ) as fd:
                    self.add_from_file(_SCREAMING_SNAKE_CASE     )
            except FileNotFoundError as fnfe:
                raise fnfe
            except UnicodeError:
                raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''     )
            return

        SCREAMING_SNAKE_CASE__											=			f.readlines()
        for lineTmp in lines:
            SCREAMING_SNAKE_CASE__											=			lineTmp.strip()
            SCREAMING_SNAKE_CASE__											=			line.rfind(''' '''     )
            if idx == -1:
                raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''     )
            SCREAMING_SNAKE_CASE__											=			line[:idx]
            SCREAMING_SNAKE_CASE__											=			len(self.encoder     )





"""simple docstring"""






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:  str =       logging.get_logger(__name__)

__snake_case				:  Union[str, Any] =       {
    'microsoft/git-base': 'https://huggingface.co/microsoft/git-base/resolve/main/config.json',
}



class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'git_vision_model'

	def __init__(	self:						Optional[int]			,     _SCREAMING_SNAKE_CASE:						List[Any]=768			,     _SCREAMING_SNAKE_CASE:						Optional[Any]=3072			,     _SCREAMING_SNAKE_CASE:						str=12			,     _SCREAMING_SNAKE_CASE:						Any=12			,     _SCREAMING_SNAKE_CASE:						Dict=3			,     _SCREAMING_SNAKE_CASE:						List[Any]=224			,     _SCREAMING_SNAKE_CASE:						List[str]=16			,     _SCREAMING_SNAKE_CASE:						str="quick_gelu"			,     _SCREAMING_SNAKE_CASE:						Any=1e-5			,     _SCREAMING_SNAKE_CASE:						List[str]=0.0			,     _SCREAMING_SNAKE_CASE:						Dict=0.02			,     **_SCREAMING_SNAKE_CASE:						str			,     )							->			Optional[Any]:

				"""simple docstring"""

				super().__init__(**_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    Union[str, Any]				=			hidden_size
				__lowerCAmelCase :    List[str]				=			intermediate_size
				__lowerCAmelCase :    List[str]				=			num_hidden_layers
				__lowerCAmelCase :    Tuple				=			num_attention_heads
				__lowerCAmelCase :    List[Any]				=			num_channels
				__lowerCAmelCase :    Dict				=			patch_size
				__lowerCAmelCase :    Optional[int]				=			image_size
				__lowerCAmelCase :    Optional[Any]				=			initializer_range
				__lowerCAmelCase :    str				=			attention_dropout
				__lowerCAmelCase :    Optional[Any]				=			layer_norm_eps
				__lowerCAmelCase :    Tuple				=			hidden_act







	@classmethod
	def 				_SCREAMING_SNAKE_CASE  (	cls:						Dict			,     _SCREAMING_SNAKE_CASE:						Union[str, os.PathLike]			,     **_SCREAMING_SNAKE_CASE:						Dict)							->			"PretrainedConfig":

				"""simple docstring"""

				cls._set_token_in_kwargs(_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase		,      __lowerCAmelCase :    List[Any]				=			cls.get_config_dict(_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				# get the vision config dict if we are loading from GITConfig
				if config_dict.get("model_type") == "git":
							__lowerCAmelCase :    str				=			config_dict["vision_config"]

				if "model_type" in config_dict and hasattr(cls			,     "model_type") and config_dict["model_type"] != cls.model_type:
							logger.warning(
							    F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
							    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors.""")

				return cls.from_dict(_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)






class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'git'

	def __init__(	self:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						Tuple=None			,     _SCREAMING_SNAKE_CASE:						Dict=3_0522			,     _SCREAMING_SNAKE_CASE:						Any=768			,     _SCREAMING_SNAKE_CASE:						str=6			,     _SCREAMING_SNAKE_CASE:						int=12			,     _SCREAMING_SNAKE_CASE:						Dict=3072			,     _SCREAMING_SNAKE_CASE:						Optional[Any]="gelu"			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=0.1			,     _SCREAMING_SNAKE_CASE:						Any=0.1			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=1024			,     _SCREAMING_SNAKE_CASE:						str=0.02			,     _SCREAMING_SNAKE_CASE:						Tuple=1e-12			,     _SCREAMING_SNAKE_CASE:						List[str]=0			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]="absolute"			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=True			,     _SCREAMING_SNAKE_CASE:						Tuple=False			,     _SCREAMING_SNAKE_CASE:						Dict=101			,     _SCREAMING_SNAKE_CASE:						Tuple=102			,     _SCREAMING_SNAKE_CASE:						Any=None			,     **_SCREAMING_SNAKE_CASE:						List[Any]			,     )							->			List[Any]:

				"""simple docstring"""

				super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE			,     eos_token_id=_SCREAMING_SNAKE_CASE			,     pad_token_id=_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				if vision_config is None:
							__lowerCAmelCase :    int				=			{}
							logger.info("vision_config is None. initializing the GitVisionConfig with default values.")

				__lowerCAmelCase :    List[str]				=			GitVisionConfig(**_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    List[Any]				=			vocab_size
				__lowerCAmelCase :    int				=			hidden_size
				__lowerCAmelCase :    Dict				=			num_hidden_layers
				__lowerCAmelCase :    Tuple				=			num_attention_heads
				__lowerCAmelCase :    Union[str, Any]				=			hidden_act
				__lowerCAmelCase :    Dict				=			intermediate_size
				__lowerCAmelCase :    List[str]				=			hidden_dropout_prob
				__lowerCAmelCase :    Union[str, Any]				=			attention_probs_dropout_prob
				__lowerCAmelCase :    Union[str, Any]				=			max_position_embeddings
				__lowerCAmelCase :    Optional[int]				=			initializer_range
				__lowerCAmelCase :    str				=			layer_norm_eps
				__lowerCAmelCase :    Optional[Any]				=			position_embedding_type
				__lowerCAmelCase :    str				=			use_cache
				__lowerCAmelCase :    str				=			tie_word_embeddings
				__lowerCAmelCase :    Optional[int]				=			num_image_with_embedding

				__lowerCAmelCase :    Optional[Any]				=			bos_token_id
				__lowerCAmelCase :    Any				=			eos_token_id







	def 				_SCREAMING_SNAKE_CASE  (	self:						Dict)							->			List[str]:

				"""simple docstring"""

				__lowerCAmelCase :    Optional[int]				=			copy.deepcopy(self.__dict__)
				__lowerCAmelCase :    List[Any]				=			self.vision_config.to_dict()
				__lowerCAmelCase :    Tuple				=			self.__class__.model_type
				return output