






from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


SCREAMING_SNAKE_CASE       							=  logging.get_logger(__name__)
class 							A_			(  nn.Module	):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:	Optional[Any]       =    4_2
 _SCREAMING_SNAKE_CASE							:	Union[str, Any]       =    jnp.floataa


 def        snake_case__ (		self)       ->						Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase :					str	     = self.config.text_config
     _UpperCAmelCase :					Optional[int]	     = self.config.vision_config

     _UpperCAmelCase :					Optional[Any]	     = self.config.projection_dim
     _UpperCAmelCase :					Union[str, Any]	     = text_config.hidden_size
     _UpperCAmelCase :					str	     = vision_config.hidden_size

     _UpperCAmelCase :					Dict	     = FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
     _UpperCAmelCase :					Union[str, Any]	     = FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__							,     _A).module_class

     _UpperCAmelCase :					List[str]	     = text_module(_A							,     dtype=self.dtype)
     _UpperCAmelCase :					Optional[int]	     = vision_module(_A							,     dtype=self.dtype)

     _UpperCAmelCase :					List[Any]	     = nn.Dense(
         self.projection_dim							,     dtype=self.dtype							,     kernel_init=jax.nn.initializers.normal(0.02)							,     use_bias=_A							,     )
     _UpperCAmelCase :					Union[str, Any]	     = nn.Dense(
         self.projection_dim							,     dtype=self.dtype							,     kernel_init=jax.nn.initializers.normal(0.02)							,     use_bias=_A							,     )
     _UpperCAmelCase :					Any	     = self.param('''logit_scale'''							,     jax.nn.initializers.ones							,     [])


 def __call__(		self							,     _A=None							,     _A=None							,     _A=None							,     _A=None							,     _A=None							,     _A = True							,     _A=None							,     _A=None							,     _A=None							,     )       ->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase :					str	     = return_dict if return_dict is not None else self.config.return_dict

     _UpperCAmelCase :					Optional[int]	     = self.vision_model(
         pixel_values=_A							,     deterministic=_A							,     output_attentions=_A							,     output_hidden_states=_A							,     return_dict=_A							,     )

     _UpperCAmelCase :					List[Any]	     = self.text_model(
         input_ids=_A							,     attention_mask=_A							,     token_type_ids=_A							,     position_ids=_A							,     deterministic=_A							,     output_attentions=_A							,     output_hidden_states=_A							,     return_dict=_A							,     )

     _UpperCAmelCase :					Optional[Any]	     = vision_outputs[1]
     _UpperCAmelCase :					Any	     = self.visual_projection(_A)

     _UpperCAmelCase :					List[str]	     = text_outputs[1]
     _UpperCAmelCase :					Dict	     = self.text_projection(_A)

     # normalized features
     _UpperCAmelCase :					Tuple	     = image_embeds / jnp.linalg.norm(_A							,     axis=-1							,     keepdims=_A)
     _UpperCAmelCase :					int	     = text_embeds / jnp.linalg.norm(_A							,     axis=-1							,     keepdims=_A)

     # cosine similarity as logits
     _UpperCAmelCase :					Optional[int]	     = jnp.exp(self.logit_scale)
     _UpperCAmelCase :					Tuple	     = jnp.matmul(_A							,     image_embeds.T) * logit_scale
     _UpperCAmelCase :					Dict	     = logits_per_text.T

     if not return_dict:
         return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

     return FlaxCLIPOutput(
         logits_per_image=_A							,     logits_per_text=_A							,     text_embeds=_A							,     image_embeds=_A							,     text_model_output=_A							,     vision_model_output=_A							,     )



class 							A_			(  a_	):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:	Optional[int]       =    HybridCLIPConfig
 _SCREAMING_SNAKE_CASE							:	str       =    FlaxHybridCLIPModule


 def __init__(		self							,     _A							,     _A = None							,     _A = 0							,     _A = jnp.floataa							,     **_A							,     )       ->						Optional[Any]:

     """simple docstring"""



     if input_shape is None:
         _UpperCAmelCase :					Dict	     = ((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

     _UpperCAmelCase :					List[Any]	     = self.module_class(config=_A							,     dtype=_A							,     **_A)
     super().__init__(_A							,     _A							,     input_shape=_A							,     seed=_A							,     dtype=_A)


 def        snake_case__ (		self							,     _A							,     _A							,     _A = None)       ->						List[Any]:

     """simple docstring"""



     _UpperCAmelCase :					Optional[Any]	     = jnp.zeros(input_shape[0]							,     dtype='''i4''')
     _UpperCAmelCase :					Any	     = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_A).shape[-1])							,     input_shape[0])
     _UpperCAmelCase :					str	     = jnp.ones_like(_A)
     _UpperCAmelCase :					List[Any]	     = jnp.ones_like(_A)

     _UpperCAmelCase :					Optional[Any]	     = jax.random.normal(_A							,     input_shape[1])

     _UpperCAmelCase :					Optional[Any]	     = jax.random.split(_A)
     _UpperCAmelCase :					List[str]	     = {'params': params_rng, 'dropout': dropout_rng}

     return self.module.init(_A							,     _A							,     _A							,     _A							,     _A							,     _A)["params"]


 def __call__(		self							,     _A							,     _A							,     _A=None							,     _A=None							,     _A=None							,     _A = None							,     _A = None							,     _A = False							,     _A = None							,     _A = None							,     _A = None							,     )       ->						Any:

     """simple docstring"""



     _UpperCAmelCase :					int	     = output_attentions if output_attentions is not None else self.config.output_attentions
     _UpperCAmelCase :					Union[str, Any]	     = (
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     _UpperCAmelCase :					Optional[Any]	     = return_dict if return_dict is not None else self.config.return_dict

     if position_ids is None:
         _UpperCAmelCase :					str	     = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_A).shape[-1])							,     input_ids.shape)

     if token_type_ids is None:
         _UpperCAmelCase :					Optional[Any]	     = jnp.zeros_like(_A)

     if attention_mask is None:
         _UpperCAmelCase :					Tuple	     = jnp.ones_like(_A)

     # Handle any PRNG if needed
     _UpperCAmelCase :					List[Any]	     = {}
     if dropout_rng is not None:
         _UpperCAmelCase :					int	     = dropout_rng

     return self.module.apply(
         {'''params''': params or self.params}							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype=jnp.floataa)							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype='''i4''')							,     not train							,     _A							,     _A							,     _A							,     rngs=_A							,     )


 def        snake_case__ (		self							,     _A							,     _A=None							,     _A=None							,     _A=None							,     _A = None							,     _A = None							,     _A=False							,     )       ->						List[str]:

     """simple docstring"""



     if position_ids is None:
         _UpperCAmelCase :					Any	     = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(_A).shape[-1])							,     input_ids.shape)

     if token_type_ids is None:
         _UpperCAmelCase :					Optional[Any]	     = jnp.zeros_like(_A)

     if attention_mask is None:
         _UpperCAmelCase :					Optional[int]	     = jnp.ones_like(_A)

     # Handle any PRNG if needed
     _UpperCAmelCase :					Union[str, Any]	     = {}
     if dropout_rng is not None:
         _UpperCAmelCase :					List[str]	     = dropout_rng

     def _get_features(_A							,     _A							,     _A							,     _A							,     _A							,     _A):
         _UpperCAmelCase :					str	     = module.text_model(
             input_ids=_A							,     attention_mask=_A							,     position_ids=_A							,     token_type_ids=_A							,     deterministic=_A							,     )
         _UpperCAmelCase :					Any	     = text_outputs[1]
         _UpperCAmelCase :					Optional[int]	     = module.text_projection(_A)
         return text_features

     return self.module.apply(
         {'''params''': params or self.params}							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype='''i4''')							,     jnp.array(_A							,     dtype='''i4''')							,     not train							,     method=_get_features							,     rngs=_A							,     )


 def        snake_case__ (		self							,     _A							,     _A = None							,     _A = None							,     _A=False)       ->						Tuple:

     """simple docstring"""



     _UpperCAmelCase :					List[str]	     = {}
     if dropout_rng is not None:
         _UpperCAmelCase :					List[str]	     = dropout_rng

     def _get_features(_A							,     _A							,     _A):
         _UpperCAmelCase :					int	     = module.vision_model(pixel_values=_A							,     deterministic=_A)
         _UpperCAmelCase :					Union[str, Any]	     = vision_outputs[1]  # pooled_output
         _UpperCAmelCase :					List[str]	     = module.visual_projection(_A)
         return image_features

     return self.module.apply(
         {'''params''': params or self.params}							,     jnp.array(_A							,     dtype=jnp.floataa)							,     not train							,     method=_get_features							,     rngs=_A							,     )


 @classmethod
 def        snake_case__ (		cls							,     _A = None							,     _A = None							,     *_A							,     **_A							,     )       ->						Dict:

     """simple docstring"""



     _UpperCAmelCase :					Tuple	     = {
         argument[len('''text_''') :]: value for argument, value in kwargs.items() if argument.startswith('''text_''')
     }

     _UpperCAmelCase :					int	     = {
         argument[len('''vision_''') :]: value for argument, value in kwargs.items() if argument.startswith('''vision_''')
     }

     # remove text, vision kwargs from kwargs
     for key in kwargs_text.keys():
         del kwargs["text_" + key]
     for key in kwargs_vision.keys():
         del kwargs["vision_" + key]

     # Load and initialize the text and vision model
     _UpperCAmelCase :					str	     = kwargs_text.pop('''model'''							,     _A)
     if text_model is None:
         assert (
             text_model_name_or_path is not None
         ), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
         from transformers import FlaxAutoModel

         if "config" not in kwargs_text:
             from transformers import AutoConfig

             _UpperCAmelCase :					Tuple	     = AutoConfig.from_pretrained(_A)
             _UpperCAmelCase :					Tuple	     = text_config

         _UpperCAmelCase :					Tuple	     = FlaxAutoModel.from_pretrained(_A							,     *_A							,     **_A)

     _UpperCAmelCase :					str	     = kwargs_vision.pop('''model'''							,     _A)
     if vision_model is None:
         assert (
             vision_model_name_or_path is not None
         ), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
         from transformers import FlaxAutoModel

         if "config" not in kwargs_vision:
             from transformers import AutoConfig

             _UpperCAmelCase :					List[Any]	     = AutoConfig.from_pretrained(_A)
             _UpperCAmelCase :					str	     = vision_config

         _UpperCAmelCase :					Tuple	     = FlaxAutoModel.from_pretrained(_A							,     *_A							,     **_A)

     # instantiate config with corresponding kwargs
     _UpperCAmelCase :					Dict	     = kwargs.pop('''dtype'''							,     jnp.floataa)
     _UpperCAmelCase :					List[Any]	     = HybridCLIPConfig.from_text_vision_configs(text_model.config							,     vision_model.config							,     **_A)

     # init model
     _UpperCAmelCase :					str	     = cls(_A							,     *_A							,     dtype=_A							,     **_A)

     if vision_config.model_type == "clip":
         _UpperCAmelCase :					List[Any]	     = vision_model.params['vision_model']
         _UpperCAmelCase :					List[Any]	     = vision_model.params['visual_projection']['kernel']
     else:
         _UpperCAmelCase :					List[Any]	     = vision_model.params

     _UpperCAmelCase :					List[Any]	     = text_model.params

     return model








'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
   A      :     Optional[int]										= None


A      :     Optional[Any]										= logging.get_logger(__name__)

A      :     Dict										= {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

A      :     Optional[int]										= {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'''
        ),
    },
}

A      :     List[str]										= {
    '''xlm-roberta-base''': 512,
    '''xlm-roberta-large''': 512,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 512,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 512,
    '''xlm-roberta-large-finetuned-conll03-english''': 512,
    '''xlm-roberta-large-finetuned-conll03-german''': 512,
}


class  __lowerCamelCase (       a_       ):
    """simple docstring"""
    a													=    VOCAB_FILES_NAMES
    a													=    PRETRAINED_VOCAB_FILES_MAP
    a													=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a													=    ["input_ids", "attention_mask"]
    a													=    XLMRobertaTokenizer


    def __init__(  self			:							Union[str, Any]      ,							SCREAMING_SNAKE_CASE			:							Dict=None      ,							SCREAMING_SNAKE_CASE			:							List[Any]=None      ,							SCREAMING_SNAKE_CASE			:							Any="<s>"      ,							SCREAMING_SNAKE_CASE			:							Tuple="</s>"      ,							SCREAMING_SNAKE_CASE			:							str="</s>"      ,							SCREAMING_SNAKE_CASE			:							Optional[Any]="<s>"      ,							SCREAMING_SNAKE_CASE			:							Any="<unk>"      ,							SCREAMING_SNAKE_CASE			:							Union[str, Any]="<pad>"      ,							SCREAMING_SNAKE_CASE			:							List[Any]="<mask>"      ,							**SCREAMING_SNAKE_CASE			:							int      ,							):
           # Mask token behave like a normal word, i.e. include the space before it
           _A     :	str      =					AddedToken(SCREAMING_SNAKE_CASE      ,							lstrip=SCREAMING_SNAKE_CASE      ,							rstrip=SCREAMING_SNAKE_CASE) if isinstance(SCREAMING_SNAKE_CASE      ,							SCREAMING_SNAKE_CASE) else mask_token

           super().__init__(
               SCREAMING_SNAKE_CASE      ,							tokenizer_file=SCREAMING_SNAKE_CASE      ,							bos_token=SCREAMING_SNAKE_CASE      ,							eos_token=SCREAMING_SNAKE_CASE      ,							sep_token=SCREAMING_SNAKE_CASE      ,							cls_token=SCREAMING_SNAKE_CASE      ,							unk_token=SCREAMING_SNAKE_CASE      ,							pad_token=SCREAMING_SNAKE_CASE      ,							mask_token=SCREAMING_SNAKE_CASE      ,							**SCREAMING_SNAKE_CASE      ,							)

           _A     :	Dict      =					vocab_file
           _A     :	Optional[Any]      =					False if not self.vocab_file else True


    def    A    (  self			:							int      ,							SCREAMING_SNAKE_CASE			:							List[int]      ,							SCREAMING_SNAKE_CASE			:							Optional[List[int]] = None):

           if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           _A     :	Optional[Any]      =					[self.cls_token_id]
           _A     :	Optional[int]      =					[self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep


    def    A    (  self			:							List[str]      ,							SCREAMING_SNAKE_CASE			:							List[int]      ,							SCREAMING_SNAKE_CASE			:							Optional[List[int]] = None):
           _A     :	Any      =					[self.sep_token_id]
           _A     :	Optional[Any]      =					[self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


    def    A    (  self			:							Dict      ,							SCREAMING_SNAKE_CASE			:							str      ,							SCREAMING_SNAKE_CASE			:							Optional[str] = None):
           if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                      'tokenizer.')

           if not os.path.isdir(SCREAMING_SNAKE_CASE):
                  logger.error(F'Vocabulary path ({save_directory}) should be a directory.')
                  return
           _A     :	Any      =					os.path.join(
               SCREAMING_SNAKE_CASE      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

           if os.path.abspath(self.vocab_file) != os.path.abspath(SCREAMING_SNAKE_CASE):
                  copyfile(self.vocab_file      ,							SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)






