



'''simple docstring'''




import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Any    					=	'''M-CLIP'''

			def __init__(							self  :				Dict							,  lowerCAmelCase__  :				Optional[int]=1_0_2_4							,  lowerCAmelCase__  :				List[str]=7_6_8							,  **lowerCAmelCase__  :				Tuple		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	transformerDimSize
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	imageDimSize
					super().__init__(**lowerCAmelCase__		)






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[int]    					=	MCLIPConfig

			def __init__(							self  :				Union[str, Any]							,  lowerCAmelCase__  :				List[str]							,  *lowerCAmelCase__  :				Any							,  **lowerCAmelCase__  :				Any		):





					"""simple docstring"""






					super().__init__(lowerCAmelCase__							,  *lowerCAmelCase__							,  **lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	XLMRobertaModel(lowerCAmelCase__		)
					__SCREAMING_SNAKE_CASE :	str					 =	torch.nn.Linear(
					    in_features=config.transformerDimensions							,  out_features=config.numDims		)



			def UpperCamelCase__      (							self  :				List[Any]							,  lowerCAmelCase__  :				Tuple							,  lowerCAmelCase__  :				Optional[Any]		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Any					 =	self.transformer(input_ids=lowerCAmelCase__							,  attention_mask=lowerCAmelCase__		)[0]
					__SCREAMING_SNAKE_CASE :	Tuple					 =	(embs * attention_mask.unsqueeze(2		)).sum(dim=1		) / attention_mask.sum(dim=1		)[:, None]
					return self.LinearTransformation(lowerCAmelCase__		), embs




'''simple docstring'''




# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCamelCase__ :			Tuple												=				'''3'''

print('''Python version:''', sys.version)
print('''transformers version:''', transformers.__version__)

try:
						import torch

						print('''Torch version:''', torch.__version__)
						print('''Cuda available:''', torch.cuda.is_available())
						print('''Cuda version:''', torch.version.cuda)
						print('''CuDNN version:''', torch.backends.cudnn.version())
						print('''Number of GPUs available:''', torch.cuda.device_count())
						print('''NCCL version:''', torch.cuda.nccl.version())
except ImportError:
						print('''Torch version:''', None)

try:
						import deepspeed

						print('''DeepSpeed version:''', deepspeed.__version__)
except ImportError:
						print('''DeepSpeed version:''', None)

try:
						import tensorflow as tf

						print('''TensorFlow version:''', tf.__version__)
						print('''TF GPUs available:''', bool(tf.config.list_physical_devices('''GPU''')))
						print('''Number of TF GPUs available:''', len(tf.config.list_physical_devices('''GPU''')))
except ImportError:
						print('''TensorFlow version:''', None)