


import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map






def 					UpperCamelCase   (      __magic_name__	:				Union[dict, list, tuple, torch.Tensor]	)  ->    List[Tuple[int, ...]]:
		"""simple docstring"""




		lowercase__	     =						[]
		if isinstance(__magic_name__       ,       __magic_name__	):
				for v in tree.values():
						shapes.extend(_fetch_dims(__magic_name__	)	)
		elif isinstance(__magic_name__       ,       (list, tuple)	):
				for t in tree:
						shapes.extend(_fetch_dims(__magic_name__	)	)
		elif isinstance(__magic_name__       ,       torch.Tensor	):
				shapes.append(tree.shape	)
		else:
				raise ValueError("""Not supported"""	)

		return shapes






@torch.jit.ignore
def 					UpperCamelCase   (      __magic_name__	:				int       ,       __magic_name__	:				Tuple[int, ...]	)  ->    Tuple[int, ...]:
		"""simple docstring"""




		lowercase__	     =						[]
		for d in reversed(__magic_name__	):
				idx.append(flat_idx % d	)
				lowercase__	     =						flat_idx // d

		return tuple(reversed(__magic_name__	)	)






@torch.jit.ignore
def 					UpperCamelCase   (      __magic_name__	:				Sequence[int]       ,       __magic_name__	:				Sequence[int]       ,       __magic_name__	:				Sequence[int]       ,       __magic_name__	:				Optional[Sequence[bool]] = None       ,       __magic_name__	:				Optional[Sequence[bool]] = None       ,       )  ->    List[Tuple[slice, ...]]:
		"""simple docstring"""




		def reduce_edge_list(__magic_name__	:				List[bool]	) -> None:
				lowercase__	     =						True
				for i in range(len(__magic_name__	)	):
						lowercase__	     =						-1 * (i + 1)
						l[reversed_idx] &= tally
						lowercase__	     =						l[reversed_idx]

		if start_edges is None:
				lowercase__	     =						[s == 0 for s in start]
				reduce_edge_list(__magic_name__	)
		if end_edges is None:
				lowercase__	     =						[e == (d - 1) for e, d in zip(__magic_name__       ,       __magic_name__	)]
				reduce_edge_list(__magic_name__	)

		# Base cases. Either start/end are empty and we're done, or the final,
		# one-dimensional tensor can be simply sliced
		if len(__magic_name__	) == 0:
				return [()]
		elif len(__magic_name__	) == 1:
				return [(slice(start[0]       ,       end[0] + 1	),)]

		lowercase__	     =						[]
		lowercase__	     =						[]

		# Dimensions common to start and end can be selected directly
		for s, e in zip(__magic_name__       ,       __magic_name__	):
				if s == e:
						path_list.append(slice(__magic_name__       ,       s + 1	)	)
				else:
						break

		lowercase__	     =						tuple(__magic_name__	)
		lowercase__	     =						len(__magic_name__	)

		# start == end, and we're done
		if divergence_idx == len(__magic_name__	):
				return [path]

		def upper() -> Tuple[Tuple[slice, ...], ...]:
				assert start_edges is not None
				assert end_edges is not None

				lowercase__	     =						start[divergence_idx]
				return tuple(
				    path + (slice(__magic_name__       ,       sdi + 1	),) + s
				    for s in _get_minimal_slice_set(
				        start[divergence_idx + 1 :]       ,       [d - 1 for d in dims[divergence_idx + 1 :]]       ,       dims[divergence_idx + 1 :]       ,       start_edges=start_edges[divergence_idx + 1 :]       ,       end_edges=[True for _ in end_edges[divergence_idx + 1 :]]       ,       )	)

		def lower() -> Tuple[Tuple[slice, ...], ...]:
				assert start_edges is not None
				assert end_edges is not None

				lowercase__	     =						end[divergence_idx]
				return tuple(
				    path + (slice(__magic_name__       ,       edi + 1	),) + s
				    for s in _get_minimal_slice_set(
				        [0 for _ in start[divergence_idx + 1 :]]       ,       end[divergence_idx + 1 :]       ,       dims[divergence_idx + 1 :]       ,       start_edges=[True for _ in start_edges[divergence_idx + 1 :]]       ,       end_edges=end_edges[divergence_idx + 1 :]       ,       )	)

		# If both start and end are at the edges of the subtree rooted at
		# divergence_idx, we can just select the whole subtree at once
		if start_edges[divergence_idx] and end_edges[divergence_idx]:
				slices.append(path + (slice(start[divergence_idx]       ,       end[divergence_idx] + 1	),)	)
		# If just start is at the edge, we can grab almost all of the subtree,
		# treating only the ragged bottom edge as an edge case
		elif start_edges[divergence_idx]:
				slices.append(path + (slice(start[divergence_idx]       ,       end[divergence_idx]	),)	)
				slices.extend(lower()	)
		# Analogous to the previous case, but the top is ragged this time
		elif end_edges[divergence_idx]:
				slices.extend(upper()	)
				slices.append(path + (slice(start[divergence_idx] + 1       ,       end[divergence_idx] + 1	),)	)
		# If both sides of the range are ragged, we need to handle both sides
		# separately. If there's contiguous meat in between them, we can index it
		# in one big chunk
		else:
				slices.extend(upper()	)
				lowercase__	     =						end[divergence_idx] - start[divergence_idx]
				if middle_ground > 1:
						slices.append(path + (slice(start[divergence_idx] + 1       ,       end[divergence_idx]	),)	)
				slices.extend(lower()	)

		return slices






@torch.jit.ignore
def 					UpperCamelCase   (      __magic_name__	:				torch.Tensor       ,       __magic_name__	:				int       ,       __magic_name__	:				int       ,       __magic_name__	:				int	)  ->    torch.Tensor:
		"""simple docstring"""




		lowercase__	     =						t.shape[:no_batch_dims]
		lowercase__	     =						list(_flat_idx_to_idx(__magic_name__       ,       __magic_name__	)	)
		# _get_minimal_slice_set is inclusive
		lowercase__	     =						list(_flat_idx_to_idx(flat_end - 1       ,       __magic_name__	)	)

		# Get an ordered list of slices to perform
		lowercase__	     =						_get_minimal_slice_set(
		    __magic_name__       ,       __magic_name__       ,       __magic_name__       ,       )

		lowercase__	     =						[t[s] for s in slices]

		return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]	) for s in sliced_tensors]	)






def 					UpperCamelCase   (      __magic_name__	:				Callable       ,       __magic_name__	:				Dict[str, Any]       ,       __magic_name__	:				int       ,       __magic_name__	:				int       ,       __magic_name__	:				bool = False       ,       __magic_name__	:				Any = None       ,       __magic_name__	:				bool = False       ,       )  ->    Any:
		"""simple docstring"""




		if not (len(__magic_name__	) > 0):
				raise ValueError("""Must provide at least one input"""	)

		lowercase__	     =						[shape[:no_batch_dims] for shape in _fetch_dims(__magic_name__	)]
		lowercase__	     =						tuple([max(__magic_name__	) for s in zip(*__magic_name__	)]	)

		def _prep_inputs(__magic_name__	:				torch.Tensor	) -> torch.Tensor:
				if not low_mem:
						if not sum(t.shape[:no_batch_dims]	) == no_batch_dims:
								lowercase__	     =						t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
						lowercase__	     =						t.reshape(-1       ,       *t.shape[no_batch_dims:]	)
				else:
						lowercase__	     =						t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
				return t

		lowercase__	     =						tensor_tree_map(_prep_inputs       ,       __magic_name__	)
		lowercase__	     =						None
		if _out is not None:
				lowercase__	     =						tensor_tree_map(lambda __magic_name__	: t.view([-1] + list(t.shape[no_batch_dims:]	)	)       ,       _out	)

		lowercase__	     =						1
		for d in orig_batch_dims:
				flat_batch_dim *= d

		lowercase__	     =						flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

		def _select_chunk(__magic_name__	:				torch.Tensor	) -> torch.Tensor:
				return t[i : i + chunk_size] if t.shape[0] != 1 else t

		lowercase__	     =						0
		lowercase__	     =						prepped_outputs
		for _ in range(__magic_name__	):
				# Chunk the input
				if not low_mem:
						lowercase__	     =						_select_chunk
				else:
						lowercase__	     =						partial(
						    _chunk_slice       ,       flat_start=__magic_name__       ,       flat_end=min(__magic_name__       ,       i + chunk_size	)       ,       no_batch_dims=len(__magic_name__	)       ,       )

				lowercase__	     =						tensor_tree_map(__magic_name__       ,       __magic_name__	)

				# Run the layer on the chunk
				lowercase__	     =						layer(**__magic_name__	)

				# Allocate space for the output
				if out is None:
						lowercase__	     =						tensor_tree_map(lambda __magic_name__	: t.new_zeros((flat_batch_dim,) + t.shape[1:]	)       ,       __magic_name__	)

				# Put the chunk in its pre-allocated space
				if isinstance(__magic_name__       ,       __magic_name__	):

						def assign(__magic_name__	:				dict       ,       __magic_name__	:				dict	) -> None:
								for k, v in da.items():
										if isinstance(__magic_name__       ,       __magic_name__	):
												assign(__magic_name__       ,       da[k]	)
										else:
												if _add_into_out:
														v[i : i + chunk_size] += da[k]
												else:
														lowercase__	     =						da[k]

						assign(__magic_name__       ,       __magic_name__	)
				elif isinstance(__magic_name__       ,       __magic_name__	):
						for xa, xa in zip(__magic_name__       ,       __magic_name__	):
								if _add_into_out:
										xa[i : i + chunk_size] += xa
								else:
										lowercase__	     =						xa
				elif isinstance(__magic_name__       ,       torch.Tensor	):
						if _add_into_out:
								out[i : i + chunk_size] += output_chunk
						else:
								lowercase__	     =						output_chunk
				else:
						raise ValueError("""Not supported"""	)

				i += chunk_size

		lowercase__	     =						tensor_tree_map(lambda __magic_name__	: t.view(orig_batch_dims + t.shape[1:]	)       ,       __magic_name__	)

		return out







class 				A       :
					'''simple docstring'''






					def __init__(self       :		List[str] ,   _UpperCAmelCase       :		int = 512 ,   )		->  str:




							"""simple docstring"""


							lowercase__	     =						max_chunk_size
							lowercase__	     =						None
							lowercase__	     =						None

					def     lowerCamelCase__	(self       :		List[Any] ,   _UpperCAmelCase       :		Callable ,   _UpperCAmelCase       :		tuple ,   _UpperCAmelCase       :		int						)		->  int:




							"""simple docstring"""


							logging.info("""Tuning chunk size..."""						)

							if min_chunk_size >= self.max_chunk_size:
									return min_chunk_size

							lowercase__	     =						[2**l for l in range(int(math.log(self.max_chunk_size ,   2						)						) + 1						)]
							lowercase__	     =						[c for c in candidates if c > min_chunk_size]
							lowercase__	     =						[min_chunk_size] + candidates
							candidates[-1] += 4

							def test_chunk_size(_UpperCAmelCase       :		int						) -> bool:
									try:
											with torch.no_grad():
													fn(*_UpperCAmelCase ,   chunk_size=_UpperCAmelCase						)
											return True
									except RuntimeError:
											return False

							lowercase__	     =						0
							lowercase__	     =						len(_UpperCAmelCase						) - 1
							while i > min_viable_chunk_size_index:
									lowercase__	     =						test_chunk_size(candidates[i]						)
									if not viable:
											lowercase__	     =						(min_viable_chunk_size_index + i) // 2
									else:
											lowercase__	     =						i
											lowercase__	     =						(i + len(_UpperCAmelCase						) - 1) // 2

							return candidates[min_viable_chunk_size_index]

					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		Iterable ,   _UpperCAmelCase       :		Iterable						)		->  bool:




							"""simple docstring"""


							lowercase__	     =						True
							for aa, aa in zip(_UpperCAmelCase ,   _UpperCAmelCase						):
									assert type(_UpperCAmelCase						) == type(_UpperCAmelCase						)
									if isinstance(_UpperCAmelCase ,   (list, tuple)						):
											consistent &= self._compare_arg_caches(_UpperCAmelCase ,   _UpperCAmelCase						)
									elif isinstance(_UpperCAmelCase ,   _UpperCAmelCase						):
											lowercase__	     =						[v for _, v in sorted(aa.items() ,   key=lambda _UpperCAmelCase						: x[0]						)]
											lowercase__	     =						[v for _, v in sorted(aa.items() ,   key=lambda _UpperCAmelCase						: x[0]						)]
											consistent &= self._compare_arg_caches(_UpperCAmelCase ,   _UpperCAmelCase						)
									else:
											consistent &= aa == aa

							return consistent


					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Callable ,   _UpperCAmelCase       :		tuple ,   _UpperCAmelCase       :		int ,   )		->  int:




							"""simple docstring"""


							lowercase__	     =						True
							lowercase__	     =						tree_map(lambda _UpperCAmelCase						: a.shape if isinstance(_UpperCAmelCase ,   torch.Tensor						) else a ,   _UpperCAmelCase ,   _UpperCAmelCase						)
							if self.cached_arg_data is not None:
									# If args have changed shape/value, we need to re-tune
									assert len(self.cached_arg_data						) == len(_UpperCAmelCase						)
									lowercase__	     =						self._compare_arg_caches(self.cached_arg_data ,   _UpperCAmelCase						)
							else:
									# Otherwise, we can reuse the precomputed value
									lowercase__	     =						False

							if not consistent:
									lowercase__	     =						self._determine_favorable_chunk_size(
									    _UpperCAmelCase ,   _UpperCAmelCase ,   _UpperCAmelCase ,   )
									lowercase__	     =						arg_data

							assert self.cached_chunk_size is not None

							return self.cached_chunk_size





import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    FlaubertForMultipleChoice,
	    FlaubertForQuestionAnswering,
	    FlaubertForQuestionAnsweringSimple,
	    FlaubertForSequenceClassification,
	    FlaubertForTokenClassification,
	    FlaubertModel,
	    FlaubertWithLMHeadModel,
	)
	from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Any=13 ,   _UpperCAmelCase       :		str=7 ,   _UpperCAmelCase       :		str=True ,   _UpperCAmelCase       :		List[Any]=True ,   _UpperCAmelCase       :		Optional[int]=True ,   _UpperCAmelCase       :		List[Any]=True ,   _UpperCAmelCase       :		Tuple=True ,   _UpperCAmelCase       :		Optional[int]=False ,   _UpperCAmelCase       :		Any=False ,   _UpperCAmelCase       :		List[Any]=False ,   _UpperCAmelCase       :		Tuple=2 ,   _UpperCAmelCase       :		Optional[int]=99 ,   _UpperCAmelCase       :		Any=0 ,   _UpperCAmelCase       :		Dict=32 ,   _UpperCAmelCase       :		Tuple=5 ,   _UpperCAmelCase       :		Union[str, Any]=4 ,   _UpperCAmelCase       :		Union[str, Any]=0.1 ,   _UpperCAmelCase       :		Optional[Any]=0.1 ,   _UpperCAmelCase       :		List[str]=512 ,   _UpperCAmelCase       :		Tuple=12 ,   _UpperCAmelCase       :		int=2 ,   _UpperCAmelCase       :		Tuple=0.02 ,   _UpperCAmelCase       :		Union[str, Any]=3 ,   _UpperCAmelCase       :		Optional[Any]=4 ,   _UpperCAmelCase       :		Dict="last" ,   _UpperCAmelCase       :		Any=None ,   _UpperCAmelCase       :		int=None ,   )		->  int:




							"""simple docstring"""


							lowercase__	     =						parent
							lowercase__	     =						batch_size
							lowercase__	     =						seq_length
							lowercase__	     =						is_training
							lowercase__	     =						use_input_lengths
							lowercase__	     =						use_token_type_ids
							lowercase__	     =						use_labels
							lowercase__	     =						gelu_activation
							lowercase__	     =						sinusoidal_embeddings
							lowercase__	     =						causal
							lowercase__	     =						asm
							lowercase__	     =						n_langs
							lowercase__	     =						vocab_size
							lowercase__	     =						n_special
							lowercase__	     =						hidden_size
							lowercase__	     =						num_hidden_layers
							lowercase__	     =						num_attention_heads
							lowercase__	     =						hidden_dropout_prob
							lowercase__	     =						attention_probs_dropout_prob
							lowercase__	     =						max_position_embeddings
							lowercase__	     =						type_vocab_size
							lowercase__	     =						type_sequence_label_size
							lowercase__	     =						initializer_range
							lowercase__	     =						num_labels
							lowercase__	     =						num_choices
							lowercase__	     =						summary_type
							lowercase__	     =						use_proj
							lowercase__	     =						scope

					def     lowerCamelCase__	(self       :		List[str]						)		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size						)
							lowercase__	     =						random_attention_mask([self.batch_size, self.seq_length]						)

							lowercase__	     =						None
							if self.use_input_lengths:
									lowercase__	     =						(
									    ids_tensor([self.batch_size] ,   vocab_size=2						) + self.seq_length - 2
									)  # small variation of seq_length

							lowercase__	     =						None
							if self.use_token_type_ids:
									lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.n_langs						)

							lowercase__	     =						None
							lowercase__	     =						None
							lowercase__	     =						None
							if self.use_labels:
									lowercase__	     =						ids_tensor([self.batch_size] ,   self.type_sequence_label_size						)
									lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels						)
									lowercase__	     =						ids_tensor([self.batch_size] ,   2						).float()
									lowercase__	     =						ids_tensor([self.batch_size] ,   self.num_choices						)

							lowercase__	     =						self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_lengths,
							    sequence_labels,
							    token_labels,
							    is_impossible_labels,
							    choice_labels,
							    input_mask,
							)

					def     lowerCamelCase__	(self       :		int						)		->  Dict:




							"""simple docstring"""


							return FlaubertConfig(
							    vocab_size=self.vocab_size ,   n_special=self.n_special ,   emb_dim=self.hidden_size ,   n_layers=self.num_hidden_layers ,   n_heads=self.num_attention_heads ,   dropout=self.hidden_dropout_prob ,   attention_dropout=self.attention_probs_dropout_prob ,   gelu_activation=self.gelu_activation ,   sinusoidal_embeddings=self.sinusoidal_embeddings ,   asm=self.asm ,   causal=self.causal ,   n_langs=self.n_langs ,   max_position_embeddings=self.max_position_embeddings ,   initializer_range=self.initializer_range ,   summary_type=self.summary_type ,   use_proj=self.use_proj ,   )

					def     lowerCamelCase__	(self       :		Tuple ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Tuple ,   )		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertModel(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase ,   lengths=_UpperCAmelCase ,   langs=_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase ,   langs=_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase						)
							self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size)						)

					def     lowerCamelCase__	(self       :		Union[str, Any] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		List[str] ,   )		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						FlaubertWithLMHeadModel(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase ,   token_type_ids=_UpperCAmelCase ,   labels=_UpperCAmelCase						)
							self.parent.assertEqual(result.loss.shape ,   ()						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size)						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Union[str, Any] ,   )		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForQuestionAnsweringSimple(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)

							lowercase__	     =						model(_UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase						)
							self.parent.assertEqual(result.start_logits.shape ,   (self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape ,   (self.batch_size, self.seq_length)						)

					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Optional[int] ,   )		->  List[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForQuestionAnswering(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)

							lowercase__	     =						model(
							    _UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase ,   cls_index=_UpperCAmelCase ,   is_impossible=_UpperCAmelCase ,   p_mask=_UpperCAmelCase ,   )

							lowercase__	     =						model(
							    _UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase ,   cls_index=_UpperCAmelCase ,   is_impossible=_UpperCAmelCase ,   )

							((lowercase__)     ,      )	     =						result_with_labels.to_tuple()

							lowercase__	     =						model(_UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase						)

							((lowercase__)     ,      )	     =						result_with_labels.to_tuple()

							self.parent.assertEqual(result_with_labels.loss.shape ,   ()						)
							self.parent.assertEqual(result.start_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top)						)
							self.parent.assertEqual(result.start_top_index.shape ,   (self.batch_size, model.config.start_n_top)						)
							self.parent.assertEqual(
							    result.end_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
							self.parent.assertEqual(
							    result.end_top_index.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
							self.parent.assertEqual(result.cls_logits.shape ,   (self.batch_size,)						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		List[str] ,   )		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForSequenceClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase ,   labels=_UpperCAmelCase						)

							self.parent.assertEqual(result.loss.shape ,   ()						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)						)

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   )		->  str:




							"""simple docstring"""


							lowercase__	     =						self.num_labels
							lowercase__	     =						FlaubertForTokenClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   labels=_UpperCAmelCase						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels)						)

					def     lowerCamelCase__	(self       :		List[str] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   )		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						self.num_choices
							lowercase__	     =						FlaubertForMultipleChoice(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						input_ids.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						token_type_ids.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						input_mask.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						model(
							    _UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   token_type_ids=_UpperCAmelCase ,   labels=_UpperCAmelCase ,   )
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_choices)						)


					def     lowerCamelCase__	(self       :		Dict						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.prepare_config_and_inputs()
							(
							    (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      
							)	     =						config_and_inputs
							lowercase__	     =						{
							    """input_ids""": input_ids,
							    """token_type_ids""": token_type_ids,
							    """lengths""": input_lengths,
							    """attention_mask""": input_mask,
							}
							return config, inputs_dict






@require_torch
class 				A       (			UpperCAmelCase__					,    UpperCAmelCase__					,    unittest.TestCase				):
					'''simple docstring'''






					A__     =					(
					    (
					        FlaubertModel,
					        FlaubertWithLMHeadModel,
					        FlaubertForQuestionAnswering,
					        FlaubertForQuestionAnsweringSimple,
					        FlaubertForSequenceClassification,
					        FlaubertForTokenClassification,
					        FlaubertForMultipleChoice,
					    )
					    if is_torch_available()
					    else ()
					)
					A__     =					(
					    {
					        '''feature-extraction''': FlaubertModel,
					        '''fill-mask''': FlaubertWithLMHeadModel,
					        '''question-answering''': FlaubertForQuestionAnsweringSimple,
					        '''text-classification''': FlaubertForSequenceClassification,
					        '''token-classification''': FlaubertForTokenClassification,
					        '''zero-shot''': FlaubertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		int						)		->  str:




							"""simple docstring"""


							if (
							    pipeline_test_casse_name == "QAPipelineTests"
							    and tokenizer_name is not None
							    and not tokenizer_name.endswith("""Fast"""						)
							):
									# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
									# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
									# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
									return True

							return False

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Union[str, Any]=False						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						super()._prepare_for_class(_UpperCAmelCase ,   _UpperCAmelCase ,   return_labels=_UpperCAmelCase						)

							if return_labels:
									if model_class.__name__ == "FlaubertForQuestionAnswering":
											lowercase__	     =						torch.zeros(
											    self.model_tester.batch_size ,   dtype=torch.long ,   device=_UpperCAmelCase						)
											lowercase__	     =						torch.zeros(
											    self.model_tester.batch_size ,   dtype=torch.long ,   device=_UpperCAmelCase						)

							return inputs_dict

					def     lowerCamelCase__	(self       :		str						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertModelTester(self						)
							lowercase__	     =						ConfigTester(self ,   config_class=_UpperCAmelCase ,   emb_dim=37						)

					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Tuple:




							"""simple docstring"""


							self.config_tester.run_common_tests()

					def     lowerCamelCase__	(self       :		List[Any]						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_model(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		List[Any]						)		->  str:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_lm_head(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  Tuple:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_simple_qa(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		str						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_qa(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		List[str]						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_sequence_classif(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_token_classif(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_multiple_choice(*_UpperCAmelCase						)

					@slow
					def     lowerCamelCase__	(self       :		int						)		->  List[Any]:




							"""simple docstring"""


							for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__	     =						FlaubertModel.from_pretrained(_UpperCAmelCase						)
									self.assertIsNotNone(_UpperCAmelCase						)


					@slow
					@require_torch_gpu
					def     lowerCamelCase__	(self       :		Dict						)		->  int:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
									# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
									if model_class == FlaubertForMultipleChoice:
											return

									lowercase__	     =						True
									lowercase__	     =						model_class(config=_UpperCAmelCase						)

									lowercase__	     =						self._prepare_for_class(_UpperCAmelCase ,   _UpperCAmelCase						)
									lowercase__	     =						torch.jit.trace(
									    _UpperCAmelCase ,   (inputs_dict["""input_ids"""].to("""cpu"""						), inputs_dict["""attention_mask"""].to("""cpu"""						))						)

									with tempfile.TemporaryDirectory() as tmp:
											torch.jit.save(_UpperCAmelCase ,   os.path.join(_UpperCAmelCase ,   """traced_model.pt"""						)						)
											lowercase__	     =						torch.jit.load(os.path.join(_UpperCAmelCase ,   """traced_model.pt"""						) ,   map_location=_UpperCAmelCase						)
											loaded(inputs_dict["""input_ids"""].to(_UpperCAmelCase						) ,   inputs_dict["""attention_mask"""].to(_UpperCAmelCase						)						)






@require_torch
class 				A       (			unittest.TestCase				):
					'''simple docstring'''






					@slow
					def     lowerCamelCase__	(self       :		Tuple						)		->  Dict:




							"""simple docstring"""


							lowercase__	     =						FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""						)
							lowercase__	     =						torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]						)
							with torch.no_grad():
									lowercase__	     =						model(_UpperCAmelCase						)[0]
							lowercase__	     =						torch.Size((1, 11, 768)						)
							self.assertEqual(output.shape ,   _UpperCAmelCase						)
							lowercase__	     =						torch.tensor(
							    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]						)

							self.assertTrue(torch.allclose(output[:, :3, :3] ,   _UpperCAmelCase ,   atol=1E-4						)						)

