


"""simple docstring"""




import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    FlaubertForMultipleChoice,
					    FlaubertForQuestionAnswering,
					    FlaubertForQuestionAnsweringSimple,
					    FlaubertForSequenceClassification,
					    FlaubertForTokenClassification,
					    FlaubertModel,
					    FlaubertWithLMHeadModel,
					)
					from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class 			__snake_case   (lowerCamelCase  ):




							def __init__(     self:    Any     ,							A_:    Optional[int]     ,							A_:    Optional[int]=13     ,							A_:    Tuple=7     ,							A_:    Optional[Any]=True     ,							A_:    Optional[int]=True     ,							A_:    str=True     ,							A_:    int=True     ,							A_:    List[str]=True     ,							A_:    Dict=False     ,							A_:    Optional[Any]=False     ,							A_:    int=False     ,							A_:    Any=2     ,							A_:    int=99     ,							A_:    str=0     ,							A_:    Dict=32     ,							A_:    str=5     ,							A_:    str=4     ,							A_:    str=0.1     ,							A_:    str=0.1     ,							A_:    List[str]=5_12     ,							A_:    Optional[Any]=12     ,							A_:    Union[str, Any]=2     ,							A_:    Union[str, Any]=0.02     ,							A_:    List[str]=3     ,							A_:    int=4     ,							A_:    Optional[int]="last"     ,							A_:    Tuple=None     ,							A_:    Union[str, Any]=None     ,							):
										__lowerCamelCase          =      parent
										__lowerCamelCase          =      batch_size
										__lowerCamelCase          =      seq_length
										__lowerCamelCase          =      is_training
										__lowerCamelCase          =      use_input_lengths
										__lowerCamelCase          =      use_token_type_ids
										__lowerCamelCase          =      use_labels
										__lowerCamelCase          =      gelu_activation
										__lowerCamelCase          =      sinusoidal_embeddings
										__lowerCamelCase          =      causal
										__lowerCamelCase          =      asm
										__lowerCamelCase          =      n_langs
										__lowerCamelCase          =      vocab_size
										__lowerCamelCase          =      n_special
										__lowerCamelCase          =      hidden_size
										__lowerCamelCase          =      num_hidden_layers
										__lowerCamelCase          =      num_attention_heads
										__lowerCamelCase          =      hidden_dropout_prob
										__lowerCamelCase          =      attention_probs_dropout_prob
										__lowerCamelCase          =      max_position_embeddings
										__lowerCamelCase          =      type_vocab_size
										__lowerCamelCase          =      type_sequence_label_size
										__lowerCamelCase          =      initializer_range
										__lowerCamelCase          =      num_labels
										__lowerCamelCase          =      num_choices
										__lowerCamelCase          =      summary_type
										__lowerCamelCase          =      use_proj
										__lowerCamelCase          =      scope




							def   __a   (     self:    int   ):
										__lowerCamelCase          =      ids_tensor([self.batch_size, self.seq_length]     ,							self.vocab_size   )
										__lowerCamelCase          =      random_attention_mask([self.batch_size, self.seq_length]   )

										__lowerCamelCase          =      None
										if self.use_input_lengths:
													__lowerCamelCase          =      (
													    ids_tensor([self.batch_size]     ,							vocab_size=2   ) + self.seq_length - 2
													)  # small variation of seq_length

										__lowerCamelCase          =      None
										if self.use_token_type_ids:
													__lowerCamelCase          =      ids_tensor([self.batch_size, self.seq_length]     ,							self.n_langs   )

										__lowerCamelCase          =      None
										__lowerCamelCase          =      None
										__lowerCamelCase          =      None
										if self.use_labels:
													__lowerCamelCase          =      ids_tensor([self.batch_size]     ,							self.type_sequence_label_size   )
													__lowerCamelCase          =      ids_tensor([self.batch_size, self.seq_length]     ,							self.num_labels   )
													__lowerCamelCase          =      ids_tensor([self.batch_size]     ,							2   ).float()
													__lowerCamelCase          =      ids_tensor([self.batch_size]     ,							self.num_choices   )

										__lowerCamelCase          =      self.get_config()

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_lengths,
										    sequence_labels,
										    token_labels,
										    is_impossible_labels,
										    choice_labels,
										    input_mask,
										)




							def   __a   (     self:    List[Any]   ):
										return FlaubertConfig(
										    vocab_size=self.vocab_size     ,							n_special=self.n_special     ,							emb_dim=self.hidden_size     ,							n_layers=self.num_hidden_layers     ,							n_heads=self.num_attention_heads     ,							dropout=self.hidden_dropout_prob     ,							attention_dropout=self.attention_probs_dropout_prob     ,							gelu_activation=self.gelu_activation     ,							sinusoidal_embeddings=self.sinusoidal_embeddings     ,							asm=self.asm     ,							causal=self.causal     ,							n_langs=self.n_langs     ,							max_position_embeddings=self.max_position_embeddings     ,							initializer_range=self.initializer_range     ,							summary_type=self.summary_type     ,							use_proj=self.use_proj     ,							)




							def   __a   (     self:    Union[str, Any]     ,							A_:    Union[str, Any]     ,							A_:    Optional[Any]     ,							A_:    Optional[int]     ,							A_:    Union[str, Any]     ,							A_:    Optional[int]     ,							A_:    Dict     ,							A_:    Union[str, Any]     ,							A_:    Optional[int]     ,							A_:    List[str]     ,							):
										__lowerCamelCase          =      FlaubertModel(config=A_   )
										model.to(A_   )
										model.eval()
										__lowerCamelCase          =      model(A_     ,							lengths=A_     ,							langs=A_   )
										__lowerCamelCase          =      model(A_     ,							langs=A_   )
										__lowerCamelCase          =      model(A_   )
										self.parent.assertEqual(result.last_hidden_state.shape     ,							(self.batch_size, self.seq_length, self.hidden_size)   )




							def   __a   (     self:    Union[str, Any]     ,							A_:    Any     ,							A_:    Optional[int]     ,							A_:    int     ,							A_:    List[Any]     ,							A_:    Union[str, Any]     ,							A_:    Tuple     ,							A_:    Any     ,							A_:    Dict     ,							A_:    Optional[Any]     ,							):
										__lowerCamelCase          =      FlaubertWithLMHeadModel(A_   )
										model.to(A_   )
										model.eval()

										__lowerCamelCase          =      model(A_     ,							token_type_ids=A_     ,							labels=A_   )
										self.parent.assertEqual(result.loss.shape     ,							()   )
										self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.seq_length, self.vocab_size)   )




							def   __a   (     self:    Tuple     ,							A_:    Tuple     ,							A_:    Optional[int]     ,							A_:    Dict     ,							A_:    Tuple     ,							A_:    Tuple     ,							A_:    List[str]     ,							A_:    Optional[Any]     ,							A_:    Optional[int]     ,							A_:    Union[str, Any]     ,							):
										__lowerCamelCase          =      FlaubertForQuestionAnsweringSimple(A_   )
										model.to(A_   )
										model.eval()

										__lowerCamelCase          =      model(A_   )

										__lowerCamelCase          =      model(A_     ,							start_positions=A_     ,							end_positions=A_   )
										self.parent.assertEqual(result.start_logits.shape     ,							(self.batch_size, self.seq_length)   )
										self.parent.assertEqual(result.end_logits.shape     ,							(self.batch_size, self.seq_length)   )




							def   __a   (     self:    Dict     ,							A_:    List[Any]     ,							A_:    Union[str, Any]     ,							A_:    Dict     ,							A_:    List[str]     ,							A_:    Dict     ,							A_:    Optional[int]     ,							A_:    int     ,							A_:    str     ,							A_:    List[str]     ,							):
										__lowerCamelCase          =      FlaubertForQuestionAnswering(A_   )
										model.to(A_   )
										model.eval()

										__lowerCamelCase          =      model(A_   )

										__lowerCamelCase          =      model(
										    A_     ,							start_positions=A_     ,							end_positions=A_     ,							cls_index=A_     ,							is_impossible=A_     ,							p_mask=A_     ,							)

										__lowerCamelCase          =      model(
										    A_     ,							start_positions=A_     ,							end_positions=A_     ,							cls_index=A_     ,							is_impossible=A_     ,							)

										((__lowerCamelCase)     ,)          =      result_with_labels.to_tuple()

										__lowerCamelCase          =      model(A_     ,							start_positions=A_     ,							end_positions=A_   )

										((__lowerCamelCase)     ,)          =      result_with_labels.to_tuple()

										self.parent.assertEqual(result_with_labels.loss.shape     ,							()   )
										self.parent.assertEqual(result.start_top_log_probs.shape     ,							(self.batch_size, model.config.start_n_top)   )
										self.parent.assertEqual(result.start_top_index.shape     ,							(self.batch_size, model.config.start_n_top)   )
										self.parent.assertEqual(
										    result.end_top_log_probs.shape     ,							(self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
										self.parent.assertEqual(
										    result.end_top_index.shape     ,							(self.batch_size, model.config.start_n_top * model.config.end_n_top)   )
										self.parent.assertEqual(result.cls_logits.shape     ,							(self.batch_size,)   )




							def   __a   (     self:    List[Any]     ,							A_:    List[str]     ,							A_:    str     ,							A_:    Optional[int]     ,							A_:    Any     ,							A_:    int     ,							A_:    Union[str, Any]     ,							A_:    Tuple     ,							A_:    int     ,							A_:    Dict     ,							):
										__lowerCamelCase          =      FlaubertForSequenceClassification(A_   )
										model.to(A_   )
										model.eval()

										__lowerCamelCase          =      model(A_   )
										__lowerCamelCase          =      model(A_     ,							labels=A_   )

										self.parent.assertEqual(result.loss.shape     ,							()   )
										self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.type_sequence_label_size)   )




							def   __a   (     self:    Dict     ,							A_:    List[Any]     ,							A_:    List[Any]     ,							A_:    Any     ,							A_:    Union[str, Any]     ,							A_:    Optional[int]     ,							A_:    int     ,							A_:    List[str]     ,							A_:    Tuple     ,							A_:    Union[str, Any]     ,							):
										__lowerCamelCase          =      self.num_labels
										__lowerCamelCase          =      FlaubertForTokenClassification(A_   )
										model.to(A_   )
										model.eval()

										__lowerCamelCase          =      model(A_     ,							attention_mask=A_     ,							labels=A_   )
										self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.seq_length, self.num_labels)   )




							def   __a   (     self:    Tuple     ,							A_:    Tuple     ,							A_:    Optional[int]     ,							A_:    List[Any]     ,							A_:    Optional[Any]     ,							A_:    int     ,							A_:    List[str]     ,							A_:    Any     ,							A_:    Union[str, Any]     ,							A_:    Optional[Any]     ,							):
										__lowerCamelCase          =      self.num_choices
										__lowerCamelCase          =      FlaubertForMultipleChoice(config=A_   )
										model.to(A_   )
										model.eval()
										__lowerCamelCase          =      input_ids.unsqueeze(1   ).expand(-1     ,							self.num_choices     ,							-1   ).contiguous()
										__lowerCamelCase          =      token_type_ids.unsqueeze(1   ).expand(-1     ,							self.num_choices     ,							-1   ).contiguous()
										__lowerCamelCase          =      input_mask.unsqueeze(1   ).expand(-1     ,							self.num_choices     ,							-1   ).contiguous()
										__lowerCamelCase          =      model(
										    A_     ,							attention_mask=A_     ,							token_type_ids=A_     ,							labels=A_     ,							)
										self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.num_choices)   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.prepare_config_and_inputs()
										(
										    (
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,(
										    __lowerCamelCase
										)     ,
										)          =      config_and_inputs
										__lowerCamelCase          =      {
										    """input_ids""": input_ids,
										    """token_type_ids""": token_type_ids,
										    """lengths""": input_lengths,
										    """attention_mask""": input_mask,
										}
										return config, inputs_dict




@require_torch
class 			__snake_case   (lowerCamelCase			,		lowerCamelCase			,		unittest.TestCase  ):
							__a					 =       (
							    (
							        FlaubertModel,
							        FlaubertWithLMHeadModel,
							        FlaubertForQuestionAnswering,
							        FlaubertForQuestionAnsweringSimple,
							        FlaubertForSequenceClassification,
							        FlaubertForTokenClassification,
							        FlaubertForMultipleChoice,
							    )
							    if is_torch_available()
							    else ()
							)
							__a					 =       (
							    {
							        '''feature-extraction''': FlaubertModel,
							        '''fill-mask''': FlaubertWithLMHeadModel,
							        '''question-answering''': FlaubertForQuestionAnsweringSimple,
							        '''text-classification''': FlaubertForSequenceClassification,
							        '''token-classification''': FlaubertForTokenClassification,
							        '''zero-shot''': FlaubertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)




							def   __a   (     self:    Optional[Any]     ,							A_:    Any     ,							A_:    List[Any]     ,							A_:    int     ,							A_:    List[str]     ,							A_:    Optional[Any]   ):
										if (
										    pipeline_test_casse_name == "QAPipelineTests"
										    and tokenizer_name is not None
										    and not tokenizer_name.endswith("""Fast"""   )
										):
													# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
													# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
													# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
													return True

										return False




							def   __a   (     self:    Tuple     ,							A_:    Tuple     ,							A_:    Optional[Any]     ,							A_:    str=False   ):
										__lowerCamelCase          =      super()._prepare_for_class(A_     ,							A_     ,							return_labels=A_   )

										if return_labels:
													if model_class.__name__ == "FlaubertForQuestionAnswering":
																__lowerCamelCase          =      torch.zeros(
																    self.model_tester.batch_size     ,							dtype=torch.long     ,							device=A_   )
																__lowerCamelCase          =      torch.zeros(
																    self.model_tester.batch_size     ,							dtype=torch.long     ,							device=A_   )

										return inputs_dict




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      FlaubertModelTester(self   )
										__lowerCamelCase          =      ConfigTester(self     ,							config_class=A_     ,							emb_dim=37   )




							def   __a   (     self:    Tuple   ):
										self.config_tester.run_common_tests()




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_model(*A_   )




							def   __a   (     self:    str   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_lm_head(*A_   )




							def   __a   (     self:    Optional[Any]   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_simple_qa(*A_   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_qa(*A_   )




							def   __a   (     self:    Dict   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_sequence_classif(*A_   )




							def   __a   (     self:    str   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_token_classif(*A_   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_flaubert_multiple_choice(*A_   )




							@slow
							def   __a   (     self:    Union[str, Any]   ):
										for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													__lowerCamelCase          =      FlaubertModel.from_pretrained(A_   )
													self.assertIsNotNone(A_   )




							@slow
							@require_torch_gpu
							def   __a   (     self:    Optional[int]   ):
										__lowerCamelCase     ,__lowerCamelCase          =      self.model_tester.prepare_config_and_inputs_for_common()
										for model_class in self.all_model_classes:
													# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
													if model_class == FlaubertForMultipleChoice:
																return

													__lowerCamelCase          =      True
													__lowerCamelCase          =      model_class(config=A_   )

													__lowerCamelCase          =      self._prepare_for_class(A_     ,							A_   )
													__lowerCamelCase          =      torch.jit.trace(
													    A_     ,							(inputs_dict["""input_ids"""].to("""cpu"""   ), inputs_dict["""attention_mask"""].to("""cpu"""   ))   )

													with tempfile.TemporaryDirectory() as tmp:
																torch.jit.save(A_     ,							os.path.join(A_     ,							"""traced_model.pt"""   )   )
																__lowerCamelCase          =      torch.jit.load(os.path.join(A_     ,							"""traced_model.pt"""   )     ,							map_location=A_   )
																loaded(inputs_dict["""input_ids"""].to(A_   )     ,							inputs_dict["""attention_mask"""].to(A_   )   )




@require_torch
class 			__snake_case   (unittest.TestCase  ):




							@slow
							def   __a   (     self:    Optional[Any]   ):
										__lowerCamelCase          =      FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""   )
										__lowerCamelCase          =      torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]   )
										with torch.no_grad():
													__lowerCamelCase          =      model(A_   )[0]
										__lowerCamelCase          =      torch.Size((1, 11, 7_68)   )
										self.assertEqual(output.shape     ,							A_   )
										__lowerCamelCase          =      torch.tensor(
										    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]   )

										self.assertTrue(torch.allclose(output[:, :3, :3]     ,							A_     ,							atol=1E-4   )   )








"""simple docstring"""




import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__						: Dict    =				get_tests_dir('fixtures/test_sentencepiece.model')
__magic_name__						: int    =				get_tests_dir('fixtures/test_sentencepiece_bpe.model')

__magic_name__						: Any    =				'pt' if is_torch_available() else 'tf'




@require_sentencepiece
@require_tokenizers
class 			__snake_case   (lowerCamelCase			,		unittest.TestCase  ):
							__a					 =       CamembertTokenizer
							__a					 =       CamembertTokenizerFast
							__a					 =       True
							__a					 =       True




							def   __a   (     self:    List[Any]   ):
										super().setUp()

										# We have a SentencePiece fixture for testing
										__lowerCamelCase          =      CamembertTokenizer(A_   )
										tokenizer.save_pretrained(self.tmpdirname   )




							def   __a   (     self:    Union[str, Any]   ):
										__lowerCamelCase          =      """<pad>"""
										__lowerCamelCase          =      1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_   )     ,							A_   )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_   )     ,							A_   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      list(self.get_tokenizer().get_vocab().keys()   )

										self.assertEqual(vocab_keys[0]     ,							"""<s>NOTUSED"""   )
										self.assertEqual(vocab_keys[1]     ,							"""<pad>"""   )
										self.assertEqual(vocab_keys[-1]     ,							"""<mask>"""   )
										self.assertEqual(len(A_   )     ,							10_04   )




							def   __a   (     self:    Any   ):
										self.assertEqual(self.get_tokenizer().vocab_size     ,							10_05   )




							def   __a   (     self:    Optional[Any]   ):
										__lowerCamelCase          =      CamembertTokenizer(A_   )
										tokenizer.save_pretrained(self.tmpdirname   )
										__lowerCamelCase          =      CamembertTokenizerFast.from_pretrained(self.tmpdirname   )

										__lowerCamelCase          =      """I was born in 92000, and this is falsé."""

										__lowerCamelCase          =      tokenizer.encode(A_   )
										__lowerCamelCase          =      rust_tokenizer.encode(A_   )
										self.assertListEqual(A_     ,							A_   )

										__lowerCamelCase          =      tokenizer.encode(A_     ,							add_special_tokens=A_   )
										__lowerCamelCase          =      rust_tokenizer.encode(A_     ,							add_special_tokens=A_   )
										self.assertListEqual(A_     ,							A_   )

										# <unk> tokens are not the same for `rust` than for `slow`.
										# Because spm gives back raw token instead of `unk` in EncodeAsPieces
										# tokens = tokenizer.tokenize(sequence)
										__lowerCamelCase          =      tokenizer.convert_ids_to_tokens(A_   )
										__lowerCamelCase          =      rust_tokenizer.tokenize(A_   )
										self.assertListEqual(A_     ,							A_   )




							def   __a   (     self:    List[str]   ):
										if not self.test_rust_tokenizer:
													return

										__lowerCamelCase          =      self.get_tokenizer()
										__lowerCamelCase          =      self.get_rust_tokenizer()

										__lowerCamelCase          =      """I was born in 92000, and this is falsé."""

										__lowerCamelCase          =      tokenizer.tokenize(A_   )
										__lowerCamelCase          =      rust_tokenizer.tokenize(A_   )
										self.assertListEqual(A_     ,							A_   )

										__lowerCamelCase          =      tokenizer.encode(A_     ,							add_special_tokens=A_   )
										__lowerCamelCase          =      rust_tokenizer.encode(A_     ,							add_special_tokens=A_   )
										self.assertListEqual(A_     ,							A_   )

										__lowerCamelCase          =      self.get_rust_tokenizer()
										__lowerCamelCase          =      tokenizer.encode(A_   )
										__lowerCamelCase          =      rust_tokenizer.encode(A_   )
										self.assertListEqual(A_     ,							A_   )




							@slow
							def   __a   (     self:    Optional[Any]   ):
										# fmt: off
										__lowerCamelCase          =      {"""input_ids""": [[5, 54, 71_96, 2_97, 30, 23, 7_76, 18, 11, 32_15, 37_05, 82_52, 22, 31_64, 11_81, 21_16, 29, 16, 8_13, 25, 7_91, 33_14, 20, 34_46, 38, 2_75_75, 1_20, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_68, 17, 11, 90_88, 20, 15_17, 8, 2_28_04, 1_88_18, 10, 38, 6_29, 6_07, 6_07, 1_42, 19, 71_96, 8_67, 56, 1_03_26, 24, 22_67, 20, 4_16, 50_72, 1_56_12, 2_33, 7_34, 7, 23_99, 27, 16, 30_15, 16_49, 7, 24, 20, 43_38, 23_99, 27, 13, 34_00, 14, 13, 61_89, 8, 9_30, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										# camembert is a french model. So we also use french texts.
										__lowerCamelCase          =      [
										    """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
										    """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
										    """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
										    """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
										    """telles que la traduction et la synthèse de texte.""",
										]

										self.tokenizer_integration_test_util(
										    expected_encoding=A_     ,							model_name="""camembert-base"""     ,							revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""     ,							sequences=A_     ,							)




