


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import (
           TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
           TFXLMForMultipleChoice,
           TFXLMForQuestionAnsweringSimple,
           TFXLMForSequenceClassification,
           TFXLMForTokenClassification,
           TFXLMModel,
           TFXLMWithLMHeadModel,
           XLMConfig,
       )


class        A_					:




      '''simple docstring'''


      def __init__(				self:       Dict						,      a:       str						,      ):
       __lowerCamelCase		:							List[str]							=						parent
       __lowerCamelCase		:							List[Any]							=						13
       __lowerCamelCase		:							List[Any]							=						7
       __lowerCamelCase		:							List[Any]							=						True
       __lowerCamelCase		:							Union[str, Any]							=						True
       __lowerCamelCase		:							Any							=						True
       __lowerCamelCase		:							int							=						True
       __lowerCamelCase		:							Tuple							=						True
       __lowerCamelCase		:							List[str]							=						False
       __lowerCamelCase		:							Tuple							=						False
       __lowerCamelCase		:							str							=						False
       __lowerCamelCase		:							Union[str, Any]							=						2
       __lowerCamelCase		:							List[str]							=						99
       __lowerCamelCase		:							Optional[Any]							=						0
       __lowerCamelCase		:							str							=						32
       __lowerCamelCase		:							Optional[Any]							=						2
       __lowerCamelCase		:							Union[str, Any]							=						4
       __lowerCamelCase		:							Tuple							=						0.1
       __lowerCamelCase		:							Optional[int]							=						0.1
       __lowerCamelCase		:							Dict							=						512
       __lowerCamelCase		:							Optional[int]							=						16
       __lowerCamelCase		:							Tuple							=						2
       __lowerCamelCase		:							Optional[int]							=						0.0_2
       __lowerCamelCase		:							Any							=						3
       __lowerCamelCase		:							Dict							=						4
       __lowerCamelCase		:							Tuple							=						'last'
       __lowerCamelCase		:							Dict							=						True
       __lowerCamelCase		:							Optional[int]							=						None
       __lowerCamelCase		:							Dict							=						0




      def      _snake_case	(				self:       Optional[Any]				):
       __lowerCamelCase		:							Optional[Any]							=						ids_tensor([self.batch_size, self.seq_length]						,      self.vocab_size				)
       __lowerCamelCase		:							Union[str, Any]							=						random_attention_mask([self.batch_size, self.seq_length]						,      dtype=tf.floataa				)

       __lowerCamelCase		:							str							=						None
       if self.use_input_lengths:
        __lowerCamelCase		:							Optional[int]							=						(
            ids_tensor([self.batch_size]						,      vocab_size=2				) + self.seq_length - 2
        )  # small variation of seq_length

       __lowerCamelCase		:							str							=						None
       if self.use_token_type_ids:
        __lowerCamelCase		:							str							=						ids_tensor([self.batch_size, self.seq_length]						,      self.n_langs				)

       __lowerCamelCase		:							Union[str, Any]							=						None
       __lowerCamelCase		:							Tuple							=						None
       __lowerCamelCase		:							str							=						None
       if self.use_labels:
        __lowerCamelCase		:							List[Any]							=						ids_tensor([self.batch_size]						,      self.type_sequence_label_size				)
        __lowerCamelCase		:							Optional[int]							=						ids_tensor([self.batch_size, self.seq_length]						,      self.num_labels				)
        __lowerCamelCase		:							Any							=						ids_tensor([self.batch_size]						,      2						,      dtype=tf.floataa				)
        __lowerCamelCase		:							List[Any]							=						ids_tensor([self.batch_size]						,      self.num_choices				)

       __lowerCamelCase		:							Union[str, Any]							=						XLMConfig(
           vocab_size=self.vocab_size						,      n_special=self.n_special						,      emb_dim=self.hidden_size						,      n_layers=self.num_hidden_layers						,      n_heads=self.num_attention_heads						,      dropout=self.hidden_dropout_prob						,      attention_dropout=self.attention_probs_dropout_prob						,      gelu_activation=self.gelu_activation						,      sinusoidal_embeddings=self.sinusoidal_embeddings						,      asm=self.asm						,      causal=self.causal						,      n_langs=self.n_langs						,      max_position_embeddings=self.max_position_embeddings						,      initializer_range=self.initializer_range						,      summary_type=self.summary_type						,      use_proj=self.use_proj						,      bos_token_id=self.bos_token_id						,      )

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )




      def      _snake_case	(				self:       int						,      a:       Union[str, Any]						,      a:       Optional[int]						,      a:       List[str]						,      a:       Dict						,      a:       Optional[Any]						,      a:       Any						,      a:       int						,      a:       str						,      a:       Union[str, Any]						,      ):
       __lowerCamelCase		:							int							=						TFXLMModel(config=a				)
       __lowerCamelCase		:							Union[str, Any]							=						{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       __lowerCamelCase		:							str							=						model(a				)

       __lowerCamelCase		:							List[str]							=						[input_ids, input_mask]
       __lowerCamelCase		:							List[str]							=						model(a				)
       self.parent.assertEqual(result.last_hidden_state.shape						,      (self.batch_size, self.seq_length, self.hidden_size)				)




      def      _snake_case	(				self:       List[str]						,      a:       Union[str, Any]						,      a:       Dict						,      a:       Union[str, Any]						,      a:       Optional[Any]						,      a:       Union[str, Any]						,      a:       Optional[int]						,      a:       Any						,      a:       Union[str, Any]						,      a:       Any						,      ):
       __lowerCamelCase		:							Optional[Any]							=						TFXLMWithLMHeadModel(a				)

       __lowerCamelCase		:							int							=						{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       __lowerCamelCase		:							Dict							=						model(a				)

       __lowerCamelCase		:							Union[str, Any]							=						outputs

       self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.vocab_size)				)




      def      _snake_case	(				self:       Dict						,      a:       List[str]						,      a:       Any						,      a:       Optional[int]						,      a:       str						,      a:       str						,      a:       str						,      a:       Dict						,      a:       str						,      a:       Union[str, Any]						,      ):
       __lowerCamelCase		:							Any							=						TFXLMForQuestionAnsweringSimple(a				)

       __lowerCamelCase		:							Union[str, Any]							=						{'input_ids': input_ids, 'lengths': input_lengths}

       __lowerCamelCase		:							Optional[Any]							=						model(a				)

       self.parent.assertEqual(result.start_logits.shape						,      (self.batch_size, self.seq_length)				)
       self.parent.assertEqual(result.end_logits.shape						,      (self.batch_size, self.seq_length)				)




      def      _snake_case	(				self:       str						,      a:       str						,      a:       Union[str, Any]						,      a:       Dict						,      a:       str						,      a:       List[str]						,      a:       Tuple						,      a:       List[str]						,      a:       List[str]						,      a:       Optional[Any]						,      ):
       __lowerCamelCase		:							str							=						TFXLMForSequenceClassification(a				)

       __lowerCamelCase		:							List[str]							=						{'input_ids': input_ids, 'lengths': input_lengths}

       __lowerCamelCase		:							List[str]							=						model(a				)

       self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.type_sequence_label_size)				)




      def      _snake_case	(				self:       str						,      a:       Tuple						,      a:       Union[str, Any]						,      a:       Dict						,      a:       str						,      a:       str						,      a:       Tuple						,      a:       Union[str, Any]						,      a:       List[Any]						,      a:       Union[str, Any]						,      ):
       __lowerCamelCase		:							str							=						self.num_labels
       __lowerCamelCase		:							Dict							=						TFXLMForTokenClassification(config=a				)
       __lowerCamelCase		:							str							=						{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
       __lowerCamelCase		:							Optional[int]							=						model(a				)
       self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.num_labels)				)




      def      _snake_case	(				self:       Optional[int]						,      a:       int						,      a:       Optional[Any]						,      a:       str						,      a:       Tuple						,      a:       Optional[int]						,      a:       Dict						,      a:       str						,      a:       Union[str, Any]						,      a:       Dict						,      ):
       __lowerCamelCase		:							Union[str, Any]							=						self.num_choices
       __lowerCamelCase		:							Union[str, Any]							=						TFXLMForMultipleChoice(config=a				)
       __lowerCamelCase		:							Union[str, Any]							=						tf.tile(tf.expand_dims(a						,      1				)						,      (1, self.num_choices, 1)				)
       __lowerCamelCase		:							List[Any]							=						tf.tile(tf.expand_dims(a						,      1				)						,      (1, self.num_choices, 1)				)
       __lowerCamelCase		:							str							=						tf.tile(tf.expand_dims(a						,      1				)						,      (1, self.num_choices, 1)				)
       __lowerCamelCase		:							Optional[int]							=						{
           'input_ids': multiple_choice_inputs_ids,
           'attention_mask': multiple_choice_input_mask,
           'token_type_ids': multiple_choice_token_type_ids,
       }
       __lowerCamelCase		:							Optional[Any]							=						model(a				)
       self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.num_choices)				)




      def      _snake_case	(				self:       int				):
       __lowerCamelCase		:							List[Any]							=						self.prepare_config_and_inputs()
       (
           (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     (
           __lowerCamelCase
       )					,     
       )		:							Optional[Any]							=						config_and_inputs
       __lowerCamelCase		:							int							=						{
           'input_ids': input_ids,
           'token_type_ids': token_type_ids,
           'langs': token_type_ids,
           'lengths': input_lengths,
       }
       return config, inputs_dict



@require_tf
class        A_					(       __UpperCamelCase					,      __UpperCamelCase					,      unittest.TestCase    ):




      '''simple docstring'''


      __snake_case					       =   (
          (
              TFXLMModel,
              TFXLMWithLMHeadModel,
              TFXLMForSequenceClassification,
              TFXLMForQuestionAnsweringSimple,
              TFXLMForTokenClassification,
              TFXLMForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      __snake_case					       =   (
          (TFXLMWithLMHeadModel,) if is_tf_available() else ()
      )  # TODO (PVP): Check other models whether language generation is also applicable
      __snake_case					       =   (
          {
              """feature-extraction""": TFXLMModel,
              """fill-mask""": TFXLMWithLMHeadModel,
              """question-answering""": TFXLMForQuestionAnsweringSimple,
              """text-classification""": TFXLMForSequenceClassification,
              """text-generation""": TFXLMWithLMHeadModel,
              """token-classification""": TFXLMForTokenClassification,
              """zero-shot""": TFXLMForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      __snake_case					       =   False
      __snake_case					       =   False




      def      _snake_case	(				self:       Dict						,      a:       Dict						,      a:       Optional[int]						,      a:       List[Any]						,      a:       int						,      a:       Union[str, Any]				):
       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('Fast'				)
       ):
        # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
        # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
        # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
        return True

       return False




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							int							=						TFXLMModelTester(self				)
       __lowerCamelCase		:							Dict							=						ConfigTester(self						,      config_class=a						,      emb_dim=37				)




      def      _snake_case	(				self:       Dict				):
       self.config_tester.run_common_tests()




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							List[Any]							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*a				)




      def      _snake_case	(				self:       str				):
       __lowerCamelCase		:							str							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*a				)




      def      _snake_case	(				self:       int				):
       __lowerCamelCase		:							Tuple							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*a				)




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							str							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*a				)




      def      _snake_case	(				self:       Any				):
       __lowerCamelCase		:							Dict							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_token_classification(*a				)




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							Any							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*a				)




      @slow
      def      _snake_case	(				self:       Optional[Any]				):
       for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        __lowerCamelCase		:							str							=						TFXLMModel.from_pretrained(a				)
        self.assertIsNotNone(a				)



@require_tf
class        A_					(       unittest.TestCase    ):




      '''simple docstring'''


      @slow
      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							Union[str, Any]							=						TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'				)
       __lowerCamelCase		:							Dict							=						tf.convert_to_tensor([[14, 447]]						,      dtype=tf.intaa				)  # the president
       __lowerCamelCase		:							Optional[int]							=						[
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       __lowerCamelCase		:							Union[str, Any]							=						model.generate(a						,      do_sample=a				)
       self.assertListEqual(output_ids[0].numpy().tolist()						,      a				)





import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_           =	get_tests_dir('fixtures/spiece.model')


@require_sentencepiece
@require_tokenizers
class        A_					(       __UpperCamelCase					,      unittest.TestCase    ):




      '''simple docstring'''


      __snake_case					       =   ErnieMTokenizer
      __snake_case					       =   False
      __snake_case					       =   True
      __snake_case					       =   False
      __snake_case					       =   False




      def      _snake_case	(				self:       Optional[int]				):
       super().setUp()

       # We have a SentencePiece fixture for testing
       __lowerCamelCase		:							List[str]							=						ErnieMTokenizer(a						,      unk_token='<unk>'						,      pad_token='<pad>'				)
       tokenizer.save_pretrained(self.tmpdirname				)




      def      _snake_case	(				self:       Dict						,      a:       List[Any]				):
       __lowerCamelCase		:							Optional[Any]							=						'this is a test'
       __lowerCamelCase		:							Optional[int]							=						'this is a test'
       return input_text, output_text




      def      _snake_case	(				self:       Tuple				):
       __lowerCamelCase		:							List[str]							=						'<pad>'
       __lowerCamelCase		:							Union[str, Any]							=						0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(a				)						,      a				)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(a				)						,      a				)




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							List[str]							=						list(self.get_tokenizer().get_vocab().keys()				)

       self.assertEqual(vocab_keys[0]						,      '<pad>'				)
       self.assertEqual(vocab_keys[1]						,      '<unk>'				)
       self.assertEqual(vocab_keys[-1]						,      '▁eloquent'				)
       self.assertEqual(len(a				)						,      3_0000				)




      def      _snake_case	(				self:       Tuple				):
       self.assertEqual(self.get_tokenizer().vocab_size						,      3_0000				)




      def      _snake_case	(				self:       int				):
       if not self.test_rust_tokenizer:
        return

       __lowerCamelCase		:							Any							=						self.get_tokenizer()
       __lowerCamelCase		:							Optional[Any]							=						self.get_rust_tokenizer()

       __lowerCamelCase		:							Optional[int]							=						'I was born in 92000, and this is falsé.'

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.tokenize(a				)
       __lowerCamelCase		:							Dict							=						rust_tokenizer.tokenize(a				)
       self.assertListEqual(a						,      a				)

       __lowerCamelCase		:							List[str]							=						tokenizer.encode(a						,      add_special_tokens=a				)
       __lowerCamelCase		:							List[Any]							=						rust_tokenizer.encode(a						,      add_special_tokens=a				)
       self.assertListEqual(a						,      a				)

       __lowerCamelCase		:							str							=						self.get_rust_tokenizer()
       __lowerCamelCase		:							str							=						tokenizer.encode(a				)
       __lowerCamelCase		:							List[str]							=						rust_tokenizer.encode(a				)
       self.assertListEqual(a						,      a				)




      def      _snake_case	(				self:       Optional[Any]				):
       __lowerCamelCase		:							Union[str, Any]							=						ErnieMTokenizer(a						,      do_lower_case=a						,      unk_token='<unk>'						,      pad_token='<pad>'				)

       __lowerCamelCase		:							List[str]							=						tokenizer.tokenize('This is a test'				)
       self.assertListEqual(a						,      ['▁this', '▁is', '▁a', '▁test']				)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)						,      [48, 25, 21, 1289]				)

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.tokenize('I was born in 92000, and this is falsé.'				)
       # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
       self.assertListEqual(
           a						,      ['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']				)
       __lowerCamelCase		:							Any							=						tokenizer.convert_tokens_to_ids(a				)
       self.assertListEqual(a						,      [31, 23, 386, 19, 518, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]				)

       __lowerCamelCase		:							int							=						tokenizer.convert_ids_to_tokens(a				)
       self.assertListEqual(
           a						,      ['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']						,      )




      def      _snake_case	(				self:       str				):
       __lowerCamelCase		:							Tuple							=						ErnieMTokenizer(a						,      unk_token='<unk>'						,      pad_token='<pad>'				)

       __lowerCamelCase		:							Tuple							=						tokenizer.encode('sequence builders'				)
       __lowerCamelCase		:							List[Any]							=						tokenizer.encode('multi-sequence build'				)

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.build_inputs_with_special_tokens(a				)
       __lowerCamelCase		:							Tuple							=						tokenizer.build_inputs_with_special_tokens(a						,      a				)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
           tokenizer.sep_token_id
       ] + text_a + [tokenizer.sep_token_id]




      @slow
      def      _snake_case	(				self:       List[Any]				):
       # fmt: off
       __lowerCamelCase		:							int							=						{'input_ids': [[0, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 9, 304, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 5, 5, 5, 16, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 6460, 1328, 4589, 42, 12_2009, 11_5774, 23, 3559, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [0, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=a						,      model_name='susnato/ernie-m-base_pytorch'						,      sequences=[
               'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
               'general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
               'Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained '
               'models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.',
               'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
               'conditioning on both left and right context in all layers.',
               'The quick brown fox jumps over the lazy dog.',
           ]						,      )

