



"""simple docstring"""
from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel

def    _snake_case		(		UpperCamelCase		:	List[str]    ,	UpperCamelCase		:	Any    ,	UpperCamelCase		:	Union[str, Any]=None    ,	UpperCamelCase		:	Tuple=None		):
 if attention_mask is None:
  UpperCAmelCase			:      List[str]			=			tf.cast(tf.math.not_equal(__SCREAMING_SNAKE_CASE    ,	config.pad_token_id		)    ,	tf.inta		)
 return {"input_ids": input_ids, "attention_mask": attention_mask}



@require_tf
class    SCREAMING_SNAKE_CASE__						:
   __lowerCAmelCase						:     Any		=		OPTConfig
   __lowerCAmelCase						:     int		=		{}
   __lowerCAmelCase						:     Union[str, Any]		=		'gelu'

   def __init__(     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE=13   ,  _SCREAMING_SNAKE_CASE=7   ,  _SCREAMING_SNAKE_CASE=True   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=99   ,  _SCREAMING_SNAKE_CASE=16   ,  _SCREAMING_SNAKE_CASE=2   ,  _SCREAMING_SNAKE_CASE=4   ,  _SCREAMING_SNAKE_CASE=4   ,  _SCREAMING_SNAKE_CASE="gelu"   ,  _SCREAMING_SNAKE_CASE=0.1   ,  _SCREAMING_SNAKE_CASE=0.1   ,  _SCREAMING_SNAKE_CASE=20   ,  _SCREAMING_SNAKE_CASE=2   ,  _SCREAMING_SNAKE_CASE=1   ,  _SCREAMING_SNAKE_CASE=0   ,  _SCREAMING_SNAKE_CASE=16   ,  _SCREAMING_SNAKE_CASE=16   ,  )						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      Dict			=			parent
    UpperCAmelCase			:      Optional[Any]			=			batch_size
    UpperCAmelCase			:      Dict			=			seq_length
    UpperCAmelCase			:      Tuple			=			is_training
    UpperCAmelCase			:      List[str]			=			use_labels
    UpperCAmelCase			:      Any			=			vocab_size
    UpperCAmelCase			:      Any			=			hidden_size
    UpperCAmelCase			:      Tuple			=			num_hidden_layers
    UpperCAmelCase			:      Optional[int]			=			num_attention_heads
    UpperCAmelCase			:      str			=			intermediate_size
    UpperCAmelCase			:      Optional[int]			=			hidden_act
    UpperCAmelCase			:      Tuple			=			hidden_dropout_prob
    UpperCAmelCase			:      Dict			=			attention_probs_dropout_prob
    UpperCAmelCase			:      Optional[Any]			=			max_position_embeddings
    UpperCAmelCase			:      int			=			eos_token_id
    UpperCAmelCase			:      List[str]			=			pad_token_id
    UpperCAmelCase			:      Dict			=			bos_token_id
    UpperCAmelCase			:      Union[str, Any]			=			embed_dim
    UpperCAmelCase			:      Union[str, Any]			=			word_embed_proj_dim
    UpperCAmelCase			:      List[Any]			=			False

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      str			=			ids_tensor([self.batch_size, self.seq_length - 1]   ,  self.vocab_size		)
    UpperCAmelCase			:      Optional[Any]			=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size		)   ,  1		)
    UpperCAmelCase			:      int			=			tf.concat([input_ids, eos_tensor]   ,  axis=1		)

    UpperCAmelCase			:      Any			=			self.config_cls(
        vocab_size=self.vocab_size   ,  hidden_size=self.hidden_size   ,  num_hidden_layers=self.num_hidden_layers   ,  num_attention_heads=self.num_attention_heads   ,  ffn_dim=self.intermediate_size   ,  dropout=self.hidden_dropout_prob   ,  attention_dropout=self.attention_probs_dropout_prob   ,  max_position_embeddings=self.max_position_embeddings   ,  eos_token_id=self.eos_token_id   ,  bos_token_id=self.bos_token_id   ,  pad_token_id=self.pad_token_id   ,  embed_dim=self.embed_dim   ,  word_embed_proj_dim=self.word_embed_proj_dim   ,  is_encoder_decoder=_SCREAMING_SNAKE_CASE   ,  **self.config_updates   ,  )
    UpperCAmelCase			:      str			=			prepare_opt_inputs_dict(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
    return config, inputs_dict




   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   str:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			TFOPTModel(config=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      int			=			inputs_dict["""input_ids"""]

    UpperCAmelCase			:      List[str]			=			input_ids[:1, :]
    UpperCAmelCase			:      Tuple			=			inputs_dict["""attention_mask"""][:1, :]
    UpperCAmelCase			:      Dict			=			1

    # first forward pass
    UpperCAmelCase			:      Dict			=			model(_SCREAMING_SNAKE_CASE   ,  attention_mask=_SCREAMING_SNAKE_CASE   ,  use_cache=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			,	UpperCAmelCase			:      str			=			outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    UpperCAmelCase			:      str			=			ids_tensor((self.batch_size, 3)   ,  config.vocab_size		)
    UpperCAmelCase			:      Dict			=			tf.cast(ids_tensor((self.batch_size, 3)   ,  2		)   ,  tf.inta		)

    # append to next input_ids and
    UpperCAmelCase			:      List[str]			=			tf.concat([input_ids, next_tokens]   ,  axis=-1		)
    UpperCAmelCase			:      Any			=			tf.concat([attention_mask, next_attn_mask]   ,  axis=-1		)

    UpperCAmelCase			:      Any			=			model(_SCREAMING_SNAKE_CASE   ,  attention_mask=_SCREAMING_SNAKE_CASE		)[0]
    UpperCAmelCase			:      Any			=			model(_SCREAMING_SNAKE_CASE   ,  attention_mask=_SCREAMING_SNAKE_CASE   ,  past_key_values=_SCREAMING_SNAKE_CASE		)[0]

    self.parent.assertEqual(next_tokens.shape[1]   ,  output_from_past.shape[1]		)

    # select random slice
    UpperCAmelCase			:      int			=			int(ids_tensor((1,)   ,  output_from_past.shape[-1]		)		)
    UpperCAmelCase			:      Union[str, Any]			=			output_from_no_past[:, -3:, random_slice_idx]
    UpperCAmelCase			:      Optional[int]			=			output_from_past[:, :, random_slice_idx]

    # test that outputs are equal for slice
    tf.debugging.assert_near(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  rtol=1E-3		)



@require_tf
class    SCREAMING_SNAKE_CASE__						(      lowercase__   ,    lowercase__   ,    unittest.TestCase				):
   __lowerCAmelCase						:     Tuple		=		(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
   __lowerCAmelCase						:     Optional[int]		=		(TFOPTForCausalLM,) if is_tf_available() else ()
   __lowerCAmelCase						:     Tuple		=		(
       {'feature-extraction': TFOPTModel, 'text-generation': TFOPTForCausalLM} if is_tf_available() else {}
   )
   __lowerCAmelCase						:     List[str]		=		False
   __lowerCAmelCase						:     Tuple		=		False
   __lowerCAmelCase						:     Union[str, Any]		=		False
   __lowerCAmelCase						:     Tuple		=		10

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Tuple:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			TFOPTModelTester(self		)
    UpperCAmelCase			:      Any			=			ConfigTester(self   ,  config_class=_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    self.config_tester.run_common_tests()

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[str]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			self.model_tester.prepare_config_and_inputs_for_common()
    self.model_tester.check_decoder_model_past_large_inputs(*_SCREAMING_SNAKE_CASE		)




   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    UpperCAmelCase			,	UpperCAmelCase			:      List[Any]			=			self.model_tester.prepare_config_and_inputs_for_common()

    def _get_word_embedding_weight(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		):
     if hasattr(_SCREAMING_SNAKE_CASE   ,  """weight"""		):
      return embedding_layer.weight
     else:
      # Here we build the word embeddings weights if not exists.
      # And then we retry to get the attribute once built.
      model.build()
      if hasattr(_SCREAMING_SNAKE_CASE   ,  """weight"""		):
       return embedding_layer.weight
      else:
       return None

    for model_class in self.all_model_classes:
     for size in [config.vocab_size - 10, config.vocab_size + 10]:
      # build the embeddings
      UpperCAmelCase			:      List[str]			=			model_class(config=_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Union[str, Any]			=			_get_word_embedding_weight(_SCREAMING_SNAKE_CASE   ,  model.get_input_embeddings()		)
      UpperCAmelCase			:      List[Any]			=			_get_word_embedding_weight(_SCREAMING_SNAKE_CASE   ,  model.get_output_embeddings()		)

      # reshape the embeddings
      model.resize_token_embeddings(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      List[Any]			=			_get_word_embedding_weight(_SCREAMING_SNAKE_CASE   ,  model.get_input_embeddings()		)
      UpperCAmelCase			:      str			=			_get_word_embedding_weight(_SCREAMING_SNAKE_CASE   ,  model.get_output_embeddings()		)

      # check that the resized embeddings size matches the desired size.
      UpperCAmelCase			:      Dict			=			size if size is not None else config.vocab_size

      self.assertEqual(new_input_embeddings.shape[0]   ,  _SCREAMING_SNAKE_CASE		)

      # check that weights remain the same after resizing
      UpperCAmelCase			:      Tuple			=			True
      for pa, pa in zip(old_input_embeddings.value()   ,  new_input_embeddings.value()		):
       if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
        UpperCAmelCase			:      Tuple			=			False
      self.assertTrue(_SCREAMING_SNAKE_CASE		)

      if old_output_embeddings is not None and new_output_embeddings is not None:
       self.assertEqual(new_output_embeddings.shape[0]   ,  _SCREAMING_SNAKE_CASE		)

       UpperCAmelCase			:      int			=			True
       for pa, pa in zip(old_output_embeddings.value()   ,  new_output_embeddings.value()		):
        if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
         UpperCAmelCase			:      List[str]			=			False
       self.assertTrue(_SCREAMING_SNAKE_CASE		)

def    _snake_case		(		UpperCamelCase		:	Union[str, Any]		):
 return tf.constant(__SCREAMING_SNAKE_CASE    ,	dtype=tf.intaa		)



@require_tf
class    SCREAMING_SNAKE_CASE__						(      unittest.TestCase				):
   __lowerCAmelCase						:     List[Any]		=		99

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   int:





    '''simple docstring'''




    UpperCAmelCase			:      Dict			=			tf.ones((4, 1)   ,  dtype=tf.intaa		) * 2
    UpperCAmelCase			:      Tuple			=			tf.concat([ids_tensor((4, 6)   ,  self.vocab_size - 3		) + 3, eos_column_vector]   ,  axis=1		)
    UpperCAmelCase			:      str			=			input_ids.shape[0]
    UpperCAmelCase			:      str			=			OPTConfig(
        vocab_size=self.vocab_size   ,  hidden_size=24   ,  num_hidden_layers=2   ,  num_attention_heads=2   ,  ffn_dim=32   ,  max_position_embeddings=48   ,  eos_token_id=2   ,  pad_token_id=1   ,  bos_token_id=0   ,  )
    return config, input_ids, batch_size



@require_sentencepiece
@require_tf
class    SCREAMING_SNAKE_CASE__						(      unittest.TestCase				):

   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Dict:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			TFOPTModel.from_pretrained("""facebook/opt-350m"""		)
    UpperCAmelCase			:      List[Any]			=			_long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]		)
    UpperCAmelCase			:      Optional[Any]			=			tf.not_equal(_SCREAMING_SNAKE_CASE   ,  model.config.pad_token_id		)
    with tf.GradientTape():
     UpperCAmelCase			:      List[str]			=			model(input_ids=_SCREAMING_SNAKE_CASE   ,  attention_mask=_SCREAMING_SNAKE_CASE		).last_hidden_state
    UpperCAmelCase			:      Optional[Any]			=			(1, 11, 512)
    self.assertEqual(output.shape   ,  _SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Any			=			tf.constant(
        [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]		)
    self.assertTrue(np.allclose(output[:, :3, :3]   ,  _SCREAMING_SNAKE_CASE   ,  atol=4E-3		)		)

    UpperCAmelCase			:      List[str]			=			tf.function(_SCREAMING_SNAKE_CASE   ,  jit_compile=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Optional[Any]			=			xla_generate(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)[0]
    self.assertTrue(np.allclose(output[:, :3, :3]   ,  _SCREAMING_SNAKE_CASE   ,  atol=4E-2		)		)



@require_tf
@slow
class    SCREAMING_SNAKE_CASE__						(      unittest.TestCase				):

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    super().setUp()
    UpperCAmelCase			:      Union[str, Any]			=			"""facebook/opt-350m"""




   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			TFOPTForCausalLM.from_pretrained(self.path_model		)
    UpperCAmelCase			:      Optional[Any]			=			GPTaTokenizer.from_pretrained(self.path_model		)

    UpperCAmelCase			:      Optional[int]			=			[
        """Today is a beautiful day and I want to""",
        """In the city of""",
        """Paris is the capital of France and""",
        """Computers and mobile phones have taken""",
    ]
    # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
    UpperCAmelCase			:      str			=			tokenizer(_SCREAMING_SNAKE_CASE   ,  return_tensors="""tf"""   ,  padding=_SCREAMING_SNAKE_CASE   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      int			=			tf.math.reduce_mean(model(inputs.input_ids   ,  attention_mask=inputs.attention_mask		)[0]   ,  axis=-1		)
    UpperCAmelCase			:      Tuple			=			tf.constant(
        [
            [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
            [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
            [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
            [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
        ]		)
    self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  atol=1E-4		)		)

    UpperCAmelCase			:      List[str]			=			tf.function(_SCREAMING_SNAKE_CASE   ,  jit_compile=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Optional[int]			=			tf.math.reduce_mean(xla_generate(inputs.input_ids   ,  attention_mask=inputs.attention_mask		)[0]   ,  axis=-1		)
    self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  atol=1E-4		)		)



@require_tf
@slow
class    SCREAMING_SNAKE_CASE__						(      unittest.TestCase				):

   @property
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[Any]:





    '''simple docstring'''




    return [
        "Today is a beautiful day and I want",
        "In the city of",
        "Paris is the capital of France and",
        "Computers and mobile phones have taken",
    ]

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			"""facebook/opt-125m"""

    UpperCAmelCase			:      Dict			=			[
        """Today is a beautiful day and I want to""",
        """In the city of New York, the city""",
        """Paris is the capital of France and the capital""",
        """Computers and mobile phones have taken over the""",
    ]

    UpperCAmelCase			:      Union[str, Any]			=			[]
    UpperCAmelCase			:      Optional[int]			=			GPTaTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			TFOPTForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)

    for prompt in self.prompts:
     UpperCAmelCase			:      Tuple			=			tokenizer(_SCREAMING_SNAKE_CASE   ,  return_tensors="""tf"""		).input_ids

     UpperCAmelCase			:      Tuple			=			model.generate(_SCREAMING_SNAKE_CASE   ,  max_length=10		)

     UpperCAmelCase			:      int			=			tokenizer.batch_decode(_SCREAMING_SNAKE_CASE   ,  skip_special_tokens=_SCREAMING_SNAKE_CASE		)
     predicted_outputs += generated_string

    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      Dict			=			"""facebook/opt-350m"""

    UpperCAmelCase			:      List[str]			=			GPTaTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      List[Any]			=			TFOPTForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      str			=			"""left"""

    # use different length sentences to test batching
    UpperCAmelCase			:      Any			=			[
        """Hello, my dog is a little""",
        """Today, I""",
    ]

    UpperCAmelCase			:      int			=			tokenizer(_SCREAMING_SNAKE_CASE   ,  return_tensors="""tf"""   ,  padding=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Optional[int]			=			inputs["""input_ids"""]

    UpperCAmelCase			:      Optional[int]			=			model.generate(input_ids=_SCREAMING_SNAKE_CASE   ,  attention_mask=inputs["""attention_mask"""]		)

    UpperCAmelCase			:      str			=			tokenizer(sentences[0]   ,  return_tensors="""tf"""		).input_ids
    UpperCAmelCase			:      Tuple			=			model.generate(input_ids=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Any			=			inputs_non_padded.shape[-1] - tf.math.reduce_sum(
        tf.cast(inputs["""attention_mask"""][-1]   ,  tf.intaa		)		)
    UpperCAmelCase			:      List[Any]			=			tokenizer(sentences[1]   ,  return_tensors="""tf"""		).input_ids
    UpperCAmelCase			:      Optional[Any]			=			model.generate(input_ids=_SCREAMING_SNAKE_CASE   ,  max_length=model.config.max_length - num_paddings		)

    UpperCAmelCase			:      List[Any]			=			tokenizer.batch_decode(_SCREAMING_SNAKE_CASE   ,  skip_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Any			=			tokenizer.decode(output_non_padded[0]   ,  skip_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Union[str, Any]			=			tokenizer.decode(output_padded[0]   ,  skip_special_tokens=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Optional[Any]			=			[
        """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
        """Today, I was in the middle of a conversation with a friend about the""",
    ]
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  [non_padded_sentence, padded_sentence]		)




   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			"""facebook/opt-350m"""

    UpperCAmelCase			:      Dict			=			[
        """Today is a beautiful day and I want to""",
        """In the city of San Francisco, the city""",
        """Paris is the capital of France and the capital""",
        """Computers and mobile phones have taken over the""",
    ]

    UpperCAmelCase			:      Optional[int]			=			[]
    UpperCAmelCase			:      Optional[int]			=			GPTaTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			TFOPTForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)

    for prompt in self.prompts:
     UpperCAmelCase			:      Optional[int]			=			tokenizer(_SCREAMING_SNAKE_CASE   ,  return_tensors="""tf"""		).input_ids

     UpperCAmelCase			:      Any			=			model.generate(_SCREAMING_SNAKE_CASE   ,  max_length=10		)

     UpperCAmelCase			:      Any			=			tokenizer.batch_decode(_SCREAMING_SNAKE_CASE   ,  skip_special_tokens=_SCREAMING_SNAKE_CASE		)
     predicted_outputs += generated_string

    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)





"""simple docstring"""
import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
      import faiss



@require_faiss
class    SCREAMING_SNAKE_CASE__						(      UpperCAmelCase__				):

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase			:      str			=			tempfile.mkdtemp()
    UpperCAmelCase			:      str			=			8

    # DPR tok
    UpperCAmelCase			:      int			=			[
        """[UNK]""",
        """[CLS]""",
        """[SEP]""",
        """[PAD]""",
        """[MASK]""",
        """want""",
        """##want""",
        """##ed""",
        """wa""",
        """un""",
        """runn""",
        """##ing""",
        """,""",
        """low""",
        """lowest""",
    ]
    UpperCAmelCase			:      Tuple			=			os.path.join(self.tmpdirname   ,  """dpr_tokenizer"""		)
    os.makedirs(_SCREAMING_SNAKE_CASE   ,  exist_ok=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			os.path.join(_SCREAMING_SNAKE_CASE   ,  DPR_VOCAB_FILES_NAMES["""vocab_file"""]		)
    with open(self.vocab_file   ,  """w"""   ,  encoding="""utf-8"""		) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

    # BART tok
    UpperCAmelCase			:      int			=			[
        """l""",
        """o""",
        """w""",
        """e""",
        """r""",
        """s""",
        """t""",
        """i""",
        """d""",
        """n""",
        """\u0120""",
        """\u0120l""",
        """\u0120n""",
        """\u0120lo""",
        """\u0120low""",
        """er""",
        """\u0120lowest""",
        """\u0120newer""",
        """\u0120wider""",
        """<unk>""",
    ]
    UpperCAmelCase			:      Tuple			=			dict(zip(_SCREAMING_SNAKE_CASE   ,  range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
    UpperCAmelCase			:      List[Any]			=			["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
    UpperCAmelCase			:      List[Any]			=			{"""unk_token""": """<unk>"""}

    UpperCAmelCase			:      int			=			os.path.join(self.tmpdirname   ,  """bart_tokenizer"""		)
    os.makedirs(_SCREAMING_SNAKE_CASE   ,  exist_ok=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Union[str, Any]			=			os.path.join(_SCREAMING_SNAKE_CASE   ,  BART_VOCAB_FILES_NAMES["""vocab_file"""]		)
    UpperCAmelCase			:      Union[str, Any]			=			os.path.join(_SCREAMING_SNAKE_CASE   ,  BART_VOCAB_FILES_NAMES["""merges_file"""]		)
    with open(self.vocab_file   ,  """w"""   ,  encoding="""utf-8"""		) as fp:
     fp.write(json.dumps(_SCREAMING_SNAKE_CASE		) + """\n"""		)
    with open(self.merges_file   ,  """w"""   ,  encoding="""utf-8"""		) as fp:
     fp.write("""\n""".join(_SCREAMING_SNAKE_CASE		)		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   DPRQuestionEncoderTokenizer:





    '''simple docstring'''




    return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,  """dpr_tokenizer"""		)		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   DPRContextEncoderTokenizer:





    '''simple docstring'''




    return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,  """dpr_tokenizer"""		)		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   BartTokenizer:





    '''simple docstring'''




    return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname   ,  """bart_tokenizer"""		)		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    shutil.rmtree(self.tmpdirname		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Dict			=			Dataset.from_dict(
        {
            """id""": ["""0""", """1"""],
            """text""": ["""foo""", """bar"""],
            """title""": ["""Foo""", """Bar"""],
            """embeddings""": [np.ones(self.retrieval_vector_size		), 2 * np.ones(self.retrieval_vector_size		)],
        }		)
    dataset.add_faiss_index("""embeddings"""   ,  string_factory="""Flat"""   ,  metric_type=faiss.METRIC_INNER_PRODUCT		)
    return dataset

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			self.get_dummy_dataset()
    UpperCAmelCase			:      List[Any]			=			RagConfig(
        retrieval_vector_size=self.retrieval_vector_size   ,  question_encoder=DPRConfig().to_dict()   ,  generator=BartConfig().to_dict()   ,  )
    with patch("""transformers.models.rag.retrieval_rag.load_dataset"""		) as mock_load_dataset:
     UpperCAmelCase			:      Tuple			=			dataset
     UpperCAmelCase			:      List[str]			=			RagRetriever(
         _SCREAMING_SNAKE_CASE   ,  question_encoder_tokenizer=self.get_dpr_tokenizer()   ,  generator_tokenizer=self.get_bart_tokenizer()   ,  )
    return retriever

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   Dict:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			self.get_dummy_dataset()
    UpperCAmelCase			:      Any			=			RagConfig(
        retrieval_vector_size=self.retrieval_vector_size   ,  question_encoder=DPRConfig().to_dict()   ,  generator=BartConfig().to_dict()   ,  index_name="""custom"""   ,  )
    if from_disk:
     UpperCAmelCase			:      str			=			os.path.join(self.tmpdirname   ,  """dataset"""		)
     UpperCAmelCase			:      Union[str, Any]			=			os.path.join(self.tmpdirname   ,  """index.faiss"""		)
     dataset.get_index("""embeddings"""		).save(os.path.join(self.tmpdirname   ,  """index.faiss"""		)		)
     dataset.drop_index("""embeddings"""		)
     dataset.save_to_disk(os.path.join(self.tmpdirname   ,  """dataset"""		)		)
     del dataset
     UpperCAmelCase			:      List[Any]			=			RagRetriever(
         _SCREAMING_SNAKE_CASE   ,  question_encoder_tokenizer=self.get_dpr_tokenizer()   ,  generator_tokenizer=self.get_bart_tokenizer()   ,  )
    else:
     UpperCAmelCase			:      Optional[Any]			=			RagRetriever(
         _SCREAMING_SNAKE_CASE   ,  question_encoder_tokenizer=self.get_dpr_tokenizer()   ,  generator_tokenizer=self.get_bart_tokenizer()   ,  index=CustomHFIndex(config.retrieval_vector_size   ,  _SCREAMING_SNAKE_CASE		)   ,  )
    return retriever

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      List[str]			=			Dataset.from_dict(
        {
            """id""": ["""0""", """1"""],
            """text""": ["""foo""", """bar"""],
            """title""": ["""Foo""", """Bar"""],
            """embeddings""": [np.ones(self.retrieval_vector_size + 1		), 2 * np.ones(self.retrieval_vector_size + 1		)],
        }		)
    dataset.add_faiss_index("""embeddings"""   ,  string_factory="""Flat"""   ,  metric_type=faiss.METRIC_INNER_PRODUCT		)

    UpperCAmelCase			:      Optional[int]			=			os.path.join(self.tmpdirname   ,  """hf_bert_base.hnswSQ8_correct_phi_128.c_index"""		)
    dataset.save_faiss_index("""embeddings"""   ,  index_file_name + """.index.dpr"""		)
    pickle.dump(dataset["""id"""]   ,  open(index_file_name + """.index_meta.dpr"""   ,  """wb"""		)		)

    UpperCAmelCase			:      Dict			=			os.path.join(self.tmpdirname   ,  """psgs_w100.tsv.pkl"""		)
    UpperCAmelCase			:      Any			=			{sample["""id"""]: [sample["""text"""], sample["""title"""]] for sample in dataset}
    pickle.dump(_SCREAMING_SNAKE_CASE   ,  open(_SCREAMING_SNAKE_CASE   ,  """wb"""		)		)

    UpperCAmelCase			:      Dict			=			RagConfig(
        retrieval_vector_size=self.retrieval_vector_size   ,  question_encoder=DPRConfig().to_dict()   ,  generator=BartConfig().to_dict()   ,  index_name="""legacy"""   ,  index_path=self.tmpdirname   ,  )
    UpperCAmelCase			:      str			=			RagRetriever(
        _SCREAMING_SNAKE_CASE   ,  question_encoder_tokenizer=self.get_dpr_tokenizer()   ,  generator_tokenizer=self.get_bart_tokenizer()		)
    return retriever

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   int:





    '''simple docstring'''




    UpperCAmelCase			:      int			=			1
    UpperCAmelCase			:      List[Any]			=			self.get_dummy_canonical_hf_index_retriever()
    UpperCAmelCase			:      List[str]			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      List[Any]			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=_SCREAMING_SNAKE_CASE		)
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  2		)
    self.assertEqual(sorted(doc_dicts[0]		)   ,  ["""embeddings""", """id""", """text""", """title"""]		)
    self.assertEqual(len(doc_dicts[0]["""id"""]		)   ,  _SCREAMING_SNAKE_CASE		)
    self.assertEqual(doc_dicts[0]["""id"""][0]   ,  """1"""		)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]["""id"""][0]   ,  """0"""		)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()   ,  [[1], [0]]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			self.get_dummy_canonical_hf_index_retriever()
    with tempfile.TemporaryDirectory() as tmp_dirname:
     with patch("""transformers.models.rag.retrieval_rag.load_dataset"""		) as mock_load_dataset:
      UpperCAmelCase			:      Optional[int]			=			self.get_dummy_dataset()
      retriever.save_pretrained(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Tuple			=			RagRetriever.from_pretrained(_SCREAMING_SNAKE_CASE		)
      self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      List[Any]			=			np.array(
          [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
      UpperCAmelCase			:      Optional[Any]			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=1		)
      self.assertTrue(out is not None		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			1
    UpperCAmelCase			:      Dict			=			self.get_dummy_custom_hf_index_retriever(from_disk=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Optional[Any]			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      Union[str, Any]			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=_SCREAMING_SNAKE_CASE		)
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  2		)
    self.assertEqual(sorted(doc_dicts[0]		)   ,  ["""embeddings""", """id""", """text""", """title"""]		)
    self.assertEqual(len(doc_dicts[0]["""id"""]		)   ,  _SCREAMING_SNAKE_CASE		)
    self.assertEqual(doc_dicts[0]["""id"""][0]   ,  """1"""		)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]["""id"""][0]   ,  """0"""		)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()   ,  [[1], [0]]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[int]			=			self.get_dummy_custom_hf_index_retriever(from_disk=_SCREAMING_SNAKE_CASE		)
    with tempfile.TemporaryDirectory() as tmp_dirname:
     retriever.save_pretrained(_SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      Union[str, Any]			=			RagRetriever.from_pretrained(_SCREAMING_SNAKE_CASE		)
     self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      Optional[int]			=			np.array(
         [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
     UpperCAmelCase			:      List[str]			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=1		)
     self.assertTrue(out is not None		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			1
    UpperCAmelCase			:      Optional[Any]			=			self.get_dummy_custom_hf_index_retriever(from_disk=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Dict			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      str			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=_SCREAMING_SNAKE_CASE		)
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  2		)
    self.assertEqual(sorted(doc_dicts[0]		)   ,  ["""embeddings""", """id""", """text""", """title"""]		)
    self.assertEqual(len(doc_dicts[0]["""id"""]		)   ,  _SCREAMING_SNAKE_CASE		)
    self.assertEqual(doc_dicts[0]["""id"""][0]   ,  """1"""		)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]["""id"""][0]   ,  """0"""		)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()   ,  [[1], [0]]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[int]			=			self.get_dummy_custom_hf_index_retriever(from_disk=_SCREAMING_SNAKE_CASE		)
    with tempfile.TemporaryDirectory() as tmp_dirname:
     retriever.save_pretrained(_SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      List[Any]			=			RagRetriever.from_pretrained(_SCREAMING_SNAKE_CASE		)
     self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      List[str]			=			np.array(
         [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
     UpperCAmelCase			:      int			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=1		)
     self.assertTrue(out is not None		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[str]:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			1
    UpperCAmelCase			:      List[Any]			=			self.get_dummy_legacy_index_retriever()
    UpperCAmelCase			:      Any			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      Dict			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=_SCREAMING_SNAKE_CASE		)
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  2		)
    self.assertEqual(sorted(doc_dicts[0]		)   ,  ["""text""", """title"""]		)
    self.assertEqual(len(doc_dicts[0]["""text"""]		)   ,  _SCREAMING_SNAKE_CASE		)
    self.assertEqual(doc_dicts[0]["""text"""][0]   ,  """bar"""		)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]["""text"""][0]   ,  """foo"""		)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()   ,  [[1], [0]]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Tuple:





    '''simple docstring'''




    UpperCAmelCase			:      str			=			self.get_dummy_legacy_index_retriever()
    with tempfile.TemporaryDirectory() as tmp_dirname:
     retriever.save_pretrained(_SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      Any			=			RagRetriever.from_pretrained(_SCREAMING_SNAKE_CASE		)
     self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      Optional[Any]			=			np.array(
         [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
     UpperCAmelCase			:      Tuple			=			retriever.retrieve(_SCREAMING_SNAKE_CASE   ,  n_docs=1		)
     self.assertTrue(out is not None		)

   @require_torch
   @require_tokenizers
   @require_sentencepiece
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[str]:





    '''simple docstring'''




    import torch

    UpperCAmelCase			:      Dict			=			1
    UpperCAmelCase			:      List[Any]			=			self.get_dummy_canonical_hf_index_retriever()
    UpperCAmelCase			:      List[str]			=			[[5, 7], [10, 11]]
    UpperCAmelCase			:      List[Any]			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			:      Union[str, Any]			=			retriever(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  prefix=retriever.config.generator.prefix   ,  n_docs=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      Any			=			(
        out["""context_input_ids"""],
        out["""context_attention_mask"""],
        out["""retrieved_doc_embeds"""],
    )
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  np.ndarray		)

    UpperCAmelCase			:      Any			=			retriever(
        _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  prefix=retriever.config.generator.prefix   ,  n_docs=_SCREAMING_SNAKE_CASE   ,  return_tensors="""pt"""   ,  )
    UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			,	UpperCAmelCase			:      Dict			=			(  # noqa: F841
        out["""context_input_ids"""],
        out["""context_attention_mask"""],
        out["""retrieved_doc_embeds"""],
        out["""doc_ids"""],
    )
    self.assertEqual(retrieved_doc_embeds.shape   ,  (2, n_docs, self.retrieval_vector_size)		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  torch.Tensor		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  torch.Tensor		)
    self.assertIsInstance(_SCREAMING_SNAKE_CASE   ,  torch.Tensor		)




   @require_torch
   @require_tokenizers
   @require_sentencepiece
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      int			=			self.get_dpr_ctx_encoder_tokenizer()
    UpperCAmelCase			:      Any			=			1
    UpperCAmelCase			:      List[str]			=			self.get_dummy_custom_hf_index_retriever(from_disk=_SCREAMING_SNAKE_CASE		)
    retriever.set_ctx_encoder_tokenizer(_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Dict			=			[[5, 7], [10, 11]]
    UpperCAmelCase			:      Optional[Any]			=			np.array(
        [np.ones(self.retrieval_vector_size		), -np.ones(self.retrieval_vector_size		)]   ,  dtype=np.floataa		)
    UpperCAmelCase			:      Tuple			=			retriever(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  prefix=retriever.config.generator.prefix   ,  n_docs=_SCREAMING_SNAKE_CASE		)

    self.assertEqual(
        len(_SCREAMING_SNAKE_CASE		)   ,  6		)  # check whether the retriever output consist of 6 attributes including tokenized docs
    self.assertEqual(
        all(k in out for k in ("""tokenized_doc_ids""", """tokenized_doc_attention_mask""")		)   ,  _SCREAMING_SNAKE_CASE		)  # check for doc token related keys in dictionary.
