
import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFEfficientFormerForImageClassification,
         TFEfficientFormerForImageClassificationWithTeacher,
         TFEfficientFormerModel,
     )
     from transformers.models.efficientformer.modeling_tf_efficientformer import (
         TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )


if is_vision_available():
     from PIL import Image

     from transformers import EfficientFormerImageProcessor




class        _SCREAMING_SNAKE_CASE	:
      def __init__(      self		,						lowerCamelCase		,						lowerCamelCase = 13		,						lowerCamelCase = 64		,						lowerCamelCase = 2		,						lowerCamelCase = 3		,						lowerCamelCase = 3		,						lowerCamelCase = True		,						lowerCamelCase = True		,						lowerCamelCase = 1_28		,						lowerCamelCase=[16, 32, 64, 1_28]		,						lowerCamelCase = 7		,						lowerCamelCase = 4		,						lowerCamelCase = 37		,						lowerCamelCase = "gelu"		,						lowerCamelCase = 0.1		,						lowerCamelCase = 0.1		,						lowerCamelCase = 10		,						lowerCamelCase = 0.0_2		,						lowerCamelCase = 2		,						lowerCamelCase = 1		,						lowerCamelCase = 1_28		,						lowerCamelCase = [2, 2, 2, 2]		,						lowerCamelCase = 2		,						lowerCamelCase = 2		,						):
          snake_case__    =					parent
          snake_case__    =					batch_size
          snake_case__    =					image_size
          snake_case__    =					patch_size
          snake_case__    =					num_channels
          snake_case__    =					is_training
          snake_case__    =					use_labels
          snake_case__    =					hidden_size
          snake_case__    =					num_hidden_layers
          snake_case__    =					num_attention_heads
          snake_case__    =					intermediate_size
          snake_case__    =					hidden_act
          snake_case__    =					hidden_dropout_prob
          snake_case__    =					attention_probs_dropout_prob
          snake_case__    =					type_sequence_label_size
          snake_case__    =					initializer_range
          snake_case__    =					encoder_stride
          snake_case__    =					num_attention_outputs
          snake_case__    =					embed_dim
          snake_case__    =					embed_dim + 1
          snake_case__    =					resolution
          snake_case__    =					depths
          snake_case__    =					hidden_sizes
          snake_case__    =					dim
          snake_case__    =					mlp_expansion_ratio
      def A_      (      self			):
          snake_case__    =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

          snake_case__    =					None
          if self.use_labels:
              snake_case__    =					ids_tensor([self.batch_size]		,						self.type_sequence_label_size			)

          snake_case__    =					self.get_config()

          return config, pixel_values, labels
      def A_      (      self			):
          return EfficientFormerConfig(
              image_size=self.image_size		,						patch_size=self.patch_size		,						num_channels=self.num_channels		,						hidden_size=self.hidden_size		,						num_hidden_layers=self.num_hidden_layers		,						num_attention_heads=self.num_attention_heads		,						intermediate_size=self.intermediate_size		,						hidden_act=self.hidden_act		,						hidden_dropout_prob=self.hidden_dropout_prob		,						attention_probs_dropout_prob=self.attention_probs_dropout_prob		,						is_decoder=lowerCamelCase		,						initializer_range=self.initializer_range		,						encoder_stride=self.encoder_stride		,						resolution=self.resolution		,						depths=self.depths		,						hidden_sizes=self.hidden_sizes		,						dim=self.dim		,						mlp_expansion_ratio=self.mlp_expansion_ratio		,						)
      def A_      (      self		,						lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase			):
          snake_case__    =					TFEfficientFormerModel(config=lowerCamelCase			)
          snake_case__    =					model(lowerCamelCase		,						training=lowerCamelCase			)
          self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)			)
      def A_      (      self		,						lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase			):
          snake_case__    =					self.type_sequence_label_size
          snake_case__    =					TFEfficientFormerForImageClassification(lowerCamelCase			)
          snake_case__    =					model(lowerCamelCase		,						labels=lowerCamelCase		,						training=lowerCamelCase			)
          self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.type_sequence_label_size)			)

          # test greyscale images
          snake_case__    =					1
          snake_case__    =					TFEfficientFormerForImageClassification(lowerCamelCase			)

          snake_case__    =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]			)
          snake_case__    =					model(lowerCamelCase		,						labels=lowerCamelCase			)
          self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.type_sequence_label_size)			)



      def A_      (      self			):
          snake_case__    =					self.prepare_config_and_inputs()
          snake_case__    ,       snake_case__    ,       snake_case__    =					config_and_inputs
          snake_case__    =					{"pixel_values": pixel_values}
          return config, inputs_dict





@require_tf
class        _SCREAMING_SNAKE_CASE	(   __UpperCamelCase							,       __UpperCamelCase							,       unittest.TestCase			):
      _A    :		str     						=  (
          (
              TFEfficientFormerModel,
              TFEfficientFormerForImageClassificationWithTeacher,
              TFEfficientFormerForImageClassification,
          )
          if is_tf_available()
          else ()
      )
      _A    :		List[str]     						=  (
          {
              'feature-extraction': TFEfficientFormerModel,
              'image-classification': (
                  TFEfficientFormerForImageClassification,
                  TFEfficientFormerForImageClassificationWithTeacher,
              ),
          }
          if is_tf_available()
          else {}
      )

      _A    :		Optional[Any]     						=  False

      _A    :		List[Any]     						=  False
      _A    :		Tuple     						=  False
      _A    :		List[Any]     						=  False
      _A    :		Any     						=  False
      def A_      (      self			):
          snake_case__    =					TFEfficientFormerModelTester(self			)
          snake_case__    =					ConfigTester(
              self		,						config_class=lowerCamelCase		,						has_text_modality=lowerCamelCase		,						hidden_size=37			)
      def A_      (      self			):
          self.config_tester.run_common_tests()
      @unittest.skip(reason="EfficientFormer does not use inputs_embeds"			)
      def A_      (      self			):
          pass
      @unittest.skip(reason="EfficientFormer does not support input and output embeddings"			)
      def A_      (      self			):
          pass
      def A_      (      self			):
          snake_case__    ,       snake_case__    =					self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              snake_case__    =					model_class(lowerCamelCase			)
              snake_case__    =					inspect.signature(model.call			)
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              snake_case__    =					[*signature.parameters.keys()]

              snake_case__    =					["pixel_values"]
              self.assertListEqual(arg_names[:1]		,						lowerCamelCase			)
      def A_      (      self			):
          def check_hidden_states_output(lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase			):
              snake_case__    =					model_class(lowerCamelCase			)

              snake_case__    =					model(**self._prepare_for_class(lowerCamelCase		,						lowerCamelCase			)		,						training=lowerCamelCase			)
              snake_case__    =					outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

              snake_case__    =					getattr(
                  self.model_tester		,						"expected_num_hidden_layers"		,						self.model_tester.num_hidden_layers + 1			)
              self.assertEqual(len(lowerCamelCase			)		,						lowerCamelCase			)

              if hasattr(self.model_tester		,						"encoder_seq_length"			):
                  snake_case__    =					self.model_tester.encoder_seq_length
                  if hasattr(self.model_tester		,						"chunk_length"			) and self.model_tester.chunk_length > 1:
                      snake_case__    =					seq_length * self.model_tester.chunk_length
              else:
                  snake_case__    =					self.model_tester.seq_length

              self.assertListEqual(
                  list(hidden_states[-1].shape[-2:]			)		,						[seq_length, self.model_tester.hidden_size]		,						)

              if config.is_encoder_decoder:
                  snake_case__    =					outputs.decoder_hidden_states

                  self.asseretIsInstance(lowerCamelCase		,						(list, tuple)			)
                  self.assertEqual(len(lowerCamelCase			)		,						lowerCamelCase			)
                  snake_case__    =					getattr(self.model_tester		,						"seq_length"		,						lowerCamelCase			)
                  snake_case__    =					getattr(self.model_tester		,						"decoder_seq_length"		,						lowerCamelCase			)

                  self.assertListEqual(
                      list(hidden_states[-1].shape[-2:]			)		,						[decoder_seq_length, self.model_tester.hidden_size]		,						)

          snake_case__    ,       snake_case__    =					self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              snake_case__    =					True
              check_hidden_states_output(lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase			)

              # check that output_hidden_states also work using config
              del inputs_dict["output_hidden_states"]
              snake_case__    =					True

              check_hidden_states_output(lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase			)
      def A_      (      self		,						lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase=False			):
          snake_case__    =					super()._prepare_for_class(lowerCamelCase		,						lowerCamelCase		,						return_labels=lowerCamelCase			)

          if return_labels:
              if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
                  del inputs_dict["labels"]

          return inputs_dict
      def A_      (      self			):
          snake_case__    =					self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowerCamelCase			)
      @unittest.skip(reason="EfficientFormer does not implement masked image modeling yet"			)
      def A_      (      self			):
          snake_case__    =					self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_image_modeling(*lowerCamelCase			)
      def A_      (      self			):
          snake_case__    =					self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_image_classification(*lowerCamelCase			)
      @slow
      def A_      (      self			):
          for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              snake_case__    =					TFEfficientFormerModel.from_pretrained(lowerCamelCase			)
              self.assertIsNotNone(lowerCamelCase			)
      def A_      (      self			):
          snake_case__    ,       snake_case__    =					self.model_tester.prepare_config_and_inputs_for_common()
          snake_case__    =					True

          snake_case__    =					getattr(self.model_tester		,						"seq_length"		,						lowerCamelCase			)
          snake_case__    =					getattr(self.model_tester		,						"encoder_seq_length"		,						lowerCamelCase			)
          snake_case__    =					getattr(self.model_tester		,						"key_length"		,						lowerCamelCase			)
          snake_case__    =					getattr(self.model_tester		,						"chunk_length"		,						lowerCamelCase			)

          if chunk_length is not None and hasattr(self.model_tester		,						"num_hashes"			):
              snake_case__    =					encoder_seq_length * self.model_tester.num_hashes

          for model_class in self.all_model_classes:
              snake_case__    =					True
              snake_case__    =					False
              snake_case__    =					True
              snake_case__    =					model_class(lowerCamelCase			)

              snake_case__    =					model(**self._prepare_for_class(lowerCamelCase		,						lowerCamelCase			)		,						training=lowerCamelCase			)
              snake_case__    =					outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
              self.assertEqual(len(lowerCamelCase			)		,						self.model_tester.num_attention_outputs			)

              # check that output_attentions also work using config
              del inputs_dict["output_attentions"]
              snake_case__    =					True
              snake_case__    =					model_class(lowerCamelCase			)
              snake_case__    =					model(**self._prepare_for_class(lowerCamelCase		,						lowerCamelCase			)		,						training=lowerCamelCase			)

              snake_case__    =					outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
              self.assertEqual(len(lowerCamelCase			)		,						self.model_tester.num_attention_outputs			)

              if chunk_length is not None:
                  self.assertListEqual(
                      list(attentions[0].shape[-4:]			)		,						[self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]		,						)
              else:
                  self.assertListEqual(
                      list(attentions[0].shape[-3:]			)		,						[self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]		,						)



      def A_      (      self			):
          # We use a simplified version of this test for EfficientFormer because it requires training=False
          # and Keras refuses to let us force that during functional construction
          snake_case__    ,       snake_case__    =					self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              # Prepare our model
              snake_case__    =					model_class(lowerCamelCase			)
              # These are maximally general inputs for the model, with multiple None dimensions
              # Hopefully this will catch any conditionals that fail for flexible shapes
              snake_case__    =					{
                  key: tf.keras.Input(shape=val.shape[1:]		,						dtype=val.dtype		,						name=lowerCamelCase			)
                  for key, val in model.input_signature.items()
                  if key in model.dummy_inputs
              }
              snake_case__    =					model(lowerCamelCase			)
              self.assertTrue(outputs_dict is not None			)





def 						SCREAMING_SNAKE_CASE__     (   ):
    snake_case__    =					Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"				)
    return image





@require_tf
@require_vision
class        _SCREAMING_SNAKE_CASE	(   unittest.TestCase			):
      @cached_property
      def A_      (      self			):
          return (
              EfficientFormerImageProcessor.from_pretrained("snap-research/efficientformer-l1-300"			)
              if is_vision_available()
              else None
          )
      @slow
      def A_      (      self			):
          snake_case__    =					TFEfficientFormerForImageClassification.from_pretrained("snap-research/efficientformer-l1-300"			)
          snake_case__    =					self.default_image_processor
          snake_case__    =					prepare_img()
          snake_case__    =					image_processor(images=lowerCamelCase		,						return_tensors="tf"			)
          # forward pass
          snake_case__    =					model(**lowerCamelCase		,						training=lowerCamelCase			)
          # verify the logits
          snake_case__    =					tf.TensorShape((1, 10_00)			)
          self.assertEqual(outputs.logits.shape		,						lowerCamelCase			)
          snake_case__    =					tf.constant([-0.0_5_5_5, 0.4_8_2_5, -0.0_8_5_2]			)
          self.assertTrue(np.allclose(outputs.logits[0, :3]		,						lowerCamelCase		,						atol=1e-4			)			)



      @slow
      def A_      (      self			):
          snake_case__    =					TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
              "snap-research/efficientformer-l1-300"			)
          snake_case__    =					self.default_image_processor
          snake_case__    =					prepare_img()
          snake_case__    =					image_processor(images=lowerCamelCase		,						return_tensors="tf"			)
          # forward pass
          snake_case__    =					model(**lowerCamelCase		,						training=lowerCamelCase			)
          # verify the logits
          snake_case__    =					tf.TensorShape((1, 10_00)			)
          self.assertEqual(outputs.logits.shape		,						lowerCamelCase			)
          snake_case__    =					tf.constant([-0.1_3_1_2, 0.4_3_5_3, -1.0_4_9_9]			)
          self.assertTrue(np.allclose(outputs.logits[0, :3]		,						lowerCamelCase		,						atol=1e-4			)			)





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__  							=	logging.get_logger(__name__)

__magic_name__  							=	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__magic_name__  							=	{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__magic_name__  							=	{
    '''ctrl''': 256,
}

__magic_name__  							=	{
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}





def 						SCREAMING_SNAKE_CASE__     (   __lowerCAmelCase				):
    snake_case__    =					set()
    snake_case__    =					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)				)
        snake_case__    =					char

    snake_case__    =					set(__lowerCAmelCase				)
    return pairs





class        _SCREAMING_SNAKE_CASE	(   __UpperCamelCase			):
      _A    :		Tuple     						=  VOCAB_FILES_NAMES
      _A    :		str     						=  PRETRAINED_VOCAB_FILES_MAP
      _A    :		int     						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A    :		List[Any]     						=  CONTROL_CODES
      def __init__(      self		,						lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase="<unk>"		,						**lowerCamelCase			):
          super().__init__(unk_token=lowerCamelCase		,						**lowerCamelCase			)

          with open(lowerCamelCase		,						encoding="utf-8"			) as vocab_handle:
              snake_case__    =					json.load(lowerCamelCase			)
          snake_case__    =					{v: k for k, v in self.encoder.items()}
          with open(lowerCamelCase		,						encoding="utf-8"			) as merges_handle:
              snake_case__    =					merges_handle.read().split("\n"			)[1:-1]
          snake_case__    =					[tuple(merge.split()			) for merge in merges]
          snake_case__    =					dict(zip(lowerCamelCase		,						range(len(lowerCamelCase			)			)			)			)
          snake_case__    =					{}
      @property
      def A_      (      self			):
          return len(self.encoder			)
      def A_      (      self			):
          return dict(self.encoder		,						**self.added_tokens_encoder			)
      def A_      (      self		,						lowerCamelCase			):
          if token in self.cache:
              return self.cache[token]
          snake_case__    =					tuple(lowerCamelCase			)
          snake_case__    =					tuple(list(word[:-1]			) + [word[-1] + "</w>"]			)
          snake_case__    =					get_pairs(lowerCamelCase			)

          if not pairs:
              return token

          while True:
              snake_case__    =					min(lowerCamelCase		,						key=lambda lowerCamelCase			: self.bpe_ranks.get(lowerCamelCase		,						float("inf"			)			)			)
              if bigram not in self.bpe_ranks:
                  break
              snake_case__    ,       snake_case__    =					bigram
              snake_case__    =					[]
              snake_case__    =					0
              while i < len(lowerCamelCase			):
                  try:
                      snake_case__    =					word.index(lowerCamelCase		,						lowerCamelCase			)
                  except ValueError:
                      new_word.extend(word[i:]			)
                      break
                  else:
                      new_word.extend(word[i:j]			)
                      snake_case__    =					j

                  if word[i] == first and i < len(lowerCamelCase			) - 1 and word[i + 1] == second:
                      new_word.append(first + second			)
                      i += 2
                  else:
                      new_word.append(word[i]			)
                      i += 1
              snake_case__    =					tuple(lowerCamelCase			)
              snake_case__    =					new_word
              if len(lowerCamelCase			) == 1:
                  break
              else:
                  snake_case__    =					get_pairs(lowerCamelCase			)
          snake_case__    =					"@@ ".join(lowerCamelCase			)
          snake_case__    =					word[:-4]
          snake_case__    =					word
          return word
      def A_      (      self		,						lowerCamelCase			):
          snake_case__    =					[]

          snake_case__    =					re.findall(r"\S+\n?"		,						lowerCamelCase			)

          for token in words:
              split_tokens.extend(list(self.bpe(lowerCamelCase			).split(" "			)			)			)
          return split_tokens
      def A_      (      self		,						lowerCamelCase			):
          return self.encoder.get(lowerCamelCase		,						self.encoder.get(self.unk_token			)			)
      def A_      (      self		,						lowerCamelCase			):
          return self.decoder.get(lowerCamelCase		,						self.unk_token			)
      def A_      (      self		,						lowerCamelCase			):
          snake_case__    =					" ".join(lowerCamelCase			).replace("@@ "		,						""			).strip()
          return out_string



      def A_      (      self		,						lowerCamelCase		,						lowerCamelCase = None			):
          if not os.path.isdir(lowerCamelCase			):
              logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
              return
          snake_case__    =					os.path.join(
              lowerCamelCase		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
          snake_case__    =					os.path.join(
              lowerCamelCase		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

          with open(lowerCamelCase		,						"w"		,						encoding="utf-8"			) as f:
              f.write(json.dumps(self.encoder		,						indent=2		,						sort_keys=lowerCamelCase		,						ensure_ascii=lowerCamelCase			) + "\n"			)

          snake_case__    =					0
          with open(lowerCamelCase		,						"w"		,						encoding="utf-8"			) as writer:
              writer.write("#version: 0.2\n"			)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,						key=lambda lowerCamelCase			: kv[1]			):
                  if index != token_index:
                      logger.warning(
                          F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                          " Please check that the tokenizer is not corrupted!"			)
                      snake_case__    =					token_index
                  writer.write(" ".join(lowerCamelCase			) + "\n"			)
                  index += 1

          return vocab_file, merge_file

      # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
      #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
      #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
      #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
      #     return ''.join(tokens_generated_so_far)



