class_docstr: Base class for TextGeneration Strategy
class_docstr_tok: ["Base","class","for","TextGeneration","Strategy"]
docstr: return the tokens and strings indicating the end of generation
  Args:
      eod_id (int): the end of document token id
      end_strings (List[str]): the list of end of generation strings
  Returns:
      a pair `(tokens, strings)` where `tokens` is a set of tokens (int) and `strings` is a list of strings,
      which must all be used to identify the end of generation (`tokens` always contains `eod_id`, while
      `strings` may be empty if all end strings are associated to unique tokens)
code: def _get_end_of_generation_tokens_and_strings(
      self, eod_id: int, end_strings: List[str]
  ) -> Tuple[Set[int], List[str]]:
      """
      return the tokens and strings indicating the end of generation
      Args:
          eod_id (int): the end of document token id
          end_strings (List[str]): the list of end of generation strings
      Returns:
          a pair `(tokens, strings)` where `tokens` is a set of tokens (int) and `strings` is a list of strings,
          which must all be used to identify the end of generation (`tokens` always contains `eod_id`, while
          `strings` may be empty if all end strings are associated to unique tokens)
      """
      tokenizer = self.model.tokenizer
      # A cache is used to remember which end strings are associated to unique tokens vs. which ones
      # require an actual string comparison.
      if (
          self._end_of_generation_cache is None
          or self._end_of_generation_cache["tokenizer"] is not tokenizer
      ):
          # Invalidate the cache.
          self._end_of_generation_cache = {
              "tokenizer": tokenizer,
              "end_string_to_token": {END_OF_SEQ: eod_id},
              "end_strings_to_check": set(),
          }
      end_string_to_token = self._end_of_generation_cache[
          "end_string_to_token"
      ]
  
      end_tokens = {
          eod_id
      }  # always include `eod_id`, even if `END_OF_SEQ` is not within `end_strings`
      end_strings_to_check = (
          []
      )  # will contain end strings that have no associated special token
  
      for end_string in end_strings:
          try:
              end_tokens.add(end_string_to_token[end_string])
              continue
          except KeyError:
              if (
                  end_string
                  in self._end_of_generation_cache[
                      "end_strings_to_check"
                  ]
              ):
                  end_strings_to_check.append(end_string)
                  continue
  
          # `end_string` does not exist in the cache yet: check if `end_string` is a special token for
          # the tokenizer. Ideally, we would simply use `tokenizer.text_to_ids(end_string)`, but some
          # tokenizers (e.g., SentencePiece) may prefix the special token with another token associated
          # to an empty string. The code below is thus meant to extract the special token associated to
          # `end_string` (if it exists). Note that we use "<extra_id_1>" as prefix string to have a low
          # risk of the tokenizer merging it with `end_string`, but this is somewhat arbitrary.
          ids_ref = tokenizer.text_to_ids("<extra_id_1>")
          ids_with_end_string = tokenizer.text_to_ids(
              f"<extra_id_1>{end_string}"
          )
          if (
              len(ids_with_end_string) == len(ids_ref) + 1
              and ids_with_end_string[:-1] == ids_ref
          ):
              # We can assume that the extra token is the one corresponding to `end_string`.
              end_string_to_token[end_string] = ids_with_end_string[-1]
              end_tokens.add(ids_with_end_string[-1])
          else:
              # No special token.
              warnings.warn(
                  f"The end string '{end_string}' has no associated special token: this may slow down "
                  "generation (consider using a different tokenizer or modifying `end_strings`)"
              )
              self._end_of_generation_cache["end_strings_to_check"].add(
                  end_string
              )
              end_strings_to_check.append(end_string)
  
      return end_tokens, end_strings_to_check
  
imports: import abc
  import copy
  import os
  import re
  import warnings
  from typing import List, Set, Tuple
  import torch
  from transformers import CLIPImageProcessor
  from nemo.collections.nlp.modules.common.lm_utils import pad_batch
  from nemo.collections.nlp.modules.common.megatron.utils import get_ltor_masks_and_position_ids
label_desc: Use image node_id 10 for calling the TextGenerationStrategy obj's underlying member method code with example usage: obj._get_end_of_generation_tokens_and_strings(eod_id, end_strings) and returns: end_tokens,  end_strings_to_check