docstr: :param src_sentence:
  :param tgt_generated:
  :return:
      List[str], valid token list
docstr_tok: [":","param","src_sentence",":",":","param","tgt_generated",":",":","return",":","List","[","str","]",",","valid","token","list"]
code: def get_state_valid_tokens(self, src_sentence, tgt_generated):
      """
  
      :param src_sentence:
      :param tgt_generated:
      :return:
          List[str], valid token list
      """
      if self.tokenizer.eos_token_id in src_sentence:
          src_sentence = src_sentence[
              : src_sentence.index(self.tokenizer.eos_token_id)
          ]
  
      if self.text_start in src_sentence:
          src_sentence = src_sentence[
              src_sentence.index(self.text_start) + 1 :
          ]
  
      state, index = self.check_state(tgt_generated)
  
      print("State: %s" % state) if debug else None
  
      if state == "error":
          print("Decode Error:")
          print(
              "Src:", self.tokenizer.convert_ids_to_tokens(src_sentence)
          )
          print(
              "Tgt:",
              self.tokenizer.convert_ids_to_tokens(tgt_generated),
          )
          valid_tokens = [self.tokenizer.eos_token_id]
  
      elif state == "start":
          valid_tokens = [self.type_start]
  
      elif state == "start_first_generation":
          valid_tokens = [self.type_start, self.type_end]
  
      elif state == "generate_trigger":
          if tgt_generated[-1] == self.type_start:
              # Start Event Label
              return list(self.type_tree.keys())
  
          elif tgt_generated[-1] == self.type_end:
              # EVENT_TYPE_LEFT: Start a new role
              # EVENT_TYPE_RIGHT: End this event
              return [self.type_start, self.type_end]
          else:
              valid_tokens = self.search_prefix_tree(
                  generated=tgt_generated[index + 1 :],
                  prefix_tree=self.type_tree,
                  end_search_tokens=[self.span_start],
              )
  
      elif state in {"generate_trigger_text"}:
          generated = tgt_generated[index + 1 :]
  
          if len(generated) > 0 and generated[-1] == self.null_span:
              return [self.type_end, self.type_start]
  
          valid_tokens = generated_search_src_sequence(
              generated=generated,
              src_sequence=src_sentence + [self.null_span],
              end_sequence_search_tokens=[
                  self.type_end,
                  self.type_start,
              ],
          )
  
      elif state in {"generate_role_text"}:
          generated = tgt_generated[index + 1 :]
  
          if len(generated) > 0 and generated[-1] == self.null_span:
              return [self.type_end]
  
          valid_tokens = generated_search_src_sequence(
              generated=generated,
              src_sequence=src_sentence + [self.null_span],
              end_sequence_search_tokens=[self.type_end],
          )
  
      elif state == "generate_role":
          if tgt_generated[-1] == self.type_start:
              # Start Role Label
              return list(self.role_tree.keys())
  
          generated = tgt_generated[index + 1 :]
          valid_tokens = self.search_prefix_tree(
              generated=generated,
              prefix_tree=self.role_tree,
              end_search_tokens=[self.span_start],
          )
  
      elif state == "end_generate":
          valid_tokens = [self.tokenizer.eos_token_id]
  
      else:
          raise NotImplementedError(
              "State `%s` for %s is not implemented."
              % (state, self.__class__)
          )
  
      print(
          "Valid: %s"
          % self.tokenizer.convert_ids_to_tokens(valid_tokens)
      ) if debug else None
      return valid_tokens
  
file_path: DeepKE/src/deepke/name_entity_re/cross/seq2seq/constraint_decoder/spotasoc_constraint_decoder.py
imports: import os
  from typing import List, Dict
  from ...extraction.label_tree import get_label_name_tree
  from ...extraction.constants import span_start, type_start, type_end, null_span, text_start
  from ...seq2seq.constraint_decoder.constraint_decoder import ConstraintDecoder, find_bracket_position, generated_search_src_sequence
label_desc: Use image node_id 4 for calling the SpotAsocConstraintDecoder obj's underlying member method code with example usage: obj.get_state_valid_tokens(src_sentence, tgt_generated) and returns: valid_tokens, list, list, list, list, list