class_docstr: Generate decoder states by using a translation model to force-decode a parallel dataset.
  
  :param model: Sockeye translation model used to generate the states.
  :param source_vocabs: source vocabs for the translation model.
  :param target_vocabs: target vocabs for the translation model.
  :param output_dir: path to the memmap (directory) storing decoder states.
  :param max_seq_len_source: maximum source length for decoding.
  :param max_seq_len_target: maximum source length for decoding.
  :param state_data_type: data type for storing decoder states.
  :param word_data_type: data type for storing word indexes.
  :param device: device (cpu/gpu) for decoding.
class_docstr_tok: ["Generate","decoder","states","by","using","a","translation","model","to","force-decode","a","parallel","dataset",".",":","param","model",":","Sockeye","translation","model","used","to","generate","the","states",".",":","param","source_vocabs",":","source","vocabs","for","the","translation","model",".",":","param","target_vocabs",":","target","vocabs","for","the","translation","model",".",":","param","output_dir",":","path","to","the","memmap","(","directory",")","storing","decoder","states",".",":","param","max_seq_len_source",":","maximum","source","length","for","decoding",".",":","param","max_seq_len_target",":","maximum","source","length","for","decoding",".",":","param","state_data_type",":","data","type","for","storing","decoder","states",".",":","param","word_data_type",":","data","type","for","storing","word","indexes",".",":","param","device",":","device","(","cpu\/gpu",")","for","decoding","."]
docstr: Generate decoder states by force-decoding the sentence pairs in `sources` and `targets` with a NMT model.
  
  :param sources: list of source segments.
  :param targets: list of target segments.
  :param batch_size: number of sentence pairs to decode at once.
  :param eop_id: End-of-prepending tag id.
docstr_tok: ["Generate","decoder","states","by","force-decoding","the","sentence","pairs","in","`","sources","`","and","`","targets","`","with","a","NMT","model",".",":","param","sources",":","list","of","source","segments",".",":","param","targets",":","list","of","target","segments",".",":","param","batch_size",":","number","of","sentence","pairs","to","decode","at","once",".",":","param","eop_id",":","End-of-prepending","tag","id","."]
code: def generate_states_and_store(
      self,
      sources: List[str],
      targets: List[str],
      batch_size: int,
      eop_id: int = C.INVALID_ID,
  ) -> None:
      """
      Generate decoder states by force-decoding the sentence pairs in `sources` and `targets` with a NMT model.
  
      :param sources: list of source segments.
      :param targets: list of target segments.
      :param batch_size: number of sentence pairs to decode at once.
      :param eop_id: End-of-prepending tag id.
      """
      assert (
          self.state_store_file != None
      ), "You should call probe_token_count first to initialize the store files."
  
      # get data iter
      data_iter = data_io.get_scoring_data_iters(
          sources=sources,
          targets=targets,
          source_vocabs=self.source_vocabs,
          target_vocabs=self.target_vocabs,
          batch_size=batch_size,
          max_seq_len_source=self.max_seq_len_source,
          max_seq_len_target=self.max_seq_len_target,
          eop_id=eop_id,
      )
  
      with pt.inference_mode():
          for batch_no, batch in enumerate(data_iter, 1):
              if (batch_no + 1) % 1000 == 0:
                  logger.debug(
                      "At batch number {0}".format(batch_no + 1)
                  )
  
              # get decoder states
              batch = batch.load(self.device)
              model_inputs = (
                  batch.source,
                  batch.source_length,
                  batch.target,
                  batch.target_length,
              )
              if self.traced_model is None:
                  trace_inputs = {"get_decoder_states": model_inputs}
                  self.traced_model = pt.jit.trace_module(
                      self.model, trace_inputs, strict=False
                  )
              # shape: (batch, seq_len, hidden_dim)
              decoder_states = self.traced_model.get_decoder_states(*model_inputs)  # type: ignore
  
              # flatten batch and seq_len dimensions, remove pads on the target
              pad_mask = (batch.target != C.PAD_ID)[
                  :, :, 0
              ]  # shape: (batch, seq_len)
              flat_target = (
                  batch.target[pad_mask].cpu().detach().numpy()
              )
              flat_states = (
                  decoder_states[pad_mask].cpu().detach().numpy()
              )
  
              # store
              self.state_store_file.add(flat_states)
              self.words_store_file.add(flat_target)
  
imports: import argparse
  import logging
  import os
  from typing import List, Optional
  import numpy
  import torch
  from .None import arguments
  from .None import constants
  from .None import data_io
  from .None import utils
  from .log import setup_main_logger
  from .model import SockeyeModel, load_model
  from .vocab import Vocab
  from .utils import check_condition
  from .knn import KNNConfig, get_state_store_path, get_word_store_path, get_config_path
label_desc: Use image node_id 4 for calling the DecoderStateGenerator obj's underlying member method code with example usage: obj.generate_states_and_store(sources, targets, batch_size, eop_id) without return types