class_docstr: Generate decoder states by using a translation model to force-decode a parallel dataset.
  
  :param model: Sockeye translation model used to generate the states.
  :param source_vocabs: source vocabs for the translation model.
  :param target_vocabs: target vocabs for the translation model.
  :param output_dir: path to the memmap (directory) storing decoder states.
  :param max_seq_len_source: maximum source length for decoding.
  :param max_seq_len_target: maximum source length for decoding.
  :param state_data_type: data type for storing decoder states.
  :param word_data_type: data type for storing word indexes.
  :param device: device (cpu/gpu) for decoding.
class_docstr_tok: ["Generate","decoder","states","by","using","a","translation","model","to","force-decode","a","parallel","dataset",".",":","param","model",":","Sockeye","translation","model","used","to","generate","the","states",".",":","param","source_vocabs",":","source","vocabs","for","the","translation","model",".",":","param","target_vocabs",":","target","vocabs","for","the","translation","model",".",":","param","output_dir",":","path","to","the","memmap","(","directory",")","storing","decoder","states",".",":","param","max_seq_len_source",":","maximum","source","length","for","decoding",".",":","param","max_seq_len_target",":","maximum","source","length","for","decoding",".",":","param","state_data_type",":","data","type","for","storing","decoder","states",".",":","param","word_data_type",":","data","type","for","storing","word","indexes",".",":","param","device",":","device","(","cpu\/gpu",")","for","decoding","."]
docstr: Count the number of tokens in the file at `target_path`, with each line truncated at `max_seq_len`.
docstr_tok: ["Count","the","number","of","tokens","in","the","file","at","`","target_path","`",",","with","each","line","truncated","at","`","max_seq_len","`","."]
code: def probe_token_count(target_path: str, max_seq_len: int) -> int:
      """Count the number of tokens in the file at `target_path`, with each line truncated at `max_seq_len`."""
      token_count = 0
      with open(target_path, "r") as f:
          for line in f:
              token_count += min(
                  len(line.split()) + 1, max_seq_len
              )  # +1 for EOS
      return token_count
  
imports: import argparse
  import logging
  import os
  from typing import List, Optional
  import numpy
  import torch
  from .None import arguments
  from .None import constants
  from .None import data_io
  from .None import utils
  from .log import setup_main_logger
  from .model import SockeyeModel, load_model
  from .vocab import Vocab
  from .utils import check_condition
  from .knn import KNNConfig, get_state_store_path, get_word_store_path, get_config_path
label_desc: Use image node_id 2 for calling the DecoderStateGenerator obj's underlying member method code with example usage: obj.probe_token_count(target_path, max_seq_len) and returns: token_count