code: def parse_token_stream(stream, soft_delimiter, hard_delimiter):
      """Parses a stream of tokens and splits it into sentences (using C{soft_delimiter} tokens)
      and blocks (using C{hard_delimiter} tokens) for use with the L{align_texts} function.
      """
      return [
          [
              sum(len(token) for token in sentence_it)
              for sentence_it in split_at(block_it, soft_delimiter)
          ]
          for block_it in split_at(stream, hard_delimiter)
      ]
  
file_path: nltk/nltk/translate/gale_church.py
imports: import math
label_desc: Use image node_id 6 for calling a global function with example usage: parse_token_stream(stream, soft_delimiter, hard_delimiter) and returns: unknown