class_docstr: This base class defines abstract methods for manipulating a State.
  
  Applying transitions may change important metadata about a State
  such as the vectors associated with LSTM hidden states, for example.
  
  The constructor forwards all unused arguments to other classes in the
  constructor sequence, so put this before other classes such as nn.Module
class_docstr_tok: ["This","base","class","defines","abstract","methods","for","manipulating","a","State",".","Applying","transitions","may","change","important","metadata","about","a","State","such","as","the","vectors","associated","with","LSTM","hidden","states",",","for","example",".","The","constructor","forwards","all","unused","arguments","to","other","classes","in","the","constructor","sequence",",","so","put","this","before","other","classes","such","as","nn.Module"]
docstr: Repeat transitions to build a list of trees from the input batches.
  
  The data_iterator should be anything which returns the data for a parse task via next()
  build_batch_fn is a function that turns that data into State objects
  This will be called to generate batches of size batch_size until the data is exhausted
  
  The return is a list of tuples: (gold_tree, [(predicted, score) ...])
  gold_tree will be left blank if the data did not include gold trees
  if keep_scores is true, the score will be the sum of the values
    returned by the model for each transition
  
  transition_choice: which method of the model to use for choosing the next transition
    predict for predicting the transition based on the model
    predict_gold to just extract the gold transition from the sequence
docstr_tok: ["Repeat","transitions","to","build","a","list","of","trees","from","the","input","batches",".","The","data_iterator","should","be","anything","which","returns","the","data","for","a","parse","task","via","next","(",")","build_batch_fn","is","a","function","that","turns","that","data","into","State","objects","This","will","be","called","to","generate","batches","of","size","batch_size","until","the","data","is","exhausted","The","return","is","a","list","of","tuples",":","(","gold_tree",",","[","(","predicted",",","score",")","...","]",")","gold_tree","will","be","left","blank","if","the","data","did","not","include","gold","trees","if","keep_scores","is","true",",","the","score","will","be","the","sum","of","the","values","returned","by","the","model","for","each","transition","transition_choice",":","which","method","of","the","model","to","use","for","choosing","the","next","transition","predict","for","predicting","the","transition","based","on","the","model","predict_gold","to","just","extract","the","gold","transition","from","the","sequence"]
code: def parse_sentences(
      self,
      data_iterator,
      build_batch_fn,
      batch_size,
      transition_choice,
      keep_state=False,
      keep_constituents=False,
      keep_scores=False,
  ):
      """
      Repeat transitions to build a list of trees from the input batches.
  
      The data_iterator should be anything which returns the data for a parse task via next()
      build_batch_fn is a function that turns that data into State objects
      This will be called to generate batches of size batch_size until the data is exhausted
  
      The return is a list of tuples: (gold_tree, [(predicted, score) ...])
      gold_tree will be left blank if the data did not include gold trees
      if keep_scores is true, the score will be the sum of the values
        returned by the model for each transition
  
      transition_choice: which method of the model to use for choosing the next transition
        predict for predicting the transition based on the model
        predict_gold to just extract the gold transition from the sequence
      """
      treebank = []
      treebank_indices = []
      state_batch = build_batch_fn(batch_size, data_iterator)
      # used to track which indices we are currently parsing
      # since the parses get finished at different times, this will let us unsort after
      batch_indices = list(range(len(state_batch)))
      horizon_iterator = iter([])
  
      if keep_constituents:
          constituents = defaultdict(list)
  
      while len(state_batch) > 0:
          pred_scores, transitions, scores = transition_choice(
              state_batch
          )
          if keep_scores and scores is not None:
              state_batch = [
                  state._replace(score=state.score + score)
                  for state, score in zip(state_batch, scores)
              ]
          state_batch = parse_transitions.bulk_apply(
              self, state_batch, transitions
          )
  
          if keep_constituents:
              for t_idx, transition in enumerate(transitions):
                  if isinstance(transition, CloseConstituent):
                      # constituents is a TreeStack with information on how to build the next state of the LSTM or attn
                      # constituents.value is the TreeStack node
                      # constituents.value.value is the Constituent itself (with the tree and the embedding)
                      constituents[batch_indices[t_idx]].append(
                          state_batch[t_idx].constituents.value.value
                      )
  
          remove = set()
          for idx, state in enumerate(state_batch):
              if state.finished(self):
                  predicted_tree = state.get_tree(self)
                  if self.reverse_sentence():
                      predicted_tree = predicted_tree.reverse()
                  gold_tree = state.gold_tree
                  treebank.append(
                      ParseResult(
                          gold_tree,
                          [ScoredTree(predicted_tree, state.score)],
                          state if keep_state else None,
                          constituents[batch_indices[idx]]
                          if keep_constituents
                          else None,
                      )
                  )
                  treebank_indices.append(batch_indices[idx])
                  remove.add(idx)
  
          if len(remove) > 0:
              state_batch = [
                  state
                  for idx, state in enumerate(state_batch)
                  if idx not in remove
              ]
              batch_indices = [
                  batch_idx
                  for idx, batch_idx in enumerate(batch_indices)
                  if idx not in remove
              ]
  
          for _ in range(batch_size - len(state_batch)):
              horizon_state = next(horizon_iterator, None)
              if not horizon_state:
                  horizon_batch = build_batch_fn(
                      batch_size, data_iterator
                  )
                  if len(horizon_batch) == 0:
                      break
                  horizon_iterator = iter(horizon_batch)
                  horizon_state = next(horizon_iterator, None)
  
              state_batch.append(horizon_state)
              batch_indices.append(len(treebank) + len(state_batch))
  
      treebank = utils.unsort(treebank, treebank_indices)
      return treebank
  
imports: from abc import ABC, abstractmethod
  from collections import defaultdict
  import logging
  import torch
  from stanza.models.common import utils
  from stanza.models.constituency import parse_transitions
  from stanza.models.constituency import transition_sequence
  from stanza.models.constituency.parse_transitions import State, TransitionScheme, CloseConstituent
  from stanza.models.constituency.parse_tree import Tree
  from stanza.models.constituency.tree_stack import TreeStack
  from stanza.server.parser_eval import ParseResult, ScoredTree
label_desc: Use image node_id 28 for calling the BaseModel obj's underlying member method code with example usage: obj.parse_sentences(data_iterator, build_batch_fn, batch_size, transition_choice, keep_state, keep_constituents, keep_scores) and returns: treebank