code: def augment_punct(
      train_data,
      augment_ratio,
      should_augment_predicate=should_augment_nopunct_predicate,
      can_augment_predicate=can_augment_nopunct_predicate,
      keep_original_sentences=True,
  ):
      """
      Adds extra training data to compensate for some models having all sentences end with PUNCT
  
      Some of the models (for example, UD_Hebrew-HTB) have the flaw that
      all of the training sentences end with PUNCT.  The model therefore
      learns to finish every sentence with punctuation, even if it is
      given a sentence with non-punct at the end.
  
      One simple way to fix this is to train on some fraction of training data with punct.
  
      Params:
      train_data: list of list of dicts, eg a conll doc
      augment_ratio: the fraction to augment.  if None, a best guess is made to get to 10%
  
      should_augment_predicate: a function which returns T/F if a sentence already ends with not PUNCT
      can_augment_predicate: a function which returns T/F if it makes sense to remove the last PUNCT
  
      TODO: do this dynamically, as part of the DataLoader or elsewhere?
      One complication is the data comes back from the DataLoader as
      tensors & indices, so it is much more complicated to manipulate
      """
      if len(train_data) == 0:
          return []
  
      if augment_ratio is None:
          augment_ratio = get_augment_ratio(
              train_data,
              should_augment_predicate,
              can_augment_predicate,
          )
  
      if augment_ratio <= 0:
          if keep_original_sentences:
              return list(train_data)
          else:
              return []
  
      new_data = []
      for sentence in train_data:
          if can_augment_predicate(sentence):
              if random.random() < augment_ratio and len(sentence) > 1:
                  # todo: could deep copy the words
                  #       or not deep copy any of this
                  new_sentence = list(sentence[:-1])
                  new_data.append(new_sentence)
              elif keep_original_sentences:
                  new_data.append(new_sentence)
  
      return new_data
  
file_path: stanza/stanza/models/common/data.py
imports: import logging
  import random
  import torch
  import stanza.models.common.seq2seq_constant
  from stanza.models.common.doc import HEAD, ID, UPOS
label_desc: Use image node_id 8 for calling a global function with example usage: augment_punct(train_data, augment_ratio, should_augment_predicate, can_augment_predicate, keep_original_sentences) and returns: new_data, list, list, list