def get_tokens_with_heads(self, snlp_doc):
    'Flatten the tokens in the StanfordNLP Doc and extract the token indices\n        of the sentence start tokens to set is_sent_start.\n\n        snlp_doc (stanfordnlp.Document): The processed StanfordNLP doc.\n        RETURNS (list): The tokens (words).\n        '
    tokens = []
    heads = []
    offset = 0
    for sentence in snlp_doc.sentences:
        for token in sentence.tokens:
            for word in token.words:
                if word.governor:
                    head = (((word.governor + offset) - len(tokens)) - 1)
                else:
                    head = 0
                heads.append(head)
                tokens.append(word)
        offset += sum((len(token.words) for token in sentence.tokens))
    return (tokens, heads)
Flatten the tokens in the StanfordNLP Doc and extract the token indices
of the sentence start tokens to set is_sent_start.

snlp_doc (stanfordnlp.Document): The processed StanfordNLP doc.
RETURNS (list): The tokens (words).
def get_tokens_with_heads(self, snlp_doc):
    'Flatten the tokens in the StanfordNLP Doc and extract the token indices\n        of the sentence start tokens to set is_sent_start.\n\n        snlp_doc (stanfordnlp.Document): The processed StanfordNLP doc.\n        RETURNS (list): The tokens (words).\n        '
    tokens = []
    heads = []
    offset = 0
    for sentence in snlp_doc.sentences:
        for token in sentence.tokens:
            for word in token.words:
                if word.governor:
                    head = (((word.governor + offset) - len(tokens)) - 1)
                else:
                    head = 0
                heads.append(head)
                tokens.append(word)
        offset += sum((len(token.words) for token in sentence.tokens))
    return (tokens, heads)