def _split_line_with_offsets(line):
    """Split a line by delimiter, but yield tuples of word and offset.

    This function works by dropping all the english-like punctuation from
    a line (so parenthesis preceded or succeeded by spaces, periods, etc)
    and then splitting on spaces.
    """
    for delimiter in re.finditer(r"[\.,:\;](?![^\s])", line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    for delimiter in re.finditer(r"[\"'\)\]\}>](?![^\.,\;:\"'\)\]\}>\s])",
                                 line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    for delimiter in re.finditer(r"(?<![^\.,\;:\"'\(\[\{<\s])[\"'\(\[\{<]",
                                 line):
        span = delimiter.span()
        line = line[:span[0]] + " " + line[span[1]:]

    # Treat hyphen separated words as separate words
    line = line.replace("-", " ")

    # Remove backticks
    line = line.replace("`", " ")

    for match in re.finditer(r"[^\s]+", line):
        content = match.group(0)
        if content.strip() != "":
            yield (match.span()[0], content)
Split a line by delimiter, but yield tuples of word and offset.

    This function works by dropping all the english-like punctuation from
    a line (so parenthesis preceded or succeeded by spaces, periods, etc)
    and then splitting on spaces.