text_extensions_for_pandas/spanner/extract.py
def create_dict(
    entries: Iterable[str], tokenizer: "spacy.tokenizer.Tokenizer" = None
) -> pd.DataFrame:
    """
    Create a dictionary from a list of entries, where each entry is expressed as a
    single string.

    Tokenizes and normalizes the dictionary entries.

    :param entries: Iterable of strings, one string per dictionary entry.

    :param tokenizer: Preconfigured tokenizer object for tokenizing
    dictionary entries.  **Must always tokenize the same way as the tokenizer
    used on the target text!**  If None, this method will use tokenizer returned by
    :func:`text_extensions_for_pandas.io.spacy.simple_tokenizer()`.

    :return: a `pd.DataFrame` with the normalized, tokenized dictionary entries.
    """
    if tokenizer is None:
        tokenizer = simple_tokenizer()

    # Tokenize with SpaCy. Produces a SpaCy document object per line.
    tokenized_entries = [tokenizer(entry.lower()) for entry in entries]

    # Determine the number of tokens in the longest dictionary entry.
    max_num_toks = max([len(e) for e in tokenized_entries])

    # Generate a column for each token. Go one past the max number of tokens so
    # that every dictionary entry ends up None-terminated.
    cols_dict = {}
    for i in range(max_num_toks + 1):
        # Extract token i from every entry that has a token i
        toks_list = [e[i].text if len(e) > i else None for e in tokenized_entries]
        cols_dict["toks_{}".format(i)] = (
            # Sparse storage for tokens 2 and onward
            toks_list
            if i == 0 or not _SPARSE_DICT_ENTRIES
            else pd.SparseArray(toks_list)
        )

    return pd.DataFrame(cols_dict)

    Create a dictionary from a list of entries, where each entry is expressed as a
    single string.

    Tokenizes and normalizes the dictionary entries.

    :param entries: Iterable of strings, one string per dictionary entry.

    :param tokenizer: Preconfigured tokenizer object for tokenizing
    dictionary entries.  **Must always tokenize the same way as the tokenizer
    used on the target text!**  If None, this method will use tokenizer returned by
    :func:`text_extensions_for_pandas.io.spacy.simple_tokenizer()`.

    :return: a `pd.DataFrame` with the normalized, tokenized dictionary entries.
    
Create a dictionary from a list of entries, where each entry is expressed as a
single string.

Tokenizes and normalizes the dictionary entries.
Create a dictionary from a list of entries where each entry is expressed as a single string . Tokenizes and normalizes the dictionary entries .
def create_dict(
    entries: Iterable[str], tokenizer: "spacy.tokenizer.Tokenizer" = None
) -> pd.DataFrame:
    if tokenizer is None:
        tokenizer = simple_tokenizer()
    tokenized_entries = [tokenizer(entry.lower()) for entry in entries]
    max_num_toks = max([len(e) for e in tokenized_entries])
    cols_dict = {}
    for i in range(max_num_toks + 1):
        toks_list = [e[i].text if len(e) > i else None for e in tokenized_entries]
        cols_dict["toks_{}".format(i)] = (
            toks_list
            if i == 0 or not _SPARSE_DICT_ENTRIES
            else pd.SparseArray(toks_list)
        )
    return pd.DataFrame(cols_dict)
def create_dict ( entries : Iterable [ str ] , tokenizer : "spacy.tokenizer.Tokenizer" = None ) -> pd . DataFrame : if tokenizer is None : tokenizer = simple_tokenizer ( ) tokenized_entries = [ tokenizer ( entry . lower ( ) ) for entry in entries ] max_num_toks = max ( [ len ( e ) for e in tokenized_entries ] ) cols_dict = { } for i in range ( max_num_toks + 1 ) : toks_list = [ e [ i ] . text if len ( e ) > i else None for e in tokenized_entries ] cols_dict [ "toks_{}" . format ( i ) ] = ( toks_list if i == 0 or not _SPARSE_DICT_ENTRIES else pd . SparseArray ( toks_list ) ) return pd . DataFrame ( cols_dict )
Create a dictionary from a list of entries, where each entry is expressed as a
single string.
Create a dictionary from a list of entries where each entry is expressed as a single string .
"""
    Create a dictionary from a list of entries, where each entry is expressed as a
    single string.

    Tokenizes and normalizes the dictionary entries.

    :param entries: Iterable of strings, one string per dictionary entry.

    :param tokenizer: Preconfigured tokenizer object for tokenizing
    dictionary entries.  **Must always tokenize the same way as the tokenizer
    used on the target text!**  If None, this method will use tokenizer returned by
    :func:`text_extensions_for_pandas.io.spacy.simple_tokenizer()`.

    :return: a `pd.DataFrame` with the normalized, tokenized dictionary entries.
    """ # Tokenize with SpaCy. Produces a SpaCy document object per line. # Determine the number of tokens in the longest dictionary entry. # Generate a column for each token. Go one past the max number of tokens so # that every dictionary entry ends up None-terminated. # Extract token i from every entry that has a token i # Sparse storage for tokens 2 and onward
a `pd.DataFrame` with the normalized, tokenized dictionary entries.
a ` pd . DataFrame ` with the normalized tokenized dictionary entries .
entries
Iterable[str]
Iterable of strings, one string per dictionary entry.
Iterable of strings one string per dictionary entry .
tokenizer
"spacy.tokenizer.Tokenizer"
Preconfigured tokenizer object for tokenizing
dictionary entries.  **Must always tokenize the same way as the tokenizer
used on the target text!**  If None, this method will use tokenizer returned by
Preconfigured tokenizer object for tokenizing dictionary entries . ** Must always tokenize the same way as the tokenizer used on the target text! ** If None this method will use tokenizer returned by
func
None