libmultilabel/data_utils.py
def load_or_build_text_dict(
    dataset,
    vocab_file=None,
    min_vocab_freq=1,
    embed_file=None,
    embed_cache_dir=None,
    silent=False,
    normalize_embed=False
):
    """Build or load the vocabulary from the training dataset or the predefined `vocab_file`.
    The pretrained embedding can be either from a self-defined `embed_file` or from one of
    the vectors defined in torchtext `vectors` (https://pytorch.org/text/0.9.0/vocab.html#torchtext.vocab.Vocab.load_vectors).

    Args:
        dataset (list): List of training instances with index, label, and tokenized text.
        vocab_file (str, optional): Path to a file holding vocabuaries. Defaults to None.
        min_vocab_freq (int, optional): The minimum frequency needed to include a token in the vocabulary. Defaults to 1.
        embed_file (str): Path to a file holding pre-trained embeddings.
        embed_cache_dir (str, optional): Path to a directory for storing cached embeddings. Defaults to None.
        silent (bool, optional): Enable silent mode. Defaults to False.
        normalize_embed (bool, optional): Whether the embeddings of each word is normalized to a unit vector. Defaults to False.
    Returns:
        torchtext.vocab.Vocab: A vocab object which maps tokens to indices.
    """
    if vocab_file:
        logging.info(f'Load vocab from {vocab_file}')
        with open(vocab_file, 'r') as fp:
            vocab_list = [PAD] + [vocab.strip() for vocab in fp.readlines()]
        vocabs = Vocab(collections.Counter(vocab_list), specials=[UNK],
                       min_freq=1, specials_first=False)  # specials_first=False to keep PAD index 0
    else:
        counter = collections.Counter()
        for data in dataset:
            unique_tokens = set(data['text'])
            counter.update(unique_tokens)
        vocabs = Vocab(counter, specials=[PAD, UNK],
                       min_freq=min_vocab_freq)
    logging.info(f'Read {len(vocabs)} vocabularies.')

    if os.path.exists(embed_file):
        logging.info(f'Load pretrained embedding from file: {embed_file}.')
        embedding_weights = get_embedding_weights_from_file(vocabs, embed_file, silent)
        vocabs.set_vectors(vocabs.stoi, embedding_weights, dim=embedding_weights.shape[1])
    elif not embed_file.isdigit():
        logging.info(f'Load pretrained embedding from torchtext.')
        vocabs.load_vectors(embed_file, cache=embed_cache_dir)
    else:
        raise NotImplementedError

    if normalize_embed:
        embedding_weights = copy.deepcopy(vocabs.vectors.detach().cpu().numpy())
        for i, vector in enumerate(embedding_weights):
            # We use the constant 1e-6 by following https://github.com/jamesmullenbach/caml-mimic/blob/44a47455070d3d5c6ee69fb5305e32caec104960/dataproc/extract_wvs.py#L60
            # for an internal experiment of reproducing their results.
            embedding_weights[i] = vector/float(np.linalg.norm(vector) + 1e-6)
        embedding_weights = torch.Tensor(embedding_weights)
        vocabs.set_vectors(vocabs.stoi, embedding_weights, dim=embedding_weights.shape[1])

    return vocabs
Build or load the vocabulary from the training dataset or the predefined `vocab_file`.
    The pretrained embedding can be either from a self-defined `embed_file` or from one of
    the vectors defined in torchtext `vectors` (https://pytorch.org/text/0.9.0/vocab.html#torchtext.vocab.Vocab.load_vectors).

    Args:
        dataset (list): List of training instances with index, label, and tokenized text.
        vocab_file (str, optional): Path to a file holding vocabuaries. Defaults to None.
        min_vocab_freq (int, optional): The minimum frequency needed to include a token in the vocabulary. Defaults to 1.
        embed_file (str): Path to a file holding pre-trained embeddings.
        embed_cache_dir (str, optional): Path to a directory for storing cached embeddings. Defaults to None.
        silent (bool, optional): Enable silent mode. Defaults to False.
        normalize_embed (bool, optional): Whether the embeddings of each word is normalized to a unit vector. Defaults to False.
    Returns:
        torchtext.vocab.Vocab: A vocab object which maps tokens to indices.
    
Build or load the vocabulary from the training dataset or the predefined `vocab_file`.
The pretrained embedding can be either from a self-defined `embed_file` or from one of
the vectors defined in torchtext `vectors` .
Build or load the vocabulary from the training dataset or the predefined ` vocab_file ` . The pretrained embedding can be either from a self - defined ` embed_file ` or from one of the vectors defined in torchtext ` vectors ` .
def load_or_build_text_dict(
    dataset,
    vocab_file=None,
    min_vocab_freq=1,
    embed_file=None,
    embed_cache_dir=None,
    silent=False,
    normalize_embed=False
):
    if vocab_file:
        logging.info(f'Load vocab from {vocab_file}')
        with open(vocab_file, 'r') as fp:
            vocab_list = [PAD] + [vocab.strip() for vocab in fp.readlines()]
        vocabs = Vocab(collections.Counter(vocab_list), specials=[UNK],
                       min_freq=1, specials_first=False)  
    else:
        counter = collections.Counter()
        for data in dataset:
            unique_tokens = set(data['text'])
            counter.update(unique_tokens)
        vocabs = Vocab(counter, specials=[PAD, UNK],
                       min_freq=min_vocab_freq)
    logging.info(f'Read {len(vocabs)} vocabularies.')
    if os.path.exists(embed_file):
        logging.info(f'Load pretrained embedding from file: {embed_file}.')
        embedding_weights = get_embedding_weights_from_file(vocabs, embed_file, silent)
        vocabs.set_vectors(vocabs.stoi, embedding_weights, dim=embedding_weights.shape[1])
    elif not embed_file.isdigit():
        logging.info(f'Load pretrained embedding from torchtext.')
        vocabs.load_vectors(embed_file, cache=embed_cache_dir)
    else:
        raise NotImplementedError
    if normalize_embed:
        embedding_weights = copy.deepcopy(vocabs.vectors.detach().cpu().numpy())
        for i, vector in enumerate(embedding_weights):
            embedding_weights[i] = vector/float(np.linalg.norm(vector) + 1e-6)
        embedding_weights = torch.Tensor(embedding_weights)
        vocabs.set_vectors(vocabs.stoi, embedding_weights, dim=embedding_weights.shape[1])
    return vocabs
def load_or_build_text_dict ( dataset , vocab_file = None , min_vocab_freq = 1 , embed_file = None , embed_cache_dir = None , silent = False , normalize_embed = False ) : if vocab_file : logging . info ( f'Load vocab from {vocab_file}' ) with open ( vocab_file , 'r' ) as fp : vocab_list = [ PAD ] + [ vocab . strip ( ) for vocab in fp . readlines ( ) ] vocabs = Vocab ( collections . Counter ( vocab_list ) , specials = [ UNK ] , min_freq = 1 , specials_first = False ) else : counter = collections . Counter ( ) for data in dataset : unique_tokens = set ( data [ 'text' ] ) counter . update ( unique_tokens ) vocabs = Vocab ( counter , specials = [ PAD , UNK ] , min_freq = min_vocab_freq ) logging . info ( f'Read {len(vocabs)} vocabularies.' ) if os . path . exists ( embed_file ) : logging . info ( f'Load pretrained embedding from file: {embed_file}.' ) embedding_weights = get_embedding_weights_from_file ( vocabs , embed_file , silent ) vocabs . set_vectors ( vocabs . stoi , embedding_weights , dim = embedding_weights . shape [ 1 ] ) elif not embed_file . isdigit ( ) : logging . info ( f'Load pretrained embedding from torchtext.' ) vocabs . load_vectors ( embed_file , cache = embed_cache_dir ) else : raise NotImplementedError if normalize_embed : embedding_weights = copy . deepcopy ( vocabs . vectors . detach ( ) . cpu ( ) . numpy ( ) ) for i , vector in enumerate ( embedding_weights ) : embedding_weights [ i ] = vector / float ( np . linalg . norm ( vector ) + 1e-6 ) embedding_weights = torch . Tensor ( embedding_weights ) vocabs . set_vectors ( vocabs . stoi , embedding_weights , dim = embedding_weights . shape [ 1 ] ) return vocabs
Build or load the vocabulary from the training dataset or the predefined `vocab_file`.
Build or load the vocabulary from the training dataset or the predefined ` vocab_file ` .
"""Build or load the vocabulary from the training dataset or the predefined `vocab_file`.
    The pretrained embedding can be either from a self-defined `embed_file` or from one of
    the vectors defined in torchtext `vectors` (https://pytorch.org/text/0.9.0/vocab.html#torchtext.vocab.Vocab.load_vectors).

    Args:
        dataset (list): List of training instances with index, label, and tokenized text.
        vocab_file (str, optional): Path to a file holding vocabuaries. Defaults to None.
        min_vocab_freq (int, optional): The minimum frequency needed to include a token in the vocabulary. Defaults to 1.
        embed_file (str): Path to a file holding pre-trained embeddings.
        embed_cache_dir (str, optional): Path to a directory for storing cached embeddings. Defaults to None.
        silent (bool, optional): Enable silent mode. Defaults to False.
        normalize_embed (bool, optional): Whether the embeddings of each word is normalized to a unit vector. Defaults to False.
    Returns:
        torchtext.vocab.Vocab: A vocab object which maps tokens to indices.
    """ # specials_first=False to keep PAD index 0 # We use the constant 1e-6 by following https://github.com/jamesmullenbach/caml-mimic/blob/44a47455070d3d5c6ee69fb5305e32caec104960/dataproc/extract_wvs.py#L60 # for an internal experiment of reproducing their results.
A vocab object which maps tokens to indices.
A vocab object which maps tokens to indices .
torchtext.vocab.Vocab
dataset
List of training instances with index, label, and tokenized text.
List of training instances with index label and tokenized text .
vocab_file
Path to a file holding vocabuaries. Defaults to None.
Path to a file holding vocabuaries . Defaults to None .
None
min_vocab_freq
The minimum frequency needed to include a token in the vocabulary. Defaults to 1.
The minimum frequency needed to include a token in the vocabulary . Defaults to 1 .
1
embed_file
Path to a file holding pre-trained embeddings.
Path to a file holding pre - trained embeddings .
embed_cache_dir
Path to a directory for storing cached embeddings. Defaults to None.
Path to a directory for storing cached embeddings . Defaults to None .
None
silent
Enable silent mode. Defaults to False.
Enable silent mode . Defaults to False .
False
normalize_embed
Whether the embeddings of each word is normalized to a unit vector. Defaults to False.
Whether the embeddings of each word is normalized to a unit vector . Defaults to False .
False