'Convert a loose word index matrix to a compact array using
a fixed loose to dense mapping. Out of vocabulary word indices
will be replaced by the out of vocabulary index. The most common
index will be mapped to 0, the next most common to 1, and so on.
Arguments
word_loose : int array
Input loose word array to be converted into a compact array.
Examples
>>> corpus = Corpus()
>>> word_indices = np.random.randint(100, size=1000)
>>> n_words = len(np.unique(word_indices))
>>> corpus.update_word_count(word_indices)
>>> corpus.finalize()
>>> word_compact = corpus.to_compact(word_indices)
>>> # The most common word in the training set will be mapped to be
>>> # right after all the special tokens, so 2 in this case.
>>> np.argmax(np.bincount(word_compact)) == 2
True
>>> most_common = np.argmax(np.bincount(word_indices))
>>> corpus.loose_to_compact[most_common] == 2
True
>>> # Out of vocabulary indices will be mapped to 1
>>> word_indices = np.random.randint(150, size=1000)
>>> word_compact_oov = corpus.to_compact(word_indices)
>>> oov = corpus.specials_to_compact[\'out_of_vocabulary\']
>>> oov
1
>>> oov in word_compact
False
>>> oov in word_compact_oov
True'

def to_compact(self, word_loose):

    self._check_finalized()
    keys = self.keys_loose
    reps = self.keys_compact
    uniques = np.unique(word_loose)
    oov = np.setdiff1d(uniques, keys, assume_unique=True)
    oov_token = self.specials_to_compact['out_of_vocabulary']
    keys = np.concatenate((keys, oov))
    reps = np.concatenate((reps, (np.zeros_like(oov) + oov_token)))
    compact = fast_replace(word_loose, keys, reps)
    msg = 'Error:    all    compact    indices    should    be    non-negative'
    assert (compact.min() >= 0), msg
    return compact
