'Subsample the most frequent words. This aggressively
replaces words with frequencies higher than `threshold`. Words
are replaced with the out_of_vocabulary token.
Words will be replaced with probability as a function of their
frequency in the training corpus:
.. math::
p(w) = 1.0 - \sqrt{threshold\over f(w)}
Arguments
words_compact: int array
The input array to subsample.
threshold: float in [0, 1]
Words with frequencies higher than this will be increasingly
subsampled.
Examples
>>> corpus = Corpus()
>>> word_indices = (np.random.power(5.0, size=1000) * 100).astype(\'i\')
>>> corpus.update_word_count(word_indices)
>>> corpus.finalize()
>>> compact = corpus.to_compact(word_indices)
>>> sampled = corpus.subsample_frequent(compact, threshold=1e-2)
>>> skip = corpus.specials_to_compact[\'skip\']
>>> np.sum(compact == skip)  # No skips in the compact tokens
0
>>> np.sum(sampled == skip) > 0  # Many skips in the sampled tokens
True
.. [1] Distributed Representations of Words and Phrases and
their Compositionality. Mikolov, Tomas and Sutskever, Ilya
and Chen, Kai and Corrado, Greg S and Dean, Jeff
Advances in Neural Information Processing Systems 26'

def subsample_frequent(self, words_compact, threshold=1e-05):

    self._check_finalized()
    freq = (self.keys_frequency + 1e-10)
    pw = (1.0 - (np.sqrt((threshold / freq)) + (threshold / freq)))
    prob = fast_replace(words_compact, self.keys_compact, pw)
    draw = np.random.uniform(size=prob.shape)
    ret = words_compact.copy()
    ret[(prob > draw)] = self.specials_to_compact['skip']
    return ret
