pub fn new<T>(rng: R, vocab: &[CountedType<T>], table_size: usize) -> Self {
        let sum = vocab.iter().map(|t| (t.count() as f32).sqrt()).sum::<f32>();

        let mut vocab_indices = Vec::new();
        for (i, counted_type) in vocab.iter().enumerate() {
            let discounted_freq = (counted_type.count() as f32).sqrt();
            let type_n_entries = (discounted_freq / sum) * table_size as f32;
            vocab_indices.extend(std::iter::repeat(i).take(type_n_entries as usize));
        }

        Self {
            rng,
            uniform: Uniform::new(0, vocab_indices.len()),
            vocab_indices,
        }
    }
/// Construct a range generator for the given vocabulary item counts.
///
/// The `table_size` parameter is used to determine how large the sampling
/// table should be. A larger table makes sampling more accurate, at the
/// cost of larger memory use and increased cache misses.
Construct a range generator for the given vocabulary item counts.
The `table_size` parameter is used to determine how large the sampling
table should be. A larger table makes sampling more accurate, at the
cost of larger memory use and increased cache misses.
Construct a range generator for the given vocabulary item counts . The ` table_size ` parameter is used to determine how large the sampling table should be . A larger table makes sampling more accurate at the cost of larger memory use and increased cache misses .
pub fn new<T>(rng: R, vocab: &[CountedType<T>], table_size: usize) -> Self {
        let sum = vocab.iter().map(|t| (t.count() as f32).sqrt()).sum::<f32>();
        let mut vocab_indices = Vec::new();
        for (i, counted_type) in vocab.iter().enumerate() {
            let discounted_freq = (counted_type.count() as f32).sqrt();
            let type_n_entries = (discounted_freq / sum) * table_size as f32;
            vocab_indices.extend(std::iter::repeat(i).take(type_n_entries as usize));
        }
        Self {
            rng,
            uniform: Uniform::new(0, vocab_indices.len()),
            vocab_indices,
        }
    }
pub fn new < T > ( rng : R , vocab : & [ CountedType < T > ] , table_size : usize ) -> Self { let sum = vocab . iter ( ) . map ( | t | ( t . count ( ) as f32 ) . sqrt ( ) ) . sum :: < f32 > ( ) ; let mut vocab_indices = Vec :: new ( ) ; for ( i , counted_type ) in vocab . iter ( ) . enumerate ( ) { let discounted_freq = ( counted_type . count ( ) as f32 ) . sqrt ( ) ; let type_n_entries = ( discounted_freq / sum ) * table_size as f32 ; vocab_indices . extend ( std :: iter :: repeat ( i ) . take ( type_n_entries as usize ) ) ; } Self { rng , uniform : Uniform :: new ( 0 , vocab_indices . len ( ) ) , vocab_indices , } }
Construct a range generator for the given vocabulary item counts.
Construct a range generator for the given vocabulary item counts .
rng
R
vocab
&[CountedType<T>]
table_size
usize
rng
R
vocab
&[CountedType<T>]
table_size
usize