pub fn new(words: impl Into<Vec<String>>, min_n: u32, max_n: u32, indexer: I) -> Self {
        let words = words.into();
        let indices = create_indices(&words);
        assert_eq!(
            words.len(),
            indices.len(),
            "words contained duplicate entries."
        );

        // Check that usize can represent the indexer's upper bound.
        assert!(
            usize::try_from(indexer.upper_bound()).is_ok(),
            "The upper bound of the indexer cannot be represented by the native word size."
        );

        // Check that usize can represent the combined vocabulary sizes.
        assert!(
            words
                .len()
                .checked_add(indexer.upper_bound() as usize)
                .is_some(),
            "The vocab + subword vocab size cannot be represented by the native word size"
        );

        SubwordVocab {
            indices,
            words,
            min_n,
            max_n,
            indexer,
        }
    }
/// Construct a new `SubwordVocab`.
///
/// Words are assigned indices in the given order. NGrams in range `(min_n..max_n)` are
/// considered. The `indexer` is used to look up indices for the NGrams produced by this
/// `SubwordVocab`.
///
/// Panics when there are duplicate words.
Construct a new `SubwordVocab`.
Words are assigned indices in the given order.

Panics when there are duplicate words.
Construct a new ` SubwordVocab ` . Words are assigned indices in the given order . Panics when there are duplicate words .
pub fn new(words: impl Into<Vec<String>>, min_n: u32, max_n: u32, indexer: I) -> Self {
        let words = words.into();
        let indices = create_indices(&words);
        assert_eq!(
            words.len(),
            indices.len(),
            "words contained duplicate entries."
        );
        // Check that usize can represent the indexer's upper bound.
        assert!(
            usize::try_from(indexer.upper_bound()).is_ok(),
            "The upper bound of the indexer cannot be represented by the native word size."
        );
        // Check that usize can represent the combined vocabulary sizes.
        assert!(
            words
                .len()
                .checked_add(indexer.upper_bound() as usize)
                .is_some(),
            "The vocab + subword vocab size cannot be represented by the native word size"
        );
        SubwordVocab {
            indices,
            words,
            min_n,
            max_n,
            indexer,
        }
    }
pub fn new ( words : impl Into < Vec < String > > , min_n : u32 , max_n : u32 , indexer : I ) -> Self { let words = words . into ( ) ; let indices = create_indices ( & words ) ; assert_eq ! ( words len ( ) indices len ( ) " " ) ; // Check that usize can represent the indexer's upper bound. assert ! ( usize try_from ( indexer upper_bound ( ) ) is_ok ( ) " " ) ; // Check that usize can represent the combined vocabulary sizes. assert ! ( words len ( ) checked_add ( indexer upper_bound ( ) as usize ) is_some ( ) " " ) ; SubwordVocab { indices , words , min_n , max_n , indexer , } }
Construct a new `SubwordVocab`.
Construct a new ` SubwordVocab ` .
words
impl Into<Vec<String>>
min_n
u32
max_n
u32
indexer
I
words
impl Into<Vec<String>>
min_n
u32
max_n
u32
indexer
I