source/sample/transformer/train/TrainDataSet.cpp
bool TrainDataSet::LoadBatchToBuf()
{
    int n = 0;

    while (n < config->common.bufSize) {
        Sample* sample = LoadSample();
        buf->Add(sample);
        n++;
    }
    LOG("loaded %d samples", n);

    /* group samples into buckets */
    SortByTgtLengthAscending();
    SortBySrcLengthAscending();

    /* build buckets for training */
    if (isTraining)
        BuildBucket();

    return true;
}
/*
load samples from a file into the buffer
*/
load samples from a file into the buffer
load samples from a file into the buffer
bool TrainDataSet::LoadBatchToBuf()
{
    int n = 0;
    while (n < config->common.bufSize) {
        Sample* sample = LoadSample();
        buf->Add(sample);
        n++;
    }
    LOG("loaded %d samples", n);
    SortByTgtLengthAscending();
    SortBySrcLengthAscending();
    if (isTraining)
        BuildBucket();
    return true;
}
bool TrainDataSet :: LoadBatchToBuf ( ) { int n = 0 ; while ( n < config -> common . bufSize ) { Sample * sample = LoadSample ( ) ; buf -> Add ( sample ) ; n ++ ; } LOG ( " " , n ) ; SortByTgtLengthAscending ( ) ; SortBySrcLengthAscending ( ) ; if ( isTraining ) BuildBucket ( ) ; return true ; }
load samples from a file into the buffer
load samples from a file into the buffer
/* group samples into buckets */ /* build buckets for training */