nl_tokens: This function takes the raw data and breaks it into records getting rid of anything preceding the first complete record if the data does not start with a complete record .
pl_tokens: function _toRecords ( rawData , delimiter , slice ) { // Since slices with a non-zero chunk offset grab the character // immediately preceding the main chunk, if one of those chunks has a // delimiter as the first or second character, it means the chunk starts // with a complete record. In this case as well as when the chunk begins // with a partial record, splitting the chunk into an array by its // delimiter will result in a single garbage record at the beginning of // the array. If the offset is 0, the array will never start with a // garbage record let outputData = rawData ; if ( rawData . endsWith ( delimiter ) ) { // Get rid of last character if it is the delimiter since that will // just result in an empty record. outputData = rawData . slice ( 0 , - delimiter . length ) ; } if ( slice . <mask> === 0 ) { return outputData . split ( delimiter ) ; } return outputData . split ( delimiter ) . splice ( 1 ) ; }