nl_tokens: This function will grab the chunk of data specified by the slice plus an extra margin if the slice does not end with the delimiter .
pl_tokens: function getChunk ( readerClient , slice , opConfig , logger , metadata ) { const delimiter = opConfig . line_delimiter ; async function getMargin ( offset , length ) { let margin = '' ; return new Promise ( async ( resolve ) => { while ( margin . indexOf ( delimiter ) === - 1 ) { // reader clients must return false-y when nothing more to read. const chunk = await readerClient ( <mask> , length ) ; // eslint-disable-line no-await-in-loop, max-len if ( ! chunk ) { resolve ( margin . split ( delimiter ) [ 0 ] ) ; return ; } margin += chunk ; offset += length ; // eslint-disable-line no-param-reassign, max-len } // Don't read too far - next slice will get it. resolve ( margin . split ( delimiter ) [ 0 ] ) ; } ) ; } let needMargin = false ; if ( slice . length ) { // Determines whether or not to grab the extra margin. if ( slice . offset + slice . length !== slice . total ) { needMargin = true ; } } return readerClient ( slice . offset , slice . length ) . then ( async ( data ) => { if ( data . endsWith ( delimiter ) ) { // Skip the margin if the raw data ends with the delimiter since // it will end with a complete record. needMargin = false ; } if ( needMargin ) { // Want to minimize reads since will typically be over the // network. Using twice the average record size as a heuristic. const avgSize = _averageRecordSize ( data . split ( delimiter ) ) ; data += await getMargin ( slice . offset + slice . length , 2 * avgSize ) ; // eslint-disable-line no-param-reassign, max-len } return data ; } ) . then ( data => chunkFormatter [ opConfig . format ] ( data , logger , opConfig , metadata , slice ) ) ; }