nl_tokens: [ { offset length } ] of chunks size assuming delimiter for a file with total size .
pl_tokens: function getOffsets ( size , total , delimiter ) { if ( total === 0 ) { return [ ] ; } if ( <mask> < size ) { return [ { length : total , offset : 0 } ] ; } const fullChunks = Math . floor ( total / size ) ; const delta = delimiter . length ; const length = size + delta ; const chunks = [ ] ; for ( let chunk = 1 ; chunk < fullChunks ; chunk += 1 ) { chunks . push ( { length , offset : ( chunk * size ) - delta } ) ; } // First chunk doesn't need +/- delta. chunks . unshift ( { offset : 0 , length : size } ) ; // When last chunk is not full chunk size. const lastChunk = total % size ; if ( lastChunk > 0 ) { chunks . push ( { offset : ( fullChunks * size ) - delta , length : lastChunk + delta } ) ; } return chunks ; }