license import { ENV } from '@tensorflow/tfjs-core' ; import { FileDataSource } from '../sources/file_data_source' ; import { TextLineDataset } from './text_line_dataset' ; const runes = `ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ
ᛋᚳᛖᚪᛚ᛫ᚦᛖᚪᚻ᛫ᛗᚪᚾᚾᚪ᛫ᚷᛖᚻᚹᛦᛚᚳ᛫ᛗᛁᚳᛚᚢᚾ᛫ᚻᛦᛏ᛫ᛞᚫᛚᚪᚾ
ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬` ; const textWithDOSLineBreaks = 'abc\rdefg\r\nhijklmn\r\nopqrst' ; const testBlob = ENV . get ( 'IS_BROWSER' ) ? new Blob ( [ runes ] ) : Buffer . from ( runes ) ; const textBlobWithDOSLineBreaks = ENV . get ( 'IS_BROWSER' ) ? new Blob ( [ textWithDOSLineBreaks ] ) : Buffer . from ( textWithDOSLineBreaks ) ; describe ( 'TextLineDataset' , ( ) => { it ( 'Produces a stream of strings containing UTF8-decoded text lines' , async ( ) => { const source = new FileDataSource ( testBlob , { chunkSize : 10 } ) ; const dataset = new TextLineDataset ( source ) ; const iter = await dataset . iterator ( ) ; const result = await iter . toArrayForTest ( ) ; expect ( result ) . toEqual ( [ 'ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ' , 'ᛋᚳᛖᚪᛚ᛫ᚦᛖᚪᚻ᛫ᛗᚪᚾᚾᚪ᛫ᚷᛖᚻᚹᛦᛚᚳ᛫ᛗᛁᚳᛚᚢᚾ᛫ᚻᛦᛏ᛫ᛞᚫᛚᚪᚾ' , 'ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬' , ] ) ; } ) ; it ( 'Parses lines from windows/DOS text correctly' , async ( ) => { const source = new FileDataSource ( textBlobWithDOSLineBreaks , { chunkSize : 10 } ) ; const dataset = new TextLineDataset ( source ) ; const iter = await dataset . iterator ( ) ; const result = await iter . toArrayForTest ( ) ; expect ( result [ 0 ] ) . toEqual ( 'abc\rdefg' ) ; expect ( result [ 1 ] ) . toEqual ( 'hijklmn' ) ; expect ( result [ 2 ] ) . toEqual ( 'opqrst' ) ; } ) ; } ) ;