license import { ENV } from '@tensorflow/tfjs-core' ; import { LazyIterator , OneToManyIterator } from './lazy_iterator' ; import { StringIterator } from './string_iterator' ; export abstract class ByteChunkIterator extends LazyIterator < Uint8Array > { /**
     * Decode a stream of UTF8-encoded byte arrays to a stream of strings.
     *
     * The byte arrays producetd from the ByteChunkIterator on which this is
     * called will be interpreted as concatenated.  No assumptions are made about
     * the boundaries of the incoming chunks, so a multi-byte UTF8 encoding of a
     * character may span the boundary between chunks.  This naturally happens,
     * for instance, when reading fixed-size byte arrays from a file.
     */ decodeUTF8 ( ) { return new Utf8Iterator ( this ) ; } } class Utf8Iterator extends StringIterator { private impl : Utf8IteratorImpl ; constructor ( protected upstream ) { super ( ) ; this . impl = new Utf8IteratorImpl ( upstream ) ; } summary ( ) { return this . impl . summary ( ) ; } async next ( ) { return this . impl . next ( ) ; } } param upstream class Utf8IteratorImpl extends OneToManyIterator < string > { decoder : any ; constructor ( protected readonly upstream ) { super ( ) ; if ( ENV . get ( 'IS_BROWSER' ) ) { this . decoder = new TextDecoder ( 'utf-8' ) ; } else { const { StringDecoder } = require ( 'string_decoder' ) ; this . decoder = new StringDecoder ( 'utf8' ) ; } } summary ( ) { return `${ this . upstream . summary ( ) } -> Utf8` ; } async pump ( ) { const chunkResult = await this . upstream . next ( ) ; let chunk ; if ( chunkResult . done ) { return false ; } else { chunk = chunkResult . value ; } let text ; if ( ENV . get ( 'IS_BROWSER' ) ) { text = this . decoder . decode ( chunk , { stream : true } ) ; } else { text = this . decoder . write ( Buffer . from ( chunk . buffer ) ) ; } this . outputQueue . push ( text ) ; return true ; } }