streamcorpus_pipeline/_lingpipe.py
def _make_token(self, start, end):
        '''
        Instantiates a Token from self._input_string[start:end]
        '''
        ## all thfift strings must be encoded first
        tok_string = self._input_string[start:end].encode('utf-8')
        if only_whitespace.match(tok_string):
            ## drop any tokens with only whitespace
            return None
        tok = Token()
        tok.token = tok_string
        tok.token_num = self.tok_num
        if 'BYTES' in self.config['offset_types']:
            tok.offsets[OffsetType.BYTES] = Offset(
                type =  OffsetType.BYTES,
                first=self.byte_idx + len(self._input_string[:start].encode('utf-8')),
                length=len(tok_string),
                value=self.config['offset_debugging'] and tok_string or None,
                )
        if 'LINES' in self.config['offset_types']:
            tok.offsets[OffsetType.LINES] = Offset(
                type =  OffsetType.LINES,
                first=self.line_idx,
                length=1,
                value=self.config['offset_debugging'] and tok_string or None,
                )
        self.tok_num += 1
        ## keep track of position within a sentence
        tok.sentence_pos = self.sent_pos
        self.sent_pos += 1
        return tok
def _make_token(self, start, end):
        '''
        Instantiates a Token from self._input_string[start:end]
        '''
        ## all thfift strings must be encoded first
        tok_string = self._input_string[start:end].encode('utf-8')
        if only_whitespace.match(tok_string):
            ## drop any tokens with only whitespace
            return None
        tok = Token()
        tok.token = tok_string
        tok.token_num = self.tok_num
        if 'BYTES' in self.config['offset_types']:
            tok.offsets[OffsetType.BYTES] = Offset(
                type =  OffsetType.BYTES,
                first=self.byte_idx + len(self._input_string[:start].encode('utf-8')),
                length=len(tok_string),
                value=self.config['offset_debugging'] and tok_string or None,
                )
        if 'LINES' in self.config['offset_types']:
            tok.offsets[OffsetType.LINES] = Offset(
                type =  OffsetType.LINES,
                first=self.line_idx,
                length=1,
                value=self.config['offset_debugging'] and tok_string or None,
                )
        self.tok_num += 1
        ## keep track of position within a sentence
        tok.sentence_pos = self.sent_pos
        self.sent_pos += 1
        return tok
def _make_token ( self , start , end ) : ## all thfift strings must be encoded first tok_string = self . _input_string [ start : end ] . encode ( 'utf-8' ) if only_whitespace . match ( tok_string ) : ## drop any tokens with only whitespace return None tok = Token ( ) tok . token = tok_string tok . token_num = self . tok_num if 'BYTES' in self . config [ 'offset_types' ] : tok . offsets [ OffsetType . BYTES ] = Offset ( type = OffsetType . BYTES , first = self . byte_idx + len ( self . _input_string [ : start ] . encode ( 'utf-8' ) ) , length = len ( tok_string ) , value = self . config [ 'offset_debugging' ] and tok_string or None , ) if 'LINES' in self . config [ 'offset_types' ] : tok . offsets [ OffsetType . LINES ] = Offset ( type = OffsetType . LINES , first = self . line_idx , length = 1 , value = self . config [ 'offset_debugging' ] and tok_string or None , ) self . tok_num += 1 ## keep track of position within a sentence tok . sentence_pos = self . sent_pos self . sent_pos += 1 return tok
Instantiates a Token from self._input_string[start:end]
Instantiates a Token from self . _input_string [ start : end ]