streamcorpus_pipeline/_lingpipe.py
def sentences(self):
        '''
        Iterate over <s> XML-like tags and tokenize with nltk
        '''
        for sentence_id, node in enumerate(self.ner_dom.childNodes):
            ## increment the char index with any text before the <s>
            ## tag.  Crucial assumption here is that the LingPipe XML
            ## tags are inserted into the original byte array without
            ## modifying the portions that are not inside the
            ## LingPipe-added tags themselves.
            if node.nodeType == node.TEXT_NODE:
                ## we expect to only see TEXT_NODE instances with whitespace
                assert only_whitespace.match(node.data), repr(node.data)

                ## must convert back to utf-8 to have expected byte offsets
                self.byte_idx += len(node.data.encode('utf-8'))

                ## count full lines, i.e. only those that end with a \n
                # 'True' here means keep the trailing newlines
                for line in node.data.splitlines(True):
                    if line.endswith('\n'):
                        self.line_idx += 1
            else:
                logger.debug('getting tokens for sentence_id=%d' % sentence_id)
                more_sentence_remains = True
                while more_sentence_remains:
                    ## always a sentence
                    sent = Sentence()

                    ## this "node" came from for loop above, and it's
                    ## childNodes list might have been popped by a
                    ## previous pass through this while loop
                    tokens = iter( self.tokens( node ) )

                    while 1:
                        try:
                            tok = tokens.next()
                            sent.tokens.append(tok)
                            #logger.debug('got token: %r  %d %d' % (tok.token, tok.mention_id, tok.sentence_pos))

                        except StopIteration:
                            yield sent
                            more_sentence_remains = False
                            break
def sentences(self):
        '''
        Iterate over <s> XML-like tags and tokenize with nltk
        '''
        for sentence_id, node in enumerate(self.ner_dom.childNodes):
            ## increment the char index with any text before the <s>
            ## tag.  Crucial assumption here is that the LingPipe XML
            ## tags are inserted into the original byte array without
            ## modifying the portions that are not inside the
            ## LingPipe-added tags themselves.
            if node.nodeType == node.TEXT_NODE:
                ## we expect to only see TEXT_NODE instances with whitespace
                assert only_whitespace.match(node.data), repr(node.data)

                ## must convert back to utf-8 to have expected byte offsets
                self.byte_idx += len(node.data.encode('utf-8'))

                ## count full lines, i.e. only those that end with a \n
                # 'True' here means keep the trailing newlines
                for line in node.data.splitlines(True):
                    if line.endswith('\n'):
                        self.line_idx += 1
            else:
                logger.debug('getting tokens for sentence_id=%d' % sentence_id)
                more_sentence_remains = True
                while more_sentence_remains:
                    ## always a sentence
                    sent = Sentence()

                    ## this "node" came from for loop above, and it's
                    ## childNodes list might have been popped by a
                    ## previous pass through this while loop
                    tokens = iter( self.tokens( node ) )

                    while 1:
                        try:
                            tok = tokens.next()
                            sent.tokens.append(tok)
                            #logger.debug('got token: %r  %d %d' % (tok.token, tok.mention_id, tok.sentence_pos))

                        except StopIteration:
                            yield sent
                            more_sentence_remains = False
                            break
def sentences ( self ) : for sentence_id , node in enumerate ( self . ner_dom . childNodes ) : ## increment the char index with any text before the <s> ## tag.  Crucial assumption here is that the LingPipe XML ## tags are inserted into the original byte array without ## modifying the portions that are not inside the ## LingPipe-added tags themselves. if node . nodeType == node . TEXT_NODE : ## we expect to only see TEXT_NODE instances with whitespace assert only_whitespace . match ( node . data ) , repr ( node . data ) ## must convert back to utf-8 to have expected byte offsets self . byte_idx += len ( node . data . encode ( 'utf-8' ) ) ## count full lines, i.e. only those that end with a \n # 'True' here means keep the trailing newlines for line in node . data . splitlines ( True ) : if line . endswith ( '\n' ) : self . line_idx += 1 else : logger . debug ( 'getting tokens for sentence_id=%d' % sentence_id ) more_sentence_remains = True while more_sentence_remains : ## always a sentence sent = Sentence ( ) ## this "node" came from for loop above, and it's ## childNodes list might have been popped by a ## previous pass through this while loop tokens = iter ( self . tokens ( node ) ) while 1 : try : tok = tokens . next ( ) sent . tokens . append ( tok ) #logger.debug('got token: %r  %d %d' % (tok.token, tok.mention_id, tok.sentence_pos)) except StopIteration : yield sent more_sentence_remains = False break
Iterate over <s> XML-like tags and tokenize with nltk
Iterate over <s > XML - like tags and tokenize with nltk