streamcorpus_pipeline/_hyperlink_labels.py
def line_href_anchors(self):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.
        '''
        idx = 0
        new_clean_html = ''
        newlines_added = 0
        ## split doc up into pieces that end on an anchor tag
        parts = self.clean_html.split('</a>')
        assert len('</a>'.join(parts) ) == len(self.clean_html)
        for i in range(len(parts)):
            part = parts[i]

            ## try to get an A tag out:
            m = anchors_re.match(part)

            ## if not, then just keep going
            if not m:
                new_clean_html += part
                if i < len(parts) - 1:
                    new_clean_html += '</a>'
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## construct a text containing bytes up to the anchor text
            pre_anchor_increment = before + ahref

            ## increment the index to get line number for the anchor
            idx += len( pre_anchor_increment.splitlines() )
            first = idx

            ## usually this will be one, but it could be more than
            ## that when an anchor text contains newlines
            length = len(m.group('anchor').split('\n'))

            ## construct new clean_html with these newlines inserted
            new_clean_html += pre_anchor_increment + '\n' + m.group('anchor') + '\n</a>'

            newlines_added += 2

            ## update the index for the next loop
            idx += length - 1

            yield m.group('href'), first, length, m.group('anchor')

        ## replace clean_html with our new one that has newlines inserted
        assert len(self.clean_html) == len(new_clean_html) - newlines_added
        self.clean_html = new_clean_html
def line_href_anchors(self):
        '''
        simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.
        '''
        idx = 0
        new_clean_html = ''
        newlines_added = 0
        ## split doc up into pieces that end on an anchor tag
        parts = self.clean_html.split('</a>')
        assert len('</a>'.join(parts) ) == len(self.clean_html)
        for i in range(len(parts)):
            part = parts[i]

            ## try to get an A tag out:
            m = anchors_re.match(part)

            ## if not, then just keep going
            if not m:
                new_clean_html += part
                if i < len(parts) - 1:
                    new_clean_html += '</a>'
                continue

            before = m.group('before')
            ahref = m.group('ahref')

            ## construct a text containing bytes up to the anchor text
            pre_anchor_increment = before + ahref

            ## increment the index to get line number for the anchor
            idx += len( pre_anchor_increment.splitlines() )
            first = idx

            ## usually this will be one, but it could be more than
            ## that when an anchor text contains newlines
            length = len(m.group('anchor').split('\n'))

            ## construct new clean_html with these newlines inserted
            new_clean_html += pre_anchor_increment + '\n' + m.group('anchor') + '\n</a>'

            newlines_added += 2

            ## update the index for the next loop
            idx += length - 1

            yield m.group('href'), first, length, m.group('anchor')

        ## replace clean_html with our new one that has newlines inserted
        assert len(self.clean_html) == len(new_clean_html) - newlines_added
        self.clean_html = new_clean_html
def line_href_anchors ( self ) : idx = 0 new_clean_html = '' newlines_added = 0 ## split doc up into pieces that end on an anchor tag parts = self . clean_html . split ( '</a>' ) assert len ( '</a>' . join ( parts ) ) == len ( self . clean_html ) for i in range ( len ( parts ) ) : part = parts [ i ] ## try to get an A tag out: m = anchors_re . match ( part ) ## if not, then just keep going if not m : new_clean_html += part if i < len ( parts ) - 1 : new_clean_html += '</a>' continue before = m . group ( 'before' ) ahref = m . group ( 'ahref' ) ## construct a text containing bytes up to the anchor text pre_anchor_increment = before + ahref ## increment the index to get line number for the anchor idx += len ( pre_anchor_increment . splitlines ( ) ) first = idx ## usually this will be one, but it could be more than ## that when an anchor text contains newlines length = len ( m . group ( 'anchor' ) . split ( '\n' ) ) ## construct new clean_html with these newlines inserted new_clean_html += pre_anchor_increment + '\n' + m . group ( 'anchor' ) + '\n</a>' newlines_added += 2 ## update the index for the next loop idx += length - 1 yield m . group ( 'href' ) , first , length , m . group ( 'anchor' ) ## replace clean_html with our new one that has newlines inserted assert len ( self . clean_html ) == len ( new_clean_html ) - newlines_added self . clean_html = new_clean_html
simple, regex-based extractor of anchor tags, so we can
        compute LINE offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)

        Also, this mangles the body.clean_html so that LINE offsets
        uniquely identify tokens in anchor tags -- quite a kludge.
simple regex - based extractor of anchor tags so we can compute LINE offsets for anchor texts and associate them with their href . Generates tuple ( href_string first_byte byte_length anchor_text )