streamcorpus_pipeline/_hyperlink_labels.py
def byte_href_anchors_state_machine(self):
        '''
        byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        tag_depth = 0
        a_tag_depth = 0
        vals = []
        href = None
        idx_bytes = enumerate( self.clean_html )
        while 1:
            end_idx, val, next_b = read_to( idx_bytes, '<' )
            tag_depth += 1

            if href:
                ## must be inside an anchor tag, so accumulate the
                ## whole anchor
                assert a_tag_depth > 0, (href, self.clean_html)
                vals.append(val)

            ## figure out if start of an "A" anchor tag or close
            ## of a previous tag
            idx, next_b1 = idx_bytes.next()
            if next_b1.lower() == 'a':
                ## could be start of "A" tag
                idx, next_b2 = idx_bytes.next()
                if next_b2 == ' ':
                    a_tag_depth += 1

                    href = None
                    for idx, attr_name, attr_val in iter_attrs( idx_bytes ):
                        if attr_name.lower() == 'href':
                            href = attr_val
                        if idx is None:
                            ## doc ended mid tag, so invalid HTML--> just bail
                            return

                    first = idx + 1

                    ## if we got an href, then we want to keep the
                    ## first byte idx of the anchor:
                    if href:
                        ## Someone could nest an A tag inside another
                        ## A tag, which is invalid (even in HTML5), so
                        ## vals could be nonempty.  We only generate
                        ## the leaf-level A tags in these rare cases
                        ## of nested A tags, so reset it:
                        vals = []

            elif next_b1 == '/':
                idx, next_b1 = idx_bytes.next()
                if next_b1 == 'a':
                    ## could be end of "A" tag
                    idx, next_b2 = idx_bytes.next()
                    if next_b2 == '>':
                        a_tag_depth -= 1
                        if href:
                            ## join is much faster than using += above
                            anchor = b''.join(vals)
                            length = len(anchor)

                            ## yield the data
                            yield href, first, len(anchor), anchor

                            ## reset, no yield again in a nested A tag
                            href = None

            else:
                ## the next_b was not part of </a> or a nested <a tag,
                ## so keep it in the output
                vals.append(next_b)
def byte_href_anchors_state_machine(self):
        '''
        byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
        '''
        tag_depth = 0
        a_tag_depth = 0
        vals = []
        href = None
        idx_bytes = enumerate( self.clean_html )
        while 1:
            end_idx, val, next_b = read_to( idx_bytes, '<' )
            tag_depth += 1

            if href:
                ## must be inside an anchor tag, so accumulate the
                ## whole anchor
                assert a_tag_depth > 0, (href, self.clean_html)
                vals.append(val)

            ## figure out if start of an "A" anchor tag or close
            ## of a previous tag
            idx, next_b1 = idx_bytes.next()
            if next_b1.lower() == 'a':
                ## could be start of "A" tag
                idx, next_b2 = idx_bytes.next()
                if next_b2 == ' ':
                    a_tag_depth += 1

                    href = None
                    for idx, attr_name, attr_val in iter_attrs( idx_bytes ):
                        if attr_name.lower() == 'href':
                            href = attr_val
                        if idx is None:
                            ## doc ended mid tag, so invalid HTML--> just bail
                            return

                    first = idx + 1

                    ## if we got an href, then we want to keep the
                    ## first byte idx of the anchor:
                    if href:
                        ## Someone could nest an A tag inside another
                        ## A tag, which is invalid (even in HTML5), so
                        ## vals could be nonempty.  We only generate
                        ## the leaf-level A tags in these rare cases
                        ## of nested A tags, so reset it:
                        vals = []

            elif next_b1 == '/':
                idx, next_b1 = idx_bytes.next()
                if next_b1 == 'a':
                    ## could be end of "A" tag
                    idx, next_b2 = idx_bytes.next()
                    if next_b2 == '>':
                        a_tag_depth -= 1
                        if href:
                            ## join is much faster than using += above
                            anchor = b''.join(vals)
                            length = len(anchor)

                            ## yield the data
                            yield href, first, len(anchor), anchor

                            ## reset, no yield again in a nested A tag
                            href = None

            else:
                ## the next_b was not part of </a> or a nested <a tag,
                ## so keep it in the output
                vals.append(next_b)
def byte_href_anchors_state_machine ( self ) : tag_depth = 0 a_tag_depth = 0 vals = [ ] href = None idx_bytes = enumerate ( self . clean_html ) while 1 : end_idx , val , next_b = read_to ( idx_bytes , '<' ) tag_depth += 1 if href : ## must be inside an anchor tag, so accumulate the ## whole anchor assert a_tag_depth > 0 , ( href , self . clean_html ) vals . append ( val ) ## figure out if start of an "A" anchor tag or close ## of a previous tag idx , next_b1 = idx_bytes . next ( ) if next_b1 . lower ( ) == 'a' : ## could be start of "A" tag idx , next_b2 = idx_bytes . next ( ) if next_b2 == ' ' : a_tag_depth += 1 href = None for idx , attr_name , attr_val in iter_attrs ( idx_bytes ) : if attr_name . lower ( ) == 'href' : href = attr_val if idx is None : ## doc ended mid tag, so invalid HTML--> just bail return first = idx + 1 ## if we got an href, then we want to keep the ## first byte idx of the anchor: if href : ## Someone could nest an A tag inside another ## A tag, which is invalid (even in HTML5), so ## vals could be nonempty.  We only generate ## the leaf-level A tags in these rare cases ## of nested A tags, so reset it: vals = [ ] elif next_b1 == '/' : idx , next_b1 = idx_bytes . next ( ) if next_b1 == 'a' : ## could be end of "A" tag idx , next_b2 = idx_bytes . next ( ) if next_b2 == '>' : a_tag_depth -= 1 if href : ## join is much faster than using += above anchor = b'' . join ( vals ) length = len ( anchor ) ## yield the data yield href , first , len ( anchor ) , anchor ## reset, no yield again in a nested A tag href = None else : ## the next_b was not part of </a> or a nested <a tag, ## so keep it in the output vals . append ( next_b )
byte-based state machine extractor of anchor tags, so we can
        compute byte offsets for anchor texts and associate them with
        their href.
        
        Generates tuple(href_string, first_byte, byte_length, anchor_text)
byte - based state machine extractor of anchor tags so we can compute byte offsets for anchor texts and associate them with their href . Generates tuple ( href_string first_byte byte_length anchor_text )