streamcorpus_pipeline/_clean_visible.py
def make_clean_visible_from_raw(_html, tag_replacement_char=' '):
    '''Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
    returns a Unicode string with all tags replaced by whitespace. In
    particular, all Unicode characters inside HTML are replaced with a
    single whitespace character.

    This *does* detect comments, style, script, link tags and replaces
    them with whitespace.  This is subtle because these tags can be
    self-closing or not.

    It does do anything with HTML-escaped characters.

    Pre-existing whitespace of any kind *except* newlines (\n) and
    linefeeds (\r\n) is converted to single spaces ' ', which has the
    same byte length (and character length).  Newlines and linefeeds
    are left unchanged.

    This is a simple state machine iterator without regexes

    '''
    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    #Strip tags with logic above
    non_tag = ''.join(non_tag_chars_from_raw(_html))

    return non_tag.encode('utf-8')
def make_clean_visible_from_raw(_html, tag_replacement_char=' '):
    '''Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
    returns a Unicode string with all tags replaced by whitespace. In
    particular, all Unicode characters inside HTML are replaced with a
    single whitespace character.

    This *does* detect comments, style, script, link tags and replaces
    them with whitespace.  This is subtle because these tags can be
    self-closing or not.

    It does do anything with HTML-escaped characters.

    Pre-existing whitespace of any kind *except* newlines (\n) and
    linefeeds (\r\n) is converted to single spaces ' ', which has the
    same byte length (and character length).  Newlines and linefeeds
    are left unchanged.

    This is a simple state machine iterator without regexes

    '''
    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    #Strip tags with logic above
    non_tag = ''.join(non_tag_chars_from_raw(_html))

    return non_tag.encode('utf-8')
def make_clean_visible_from_raw ( _html , tag_replacement_char = ' ' ) : if not isinstance ( _html , unicode ) : _html = unicode ( _html , 'utf-8' ) #Strip tags with logic above non_tag = '' . join ( non_tag_chars_from_raw ( _html ) ) return non_tag . encode ( 'utf-8' )
Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
    returns a Unicode string with all tags replaced by whitespace. In
    particular, all Unicode characters inside HTML are replaced with a
    single whitespace character.

    This *does* detect comments, style, script, link tags and replaces
    them with whitespace.  This is subtle because these tags can be
    self-closing or not.

    It does do anything with HTML-escaped characters.

    Pre-existing whitespace of any kind *except* newlines (\n) and
    linefeeds (\r\n) is converted to single spaces ' ', which has the
    same byte length (and character length).  Newlines and linefeeds
    are left unchanged.

    This is a simple state machine iterator without regexes
Takes an HTML - like Unicode ( or UTF - 8 encoded ) string as input and returns a Unicode string with all tags replaced by whitespace . In particular all Unicode characters inside HTML are replaced with a single whitespace character .