source/io/web_scraper.py
def remove_junk(text):
    """Get rid of HTML and script tags in the citation text"""
    output = ''
    #Rm tags and scripts
    word_blacklist = [
        '[document]',
        'noscript',
        'header',
        'html',
        'meta',
        'head',
        'input',
        'script',
        'footer',
        'style',
    ]

    for word in text:
        if word.parent.name not in word_blacklist:
            output += '{} '.format(word)

    #Strip tabs, newlines, etc
    output = re.sub(r'(^[ \t]+|[ \t]+(?=:))', '', output, flags=re.M)
    output = " ".join(output.split())

    return output.strip()
Get rid of HTML and script tags in the citation text
Get rid of HTML and script tags in the citation text
Get rid of HTML and script tags in the citation text
def remove_junk(text):
    output = ''
    word_blacklist = [
        '[document]',
        'noscript',
        'header',
        'html',
        'meta',
        'head',
        'input',
        'script',
        'footer',
        'style',
    ]
    for word in text:
        if word.parent.name not in word_blacklist:
            output += '{} '.format(word)
    output = re.sub(r'(^[ \t]+|[ \t]+(?=:))', '', output, flags=re.M)
    output = " ".join(output.split())
    return output.strip()
def remove_junk ( text ) : output = '' word_blacklist = [ '[document]' , 'noscript' , 'header' , 'html' , 'meta' , 'head' , 'input' , 'script' , 'footer' , 'style' , ] for word in text : if word . parent . name not in word_blacklist : output += '{} ' . format ( word ) output = re . sub ( r'(^[ \t]+|[ \t]+(?=:))' , '' , output , flags = re . M ) output = " " . join ( output . split ( ) ) return output . strip ( )
Get rid of HTML and script tags in the citation text
Get rid of HTML and script tags in the citation text
"""Get rid of HTML and script tags in the citation text""" #Rm tags and scripts #Strip tabs, newlines, etc
text