source/io/web_scraper.py
def download_article(article_title, language):
    """Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same"""
    response = requests.get(
    'https://'+language+'.wikipedia.org/w/api.php',
    params={
    'action': 'query',
    'titles': article_title,
    'format': 'json',
    'prop': 'extracts', #https://www.mediawiki.org/w/api.php?action=help&modules=query%2Bextracts
    'exsectionformat': 'plain',
    },
    headers = generate_api_header()
    ).json()
    page = next(iter(response['query']['pages'].values()))
    html_page = page['extract']
    
    html_page = html_page.replace('<h2>',' _HEADER2START_ ').replace('</h2>',' _HEADER2END_ ').replace('<h3>',
                        ' _HEADER3START_ ').replace('</h3>',' _HEADER3END_ ').replace('<b>',
                        ' _BOLDSTART_ ').replace('</b>',' _BOLDEND_ ').replace("<i>",' _ITALICSTART_ ').replace("</i>",
                        ' _ITALICEND_ ').replace("<p>",' _PARAGRAPHSTART_ ').replace("</p>",' _PARAGRAPHEND_ ')
    parsed_html = BeautifulSoup(html_page, 'html.parser')
    text = parsed_html.get_text()

    return text
Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same
Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same
Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same
def download_article(article_title, language):
    response = requests.get(
    'https://'+language+'.wikipedia.org/w/api.php',
    params={
    'action': 'query',
    'titles': article_title,
    'format': 'json',
    'prop': 'extracts', 
    'exsectionformat': 'plain',
    },
    headers = generate_api_header()
    ).json()
    page = next(iter(response['query']['pages'].values()))
    html_page = page['extract']
    html_page = html_page.replace('<h2>',' _HEADER2START_ ').replace('</h2>',' _HEADER2END_ ').replace('<h3>',
                        ' _HEADER3START_ ').replace('</h3>',' _HEADER3END_ ').replace('<b>',
                        ' _BOLDSTART_ ').replace('</b>',' _BOLDEND_ ').replace("<i>",' _ITALICSTART_ ').replace("</i>",
                        ' _ITALICEND_ ').replace("<p>",' _PARAGRAPHSTART_ ').replace("</p>",' _PARAGRAPHEND_ ')
    parsed_html = BeautifulSoup(html_page, 'html.parser')
    text = parsed_html.get_text()
    return text
def download_article ( article_title , language ) : response = requests . get ( 'https://' + language + '.wikipedia.org/w/api.php' , params = { 'action' : 'query' , 'titles' : article_title , 'format' : 'json' , 'prop' : 'extracts' , 'exsectionformat' : 'plain' , } , headers = generate_api_header ( ) ) . json ( ) page = next ( iter ( response [ 'query' ] [ 'pages' ] . values ( ) ) ) html_page = page [ 'extract' ] html_page = html_page . replace ( '<h2>' , ' _HEADER2START_ ' ) . replace ( '</h2>' , ' _HEADER2END_ ' ) . replace ( '<h3>' , ' _HEADER3START_ ' ) . replace ( '</h3>' , ' _HEADER3END_ ' ) . replace ( '<b>' , ' _BOLDSTART_ ' ) . replace ( '</b>' , ' _BOLDEND_ ' ) . replace ( "<i>" , ' _ITALICSTART_ ' ) . replace ( "</i>" , ' _ITALICEND_ ' ) . replace ( "<p>" , ' _PARAGRAPHSTART_ ' ) . replace ( "</p>" , ' _PARAGRAPHEND_ ' ) parsed_html = BeautifulSoup ( html_page , 'html.parser' ) text = parsed_html . get_text ( ) return text
Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same
Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same
"""Download the Wikipedia article text and mark safe HTML elements with codes so they remain the same""" #https://www.mediawiki.org/w/api.php?action=help&modules=query%2Bextracts
article_title
language