Scrape special characters in Python Beautiful Soup
Question: How can I remove (or encode) the special characters from the page referenced
below?

    
    
    import urllib2
    from bs4 import BeautifulSoup
    import re
    
    link = "https://www.sec.gov/Archives/edgar/data/4281/000119312513062916/R2.htm"
    
    request_headers = {"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Referer": "http://google.com", "Connection": "keep-alive"}
    request = urllib2.Request(link, headers=request_headers)
    html = urllib2.urlopen(request).read()
    soup = BeautifulSoup(html, "html.parser")
    soup = soup.encode('utf-8', 'ignore')
    print(soup)
    


Answer: Unicode objects can only be printed if they can be converted to ASCII. If it
can't be encoded in ASCII, you'll get that error. You probably want to
explicitly encode it and then print the resulting soup:

    
    
    import requests
    from bs4 import BeautifulSoup
    import re
    
    link = "https://www.sec.gov/Archives/edgar/data/4281/000119312513062916/R2.htm"
    
    request_headers = {"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Referer": "http://google.com", "Connection": "keep-alive"}
    reuest = requests.get(link, headers=request_headers)
    soup = BeautifulSoup(reuest.text,"lxml")
    print(soup.encode('utf-8'))
    

