Python multiple domain crawler InvalidSchema exception
Question: This is my code. My goal is crawl multiple domain. I set domains in url array
but I can't crawl.

The code can find urls but don't parse or crawl.

This is the results: my code run ('Total Link Number : ', 387) ('Links for
News: ', 146)

    
    
    # -*- coding: utf-8 -*-
    import requests
    from bs4 import BeautifulSoup
    import codecs
    
    headers = {
        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5)",
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "accept-charset": "cp1254,ISO-8859-9,utf-8;q=0.7,*;q=0.3",
        "accept-encoding": "gzip,deflate,sdch",
        "accept-language": "tr,tr-TR,en-US,en;q=0.8",
    }
    
    def haber_oku(haber_url):
        r = requests.get(haber_url, headers=headers)
        if r.status_code != 200:
            return
        soup = BeautifulSoup(r.content)
        result = soup.find("div", {'itemprop': 'articleBody'})
        if result:
            return result.get_text()
        else:
            result = soup.find("div", {'itemprop': 'description'})
            if result:
                return result.get_text()
        return
    
    def scrape_hurriyet(keywords, detay_goster, url):
    
        if len(keywords) > 0:
            keywords = keywords.split(',')
        s = 0
    
        r = requests.get(url, headers=headers)
        if r.status_code != 200:
            print("request reddedildi")
            return
    
        soup = BeautifulSoup(r.content)
        results = soup.findAll("a")
        print ("Toplam link sayisi : ", len(results))
        liste_link = []
        liste_text = []
        haberler = []
        for result in results:
            h = result.get('href')
            t = result.get_text()
            if h is not None:
                if str(h).find('http://www.hurriyet.com.tr/') or str(h).find('http://www.milliyet.com.tr/spor') >= 0:
                    if h not in liste_link:
                        if h.find('.asp') or h.find('.htm') > 0:
                            liste_link.append(h)
                            liste_text.append(t)
    
        print ("Tekil linkler: ", len(liste_link))
        i = 0
        while i < len(liste_link):
            h = liste_link[i]
            t = liste_text[i]
            haber = haber_oku(h)
    
            if haber is not None:
                haber = BeautifulSoup(haber).get_text()
                ok = 0
                found = ""
    
                if len(keywords) == 0:
                    haberler.append(haber)
    
                else:
                    for keyword in keywords:
                        print ('----------------------')
                        if haber.find(keyword) >= 0:
                            found = found + " " + keyword
                            ok += 1
                        if ok > 0:
                            print ("3", h, t, found)
                        if detay_goster is True:
                            haberler.append(haber)
            i += 1
        k = 0
        while k < len(haberler):
            f = codecs.open("abc" + str(k+1) + ".txt", encoding='utf-8', mode='w+')
            f.write(haberler[k])
            k += 1
        f.close()
    
    keywords = ''
    url = ['http://www.hurriyet.com.tr/', 'http://www.milliyet.com.tr/']
    s = 0
    while s < len(url):
        scrape_hurriyet(keywords, True, url[s])
        s += 1
    

They are exceptions:

    
    
    Traceback (most recent call last):
      File "C:/Users/KerimCaner/PycharmProjects/Hurriyet/hurriyet.py", line 94, in <module>
        scrape_hurriyet(keywords, True, url[s])
      File "C:/Users/KerimCaner/PycharmProjects/Hurriyet/hurriyet.py", line 62, in scrape_hurriyet
        haber = haber_oku(h)
      File "C:/Users/KerimCaner/PycharmProjects/Hurriyet/hurriyet.py", line 17, in haber_oku
        r = requests.get(haber_url, headers=headers)
      File "C:\Users\KerimCaner\AppData\Roaming\Python\Python27\site-packages\requests\api.py", line 69, in get
        return request('get', url, params=params, **kwargs)
      File "C:\Users\KerimCaner\AppData\Roaming\Python\Python27\site-packages\requests\api.py", line 50, in request
        response = session.request(method=method, url=url, **kwargs)
      File "C:\Users\KerimCaner\AppData\Roaming\Python\Python27\site-packages\requests\sessions.py", line 465, in request
        resp = self.send(prep, **send_kwargs)
      File "C:\Users\KerimCaner\AppData\Roaming\Python\Python27\site-packages\requests\sessions.py", line 567, in send
        adapter = self.get_adapter(url=request.url)
      File "C:\Users\KerimCaner\AppData\Roaming\Python\Python27\site-packages\requests\sessions.py", line 641, in get_adapter
        raise InvalidSchema("No connection adapters were found for '%s'" % url)
    requests.exceptions.InvalidSchema: No connection adapters were found for 'javascript:;'
    


Answer: The error you got: `requests.exceptions.InvalidSchema: No connection adapters
were found for 'javascript:;'` states that you are trying to crawl a piece of
javascript. You are currently crawling all url's in anchor-tags, but you need
to filter out javascript-url's. You should replace the following line:

    
    
        if h is not None:
    

with something like this:

    
    
        if h is not None and not(h.startswith("javascript")):
    

