extraction/link_extractor.py
def extract_external_links(self, url, html):
        '''
        Extract external outlinks, that link to different websites
        Returns: 
            - list of unique urls
        '''
        try:
            soup = BeautifulSoup(html, 'lxml')
            links = set()
            tld = URLUtility.get_tld(url)

            for tag in soup.findAll('a', href=True):
                link = tag['href']
                values = urlparse.urlparse(link)
                if (values.netloc=="") or (values.netloc==tld) or (tld in values.netloc):
                    continue
                link = URLUtility.validate_link(link)
                if link:
                    link = URLUtility.normalize(link)
                    if link:
                        links.add(link)
            return list(links)
        except:
            traceback.print_exc()
            return []

        Extract external outlinks, that link to different websites
        Returns: 
            - list of unique urls
        
Extract external outlinks, that link to different websites
Extract external outlinks that link to different websites
def extract_external_links(self, url, html):
        try:
            soup = BeautifulSoup(html, 'lxml')
            links = set()
            tld = URLUtility.get_tld(url)
            for tag in soup.findAll('a', href=True):
                link = tag['href']
                values = urlparse.urlparse(link)
                if (values.netloc=="") or (values.netloc==tld) or (tld in values.netloc):
                    continue
                link = URLUtility.validate_link(link)
                if link:
                    link = URLUtility.normalize(link)
                    if link:
                        links.add(link)
            return list(links)
        except:
            traceback.print_exc()
            return []
def extract_external_links ( self , url , html ) : try : soup = BeautifulSoup ( html , 'lxml' ) links = set ( ) tld = URLUtility . get_tld ( url ) for tag in soup . findAll ( 'a' , href = True ) : link = tag [ 'href' ] values = urlparse . urlparse ( link ) if ( values . netloc == "" ) or ( values . netloc == tld ) or ( tld in values . netloc ) : continue link = URLUtility . validate_link ( link ) if link : link = URLUtility . normalize ( link ) if link : links . add ( link ) return list ( links ) except : traceback . print_exc ( ) return [ ]
Extract external outlinks, that link to different websites
Extract external outlinks that link to different websites
'''
        Extract external outlinks, that link to different websites
        Returns: 
            - list of unique urls
        '''
list of unique urls
list of unique urls
self
url
html