utils/multiproc_fetcher.py
def _select_by_search(self, page, max_pages):
        """
        Select pages inside a given website using site search

        Args:
            max_pages: maximum number of pages selected in the site
        """
        "Selecing pages using bing search"
        host =  page.get_host()
        keyword = "gun"
        urls = self.bing_search.search_site(keyword, host)
        ret = []
        for url in urls:
            if len(ret)<max_pages:
                if url!=page.get_url():
                    ret.append(url)
        """
        if host=="http://www.armslist.com/":
            print urls
            print ret
            print max_pages
        """
        return ret

        Select pages inside a given website using site search

        Args:
            max_pages: maximum number of pages selected in the site
        
Select pages inside a given website using site search
Select pages inside a given website using site search
def _select_by_search(self, page, max_pages):
        host =  page.get_host()
        keyword = "gun"
        urls = self.bing_search.search_site(keyword, host)
        ret = []
        for url in urls:
            if len(ret)<max_pages:
                if url!=page.get_url():
                    ret.append(url)
        return ret
def _select_by_search ( self , page , max_pages ) : "Selecing pages using bing search" host = page . get_host ( ) keyword = "gun" urls = self . bing_search . search_site ( keyword , host ) ret = [ ] for url in urls : if len ( ret ) < max_pages : if url != page . get_url ( ) : ret . append ( url ) """
        if host=="http://www.armslist.com/":
            print urls
            print ret
            print max_pages
        """ return ret
Select pages inside a given website using site search
Select pages inside a given website using site search
"""
        Select pages inside a given website using site search

        Args:
            max_pages: maximum number of pages selected in the site
        """ "Selecing pages using bing search" """
        if host=="http://www.armslist.com/":
            print urls
            print ret
            print max_pages
        """
self
page
max_pages
maximum number of pages selected in the site
maximum number of pages selected in the site