simple_site_crawler/site_crawler.py
def add_page(self, url):
        """
        Add webpage to the queue if it wasn't crawled before and belongs to
        the root domain

        :param url: webpage url
        :type url: str
        """
        domain = urlsplit(url).netloc

        if url not in self.crawled_pages and domain == self.root_domain:
            # Make sure we don't the add same URL to the queue twice
            self.crawled_pages[url] = None
            self.queue.put_nowait(url)

        Add webpage to the queue if it wasn't crawled before and belongs to
        the root domain

        :param url: webpage url
        :type url: str
        
Add webpage to the queue if it wasn't crawled before and belongs to
the root domain
Add webpage to the queue if it wasn ' t crawled before and belongs to the root domain
def add_page(self, url):
        domain = urlsplit(url).netloc
        if url not in self.crawled_pages and domain == self.root_domain:
            self.crawled_pages[url] = None
            self.queue.put_nowait(url)
def add_page ( self , url ) : domain = urlsplit ( url ) . netloc if url not in self . crawled_pages and domain == self . root_domain : self . crawled_pages [ url ] = None self . queue . put_nowait ( url )
Add webpage to the queue if it wasn't crawled before and belongs to
the root domain
Add webpage to the queue if it wasn ' t crawled before and belongs to the root domain
"""
        Add webpage to the queue if it wasn't crawled before and belongs to
        the root domain

        :param url: webpage url
        :type url: str
        """ # Make sure we don't the add same URL to the queue twice
self
url
None