harvester_odt/comunweb/crawler.py
def _scan_pages(self, start_url):
        """
        Keep downloading pages from a paged API request and yield
        objects found in each page, until the end is reached.

        Each yielded item is a dict like this::

            {
            "classIdentifier": "open_data",
            "dateModified": 1399274108,
            "datePublished": 1399240800,
            "fullUrl": "http://www.comune.trento.it/Comune/Documenti/Bilanci/"
                       "Bilanci-di-rendicontazione/Rendiconti-di-gestione/"
                       "Rendiconto-del-2013/Rendiconto-del-2013-Open-Data",
            "link": "http://www.comune.trento.it"
                    "/api/opendata/v1/content/node/831248",
            "nodeId": 831248,
            "nodeRemoteId": "dc04403fe707a2b5f36efba071bd119e",
            "objectId": 849404,
            "objectName": "Rendiconto del 2013 (Open Data)",
            "objectRemoteId": "260e6a5ebdc2e6319f3353a0d9b2f5bd"
            },

        """

        offset, limit = 0, 50

        while True:
            page_url = '{0}/offset/{1}/limit/{2}'.format(
                start_url.rstrip('/'), offset, limit)
            response = requests.get(page_url)
            nodes = response.json()['nodes']

            if len(nodes) < 1:
                # This was the last page
                return

            for item in nodes:
                yield item
                offset += 1

        Keep downloading pages from a paged API request and yield
        objects found in each page, until the end is reached.

        Each yielded item is a dict like this::

            {
            "classIdentifier": "open_data",
            "dateModified": 1399274108,
            "datePublished": 1399240800,
            "fullUrl": "http://www.comune.trento.it/Comune/Documenti/Bilanci/"
                       "Bilanci-di-rendicontazione/Rendiconti-di-gestione/"
                       "Rendiconto-del-2013/Rendiconto-del-2013-Open-Data",
            "link": "http://www.comune.trento.it"
                    "/api/opendata/v1/content/node/831248",
            "nodeId": 831248,
            "nodeRemoteId": "dc04403fe707a2b5f36efba071bd119e",
            "objectId": 849404,
            "objectName": "Rendiconto del 2013 (Open Data)",
            "objectRemoteId": "260e6a5ebdc2e6319f3353a0d9b2f5bd"
            },

        
Keep downloading pages from a paged API request and yield
objects found in each page, until the end is reached.

Each yielded item is a dict like this:.


Keep downloading pages from a paged API request and yield objects found in each page until the end is reached . Each yielded item is a dict like this : .
def _scan_pages(self, start_url):
        offset, limit = 0, 50
        while True:
            page_url = '{0}/offset/{1}/limit/{2}'.format(
                start_url.rstrip('/'), offset, limit)
            response = requests.get(page_url)
            nodes = response.json()['nodes']
            if len(nodes) < 1:
                return
            for item in nodes:
                yield item
                offset += 1
def _scan_pages ( self , start_url ) : offset , limit = 0 , 50 while True : page_url = '{0}/offset/{1}/limit/{2}' . format ( start_url . rstrip ( '/' ) , offset , limit ) response = requests . get ( page_url ) nodes = response . json ( ) [ 'nodes' ] if len ( nodes ) < 1 : return for item in nodes : yield item offset += 1
Keep downloading pages from a paged API request and yield
objects found in each page, until the end is reached.
Keep downloading pages from a paged API request and yield objects found in each page until the end is reached .
"""
        Keep downloading pages from a paged API request and yield
        objects found in each page, until the end is reached.

        Each yielded item is a dict like this::

            {
            "classIdentifier": "open_data",
            "dateModified": 1399274108,
            "datePublished": 1399240800,
            "fullUrl": "http://www.comune.trento.it/Comune/Documenti/Bilanci/"
                       "Bilanci-di-rendicontazione/Rendiconti-di-gestione/"
                       "Rendiconto-del-2013/Rendiconto-del-2013-Open-Data",
            "link": "http://www.comune.trento.it"
                    "/api/opendata/v1/content/node/831248",
            "nodeId": 831248,
            "nodeRemoteId": "dc04403fe707a2b5f36efba071bd119e",
            "objectId": 849404,
            "objectName": "Rendiconto del 2013 (Open Data)",
            "objectRemoteId": "260e6a5ebdc2e6319f3353a0d9b2f5bd"
            },

        """ # This was the last page
self
start_url