scripts/scraper.py
def fetch_page_of_articles(self, page=1, page_size=200):
        """Fetches a single set of articles from today's issue.
        Returns a list of dicts, each dict an article's data.
        Or False if there was an error.
        """

        api_url = "http://content.guardianapis.com/search"

        url_args = {
            "page": page,
            "page-size": page_size,
            "api-key": self.guardian_api_key,
            "format": "json",
            "show-fields": (
                "body,byline,headline,newspaperPageNumber,publication,shortUrl,"
                "standfirst,thumbnail,wordcount"
            ),
            "show-elements": "all",
            "show-tags": "contributor,newspaper-book,newspaper-book-section,tone",
            # Get the articles from today's edition:
            "use-date": "newspaper-edition",
            "from-date": self.issue_date.strftime("%Y-%m-%d"),
            "to-date": self.issue_date.strftime("%Y-%m-%d"),
            # The most we can fetch per page:
        }

        self.message("Fetching page %s of up to %s articles." % (page, page_size))

        error_message = ""
        response = None

        try:
            response = requests.get(api_url, params=url_args, timeout=20)
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            error_message = "HTTP Error: %s" % response.status_code
        except requests.exceptions.ConnectionError as e:
            error_message = "Can't connect to domain: %s" % e
        except requests.exceptions.ConnectTimeout as e:
            error_message = "Connection timed out: %s" % e
        except requests.exceptions.ReadTimeout as e:
            error_message = "Read timed out: %s" % e
        except requests.exceptions.RequestException as e:
            # Catches any other requests exceptions.
            error_message = "RequestException: %s" % e

        if error_message == "":
            # All good so far. Check the returned data.
            data = response.json()
            if (
                "response" in data
                and "status" in data["response"]
                and "results" in data["response"]
            ):
                if data["response"]["status"] != "ok":
                    error_message = (
                        "The API returned the status '%s'" % data["response"]["ok"]
                    )
            else:
                error_message = "The returned data was not the expected format."

        if error_message == "":
            # Still OK!
            articles = data["response"]["results"]
            self.message("Fetched %s articles." % len(articles))
            return articles
        else:
            self.message("ERROR: %s" % error_message)
            return False
Fetches a single set of articles from today's issue.
        Returns a list of dicts, each dict an article's data.
        Or False if there was an error.
        
Fetches a single set of articles from today's issue.
Returns a list of dicts, each dict an article's data.
Or False if there was an error.
Fetches a single set of articles from today ' s issue . Returns a list of dicts each dict an article ' s data . Or False if there was an error .
def fetch_page_of_articles(self, page=1, page_size=200):
        api_url = "http://content.guardianapis.com/search"
        url_args = {
            "page": page,
            "page-size": page_size,
            "api-key": self.guardian_api_key,
            "format": "json",
            "show-fields": (
                "body,byline,headline,newspaperPageNumber,publication,shortUrl,"
                "standfirst,thumbnail,wordcount"
            ),
            "show-elements": "all",
            "show-tags": "contributor,newspaper-book,newspaper-book-section,tone",
            "use-date": "newspaper-edition",
            "from-date": self.issue_date.strftime("%Y-%m-%d"),
            "to-date": self.issue_date.strftime("%Y-%m-%d"),
        }
        self.message("Fetching page %s of up to %s articles." % (page, page_size))
        error_message = ""
        response = None
        try:
            response = requests.get(api_url, params=url_args, timeout=20)
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            error_message = "HTTP Error: %s" % response.status_code
        except requests.exceptions.ConnectionError as e:
            error_message = "Can't connect to domain: %s" % e
        except requests.exceptions.ConnectTimeout as e:
            error_message = "Connection timed out: %s" % e
        except requests.exceptions.ReadTimeout as e:
            error_message = "Read timed out: %s" % e
        except requests.exceptions.RequestException as e:
            error_message = "RequestException: %s" % e
        if error_message == "":
            data = response.json()
            if (
                "response" in data
                and "status" in data["response"]
                and "results" in data["response"]
            ):
                if data["response"]["status"] != "ok":
                    error_message = (
                        "The API returned the status '%s'" % data["response"]["ok"]
                    )
            else:
                error_message = "The returned data was not the expected format."
        if error_message == "":
            articles = data["response"]["results"]
            self.message("Fetched %s articles." % len(articles))
            return articles
        else:
            self.message("ERROR: %s" % error_message)
            return False
def fetch_page_of_articles ( self , page = 1 , page_size = 200 ) : api_url = "http://content.guardianapis.com/search" url_args = { "page" : page , "page-size" : page_size , "api-key" : self . guardian_api_key , "format" : "json" , "show-fields" : ( "body,byline,headline,newspaperPageNumber,publication,shortUrl," "standfirst,thumbnail,wordcount" ) , "show-elements" : "all" , "show-tags" : "contributor,newspaper-book,newspaper-book-section,tone" , "use-date" : "newspaper-edition" , "from-date" : self . issue_date . strftime ( "%Y-%m-%d" ) , "to-date" : self . issue_date . strftime ( "%Y-%m-%d" ) , } self . message ( "Fetching page %s of up to %s articles." % ( page , page_size ) ) error_message = "" response = None try : response = requests . get ( api_url , params = url_args , timeout = 20 ) response . raise_for_status ( ) except requests . exceptions . HTTPError : error_message = "HTTP Error: %s" % response . status_code except requests . exceptions . ConnectionError as e : error_message = "Can't connect to domain: %s" % e except requests . exceptions . ConnectTimeout as e : error_message = "Connection timed out: %s" % e except requests . exceptions . ReadTimeout as e : error_message = "Read timed out: %s" % e except requests . exceptions . RequestException as e : error_message = "RequestException: %s" % e if error_message == "" : data = response . json ( ) if ( "response" in data and "status" in data [ "response" ] and "results" in data [ "response" ] ) : if data [ "response" ] [ "status" ] != "ok" : error_message = ( "The API returned the status '%s'" % data [ "response" ] [ "ok" ] ) else : error_message = "The returned data was not the expected format." if error_message == "" : articles = data [ "response" ] [ "results" ] self . message ( "Fetched %s articles." % len ( articles ) ) return articles else : self . message ( "ERROR: %s" % error_message ) return False
Fetches a single set of articles from today's issue.
Fetches a single set of articles from today ' s issue .
"""Fetches a single set of articles from today's issue.
        Returns a list of dicts, each dict an article's data.
        Or False if there was an error.
        """ # Get the articles from today's edition: # The most we can fetch per page: # Catches any other requests exceptions. # All good so far. Check the returned data. # Still OK!
self
page
page_size