scripts/scraper.py
def fetch_articles(self, page=1):
        """
        Fetches all of the contents of the articles from the API and saves the
        HTMLised version to disk.
        """

        max_articles_to_fetch = 200

        fetched_articles = self.fetch_page_of_articles(
            page=page, page_size=max_articles_to_fetch
        )

        if fetched_articles is False:
            raise ScraperError("Error when fetching data from API.")

        for article in fetched_articles:
            # We'll put any tags we want to keep in article itself:
            tags = article["tags"]
            del article["tags"]

            if len(tags) == 0:
                self.message(
                    "* %s has no tags; unable to put into section." % article["id"]
                )
                continue

            if "newspaperPageNumber" not in article["fields"]:
                self.message(
                    "* %s has no page number; unable to put into section."
                    % article["id"]
                )
                continue

            # Just get the dicts for book and book_section out of the tags list.
            book = next((tag for tag in tags if tag["type"] == "newspaper-book"), None)
            book_section = next(
                (tag for tag in tags if tag["type"] == "newspaper-book-section"), None
            )
            # There might be more than contributor; we're only using one:
            contributor = next(
                (tag for tag in tags if tag["type"] == "contributor"), None
            )

            if book is None:
                self.message(
                    "* %s has no 'book' tag; unable to put into section."
                    % article["id"]
                )
                continue

            # Save these in easy to get places for the template:
            article["newspaperBook"] = book
            article["newspaperBookSection"] = book_section
            article["contributor"] = contributor

            article["tone"] = self.get_tone(tags)

            if book["id"] not in self.fetched_books:
                self.fetched_books[book["id"]] = {
                    "meta": book,
                    "articles": [],
                }

            # Make page number into an int.
            article["fields"]["newspaperPageNumber"] = int(
                article["fields"]["newspaperPageNumber"]
            )

            # Make wordcount int, and increase issue's max wordcount if appropriate.
            article["fields"]["wordcount"] = int(article["fields"]["wordcount"])
            words = article["fields"]["wordcount"]
            if words > self.contents["meta"]["max_words"]:
                self.contents["meta"]["max_words"] = words

            # Save file and store its filename:
            article["file"] = self.save_article_html(article)

            self.fetched_books[book["id"]]["articles"].append(article)

        if len(fetched_articles) >= max_articles_to_fetch:
            # We fetched the maximum, so there might be another page.
            # Call this same method again:
            # Pause, be nice.
            time.sleep(1)
            self.fetch_articles(page=page + 1)

        Fetches all of the contents of the articles from the API and saves the
        HTMLised version to disk.
        
Fetches all of the contents of the articles from the API and saves the
HTMLised version to disk.
Fetches all of the contents of the articles from the API and saves the HTMLised version to disk .
def fetch_articles(self, page=1):
        max_articles_to_fetch = 200
        fetched_articles = self.fetch_page_of_articles(
            page=page, page_size=max_articles_to_fetch
        )
        if fetched_articles is False:
            raise ScraperError("Error when fetching data from API.")
        for article in fetched_articles:
            tags = article["tags"]
            del article["tags"]
            if len(tags) == 0:
                self.message(
                    "* %s has no tags; unable to put into section." % article["id"]
                )
                continue
            if "newspaperPageNumber" not in article["fields"]:
                self.message(
                    "* %s has no page number; unable to put into section."
                    % article["id"]
                )
                continue
            book = next((tag for tag in tags if tag["type"] == "newspaper-book"), None)
            book_section = next(
                (tag for tag in tags if tag["type"] == "newspaper-book-section"), None
            )
            contributor = next(
                (tag for tag in tags if tag["type"] == "contributor"), None
            )
            if book is None:
                self.message(
                    "* %s has no 'book' tag; unable to put into section."
                    % article["id"]
                )
                continue
            article["newspaperBook"] = book
            article["newspaperBookSection"] = book_section
            article["contributor"] = contributor
            article["tone"] = self.get_tone(tags)
            if book["id"] not in self.fetched_books:
                self.fetched_books[book["id"]] = {
                    "meta": book,
                    "articles": [],
                }
            article["fields"]["newspaperPageNumber"] = int(
                article["fields"]["newspaperPageNumber"]
            )
            article["fields"]["wordcount"] = int(article["fields"]["wordcount"])
            words = article["fields"]["wordcount"]
            if words > self.contents["meta"]["max_words"]:
                self.contents["meta"]["max_words"] = words
            article["file"] = self.save_article_html(article)
            self.fetched_books[book["id"]]["articles"].append(article)
        if len(fetched_articles) >= max_articles_to_fetch:
            time.sleep(1)
            self.fetch_articles(page=page + 1)
def fetch_articles ( self , page = 1 ) : max_articles_to_fetch = 200 fetched_articles = self . fetch_page_of_articles ( page = page , page_size = max_articles_to_fetch ) if fetched_articles is False : raise ScraperError ( "Error when fetching data from API." ) for article in fetched_articles : tags = article [ "tags" ] del article [ "tags" ] if len ( tags ) == 0 : self . message ( "* %s has no tags; unable to put into section." % article [ "id" ] ) continue if "newspaperPageNumber" not in article [ "fields" ] : self . message ( "* %s has no page number; unable to put into section." % article [ "id" ] ) continue book = next ( ( tag for tag in tags if tag [ "type" ] == "newspaper-book" ) , None ) book_section = next ( ( tag for tag in tags if tag [ "type" ] == "newspaper-book-section" ) , None ) contributor = next ( ( tag for tag in tags if tag [ "type" ] == "contributor" ) , None ) if book is None : self . message ( "* %s has no 'book' tag; unable to put into section." % article [ "id" ] ) continue article [ "newspaperBook" ] = book article [ "newspaperBookSection" ] = book_section article [ "contributor" ] = contributor article [ "tone" ] = self . get_tone ( tags ) if book [ "id" ] not in self . fetched_books : self . fetched_books [ book [ "id" ] ] = { "meta" : book , "articles" : [ ] , } article [ "fields" ] [ "newspaperPageNumber" ] = int ( article [ "fields" ] [ "newspaperPageNumber" ] ) article [ "fields" ] [ "wordcount" ] = int ( article [ "fields" ] [ "wordcount" ] ) words = article [ "fields" ] [ "wordcount" ] if words > self . contents [ "meta" ] [ "max_words" ] : self . contents [ "meta" ] [ "max_words" ] = words article [ "file" ] = self . save_article_html ( article ) self . fetched_books [ book [ "id" ] ] [ "articles" ] . append ( article ) if len ( fetched_articles ) >= max_articles_to_fetch : time . sleep ( 1 ) self . fetch_articles ( page = page + 1 )
Fetches all of the contents of the articles from the API and saves the
HTMLised version to disk.
Fetches all of the contents of the articles from the API and saves the HTMLised version to disk .
"""
        Fetches all of the contents of the articles from the API and saves the
        HTMLised version to disk.
        """ # We'll put any tags we want to keep in article itself: # Just get the dicts for book and book_section out of the tags list. # There might be more than contributor; we're only using one: # Save these in easy to get places for the template: # Make page number into an int. # Make wordcount int, and increase issue's max wordcount if appropriate. # Save file and store its filename: # We fetched the maximum, so there might be another page. # Call this same method again: # Pause, be nice.
self
page