scripts/scraper.py
def start(self):
        """
        The main action. Fetches all of the required data for today's paper and
        saves it locally.
        """
        self.checkForOldProcesses()
        self.makeLockfile()

        # Sets the date and paper (guardian or observer).
        self.set_issue_date()

        if self.issue_date != "":
            self.issue_archive_dir = (
                self.archive_dir + self.issue_date.strftime("%Y-%m-%d") + "/"
            )
        else:
            raise ScraperError(
                "We don't have an issue date, so can't make an archive directory."
            )

        # Make the directory we'll save the HTML files in.
        if not os.path.exists(self.issue_archive_dir):
            os.makedirs(self.issue_archive_dir)

        # Get all of today's articles and save HTML versions to disk.
        self.fetch_articles()

        self.sort_articles()

        # Delete the day-before-yesterday's files.
        # (Not yesterday's, just in case someone is currently viewing them.)
        old_date = self.issue_date - datetime.timedelta(2)
        old_dir = self.archive_dir + old_date.strftime("%Y-%m-%d")
        if os.path.exists(old_dir):
            shutil.rmtree(old_dir)

        # Write all the information about this issue to a contents.json file
        # within the dated folder.
        try:
            with open(self.issue_archive_dir + "contents.json", mode="w") as fp:
                json.dump(self.contents, fp)
        except EnvironmentError:
            raise ScraperError("Unable to write the contents.json file.")

        self.removeLockfile()

        The main action. Fetches all of the required data for today's paper and
        saves it locally.
        
The main action. Fetches all of the required data for today's paper and
saves it locally.
The main action . Fetches all of the required data for today ' s paper and saves it locally .
def start(self):
        self.checkForOldProcesses()
        self.makeLockfile()
        self.set_issue_date()
        if self.issue_date != "":
            self.issue_archive_dir = (
                self.archive_dir + self.issue_date.strftime("%Y-%m-%d") + "/"
            )
        else:
            raise ScraperError(
                "We don't have an issue date, so can't make an archive directory."
            )
        if not os.path.exists(self.issue_archive_dir):
            os.makedirs(self.issue_archive_dir)
        self.fetch_articles()
        self.sort_articles()
        old_date = self.issue_date - datetime.timedelta(2)
        old_dir = self.archive_dir + old_date.strftime("%Y-%m-%d")
        if os.path.exists(old_dir):
            shutil.rmtree(old_dir)
        try:
            with open(self.issue_archive_dir + "contents.json", mode="w") as fp:
                json.dump(self.contents, fp)
        except EnvironmentError:
            raise ScraperError("Unable to write the contents.json file.")
        self.removeLockfile()
def start ( self ) : self . checkForOldProcesses ( ) self . makeLockfile ( ) self . set_issue_date ( ) if self . issue_date != "" : self . issue_archive_dir = ( self . archive_dir + self . issue_date . strftime ( "%Y-%m-%d" ) + "/" ) else : raise ScraperError ( "We don't have an issue date, so can't make an archive directory." ) if not os . path . exists ( self . issue_archive_dir ) : os . makedirs ( self . issue_archive_dir ) self . fetch_articles ( ) self . sort_articles ( ) old_date = self . issue_date - datetime . timedelta ( 2 ) old_dir = self . archive_dir + old_date . strftime ( "%Y-%m-%d" ) if os . path . exists ( old_dir ) : shutil . rmtree ( old_dir ) try : with open ( self . issue_archive_dir + "contents.json" , mode = "w" ) as fp : json . dump ( self . contents , fp ) except EnvironmentError : raise ScraperError ( "Unable to write the contents.json file." ) self . removeLockfile ( )
The main action.
The main action .
"""
        The main action. Fetches all of the required data for today's paper and
        saves it locally.
        """ # Sets the date and paper (guardian or observer). # Make the directory we'll save the HTML files in. # Get all of today's articles and save HTML versions to disk. # Delete the day-before-yesterday's files. # (Not yesterday's, just in case someone is currently viewing them.) # Write all the information about this issue to a contents.json file # within the dated folder.
self