scripts/scraper.py
def load_config(self):
        "Sets initial object variables, and loads others from the scraper.cfg file"

        # First, variables which are set here, not in the config file.

        # The array used to create the contents.json once all processin's done:
        #
        # {
        #     'meta': {
        #         'max_words': 2340,
        #         'paper_name': 'observer'
        #     },
        #     'books': [
        #         {
        #             'meta': {
        #                   'id': '',
        #                   'webUrl': 'http...',
        #                   'webTitle':'Main section',
        #                   ...},
        #             'articles': [{
        #                   'id': 'world/gallery/2010/jul/07/spain-spain',
        #                   'path': '/world/gallery/2010/jul/07/spain-spain',
        #                   'title': 'Article title...',
        #                   'file': 'spain-spain.html',
        #                   'words': 374,
        #                   'newspaperPageNumber': 34,
        #                   'newspaperBookSection': {'id':'','title':'', ...}
        #               }, {}, {}]
        #         },
        #         {}
        #     ]
        # }
        self.contents = {"meta": {"max_words": 0}, "books": []}

        # Temporary place for what will go in self.contents['books'].
        # In this case it's structured like:
        # {
        #   'theguardian/sport': {
        #       'meta': {...},
        #       'articles': [{...}, {...}, ...],
        #   },
        #   ...
        # }
        #
        # Then, after fetching we'll put all the values from this into
        # self.contents['books'] in the correct order.
        self.fetched_books = {}

        # Will be set depending on what issue we're fetching.
        # A datetime object.
        self.issue_date = None

        # Will be set in start() to a path of self.archive_dir plus
        # self.issue_date
        self.issue_archive_dir = ""

        # The URLs of the full list of the current issue of the Guardian and
        # Observer.
        self.source_urls = {
            "guardian": "http://www.theguardian.com/theguardian",
            "observer": "http://www.theguardian.com/theobserver",
        }

        # When we've worked out what date we're on, this will be either
        # 'guardian' or 'observer'.
        self.paper_name = ""

        # Will be the lockfile we check to make sure this script doesn't run
        # multiple times.
        self.lockfile_path = sys.path[0] + "/lock.pid"

        # Second, load stuff from the config file.

        config_file = sys.path[0] + "/scraper.cfg"
        config = configparser.ConfigParser()

        try:
            config.read_file(open(config_file))
        except IOError:
            raise ScraperError("Can't read config file: " + config_file)

        self.guardian_api_key = config.get("Settings", "guardian_api_key")

        self.archive_dir = config.get("Settings", "archive_dir")

        self.verbose = config.getboolean("Settings", "verbose")

        # Set up the template we'll use to render each article to a file.
        jinja_env = Environment(loader=PackageLoader("scraper", "../templates"))
        jinja_env.filters["typogrify"] = jinja_filters.typogrify
        self.template = jinja_env.get_template("article.html")
Sets initial object variables, and loads others from the scraper.cfg file
Sets initial object variables, and loads others from the scraper.cfg file
Sets initial object variables and loads others from the scraper . cfg file
def load_config(self):
         {
             'meta': {
                 'max_words': 2340,
                 'paper_name': 'observer'
             },
             'books': [
                 {
                     'meta': {
                           'id': '',
                           'webUrl': 'http...',
                           'webTitle':'Main section',
                           ...},
                     'articles': [{
                           'id': 'world/gallery/2010/jul/07/spain-spain',
                           'path': '/world/gallery/2010/jul/07/spain-spain',
                           'title': 'Article title...',
                           'file': 'spain-spain.html',
                           'words': 374,
                           'newspaperPageNumber': 34,
                           'newspaperBookSection': {'id':'','title':'', ...}
                       }, {}, {}]
                 },
                 {}
             ]
         }
        self.contents = {"meta": {"max_words": 0}, "books": []}
         Temporary place for what will go in self.contents['books'].
         In this case it's structured like:
         {
           'theguardian/sport': {
               'meta': {...},
               'articles': [{...}, {...}, ...],
           },
           ...
         }
         Then, after fetching we'll put all the values from this into
         self.contents['books'] in the correct order.
        self.fetched_books = {}
         Will be set depending on what issue we're fetching.
         A datetime object.
        self.issue_date = None
         Will be set in start() to a path of self.archive_dir plus
         self.issue_date
        self.issue_archive_dir = ""
         The URLs of the full list of the current issue of the Guardian and
         Observer.
        self.source_urls = {
            "guardian": "http://www.theguardian.com/theguardian",
            "observer": "http://www.theguardian.com/theobserver",
        }
         When we've worked out what date we're on, this will be either
         'guardian' or 'observer'.
        self.paper_name = ""
         Will be the lockfile we check to make sure this script doesn't run
         multiple times.
        self.lockfile_path = sys.path[0] + "/lock.pid"
         Second, load stuff from the config file.
        config_file = sys.path[0] + "/scraper.cfg"
        config = configparser.ConfigParser()
        try:
            config.read_file(open(config_file))
        except IOError:
            raise ScraperError("Can't read config file: " + config_file)
        self.guardian_api_key = config.get("Settings", "guardian_api_key")
        self.archive_dir = config.get("Settings", "archive_dir")
        self.verbose = config.getboolean("Settings", "verbose")
         Set up the template we'll use to render each article to a file.
        jinja_env = Environment(loader=PackageLoader("scraper", "../templates"))
        jinja_env.filters["typogrify"] = jinja_filters.typogrify
        self.template = jinja_env.get_template("article.html")
def load_config ( self ) : self . contents = { "meta" : { "max_words" : 0 } , "books" : [ ] } self . fetched_books = { } self . issue_date = None self . issue_archive_dir = "" self . source_urls = { "guardian" : "http://www.theguardian.com/theguardian" , "observer" : "http://www.theguardian.com/theobserver" , } self . paper_name = "" self . lockfile_path = sys . path [ 0 ] + "/lock.pid" config_file = sys . path [ 0 ] + "/scraper.cfg" config = configparser . ConfigParser ( ) try : config . read_file ( open ( config_file ) ) except IOError : raise ScraperError ( "Can't read config file: " + config_file ) self . guardian_api_key = config . get ( "Settings" , "guardian_api_key" ) self . archive_dir = config . get ( "Settings" , "archive_dir" ) self . verbose = config . getboolean ( "Settings" , "verbose" ) jinja_env = Environment ( loader = PackageLoader ( "scraper" , "../templates" ) ) jinja_env . filters [ "typogrify" ] = jinja_filters . typogrify self . template = jinja_env . get_template ( "article.html" )
Sets initial object variables, and loads others from the scraper.cfg file
Sets initial object variables and loads others from the scraper . cfg file
"Sets initial object variables, and loads others from the scraper.cfg file" # First, variables which are set here, not in the config file. # The array used to create the contents.json once all processin's done: # # { #     'meta': { #         'max_words': 2340, #         'paper_name': 'observer' #     }, #     'books': [ #         { #             'meta': { #                   'id': '', #                   'webUrl': 'http...', #                   'webTitle':'Main section', #                   ...}, #             'articles': [{ #                   'id': 'world/gallery/2010/jul/07/spain-spain', #                   'path': '/world/gallery/2010/jul/07/spain-spain', #                   'title': 'Article title...', #                   'file': 'spain-spain.html', #                   'words': 374, #                   'newspaperPageNumber': 34, #                   'newspaperBookSection': {'id':'','title':'', ...} #               }, {}, {}] #         }, #         {} #     ] # } # Temporary place for what will go in self.contents['books']. # In this case it's structured like: # { #   'theguardian/sport': { #       'meta': {...}, #       'articles': [{...}, {...}, ...], #   }, #   ... # } # # Then, after fetching we'll put all the values from this into # self.contents['books'] in the correct order. # Will be set depending on what issue we're fetching. # A datetime object. # Will be set in start() to a path of self.archive_dir plus # self.issue_date # The URLs of the full list of the current issue of the Guardian and # Observer. # When we've worked out what date we're on, this will be either # 'guardian' or 'observer'. # Will be the lockfile we check to make sure this script doesn't run # multiple times. # Second, load stuff from the config file. # Set up the template we'll use to render each article to a file.
self