covid19-etl/etl/owid2.py
def parse_file(self, url):
        """
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        """
        print("Getting data from {}".format(url))

        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')

            headers = next(reader)

            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return

            expected_h = self.expected_csv_headers
            assert (
                set(expected_h).issubset(headers) == True
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, headers
            )

            pre_row = None

            for row in reader:
                res = self.parse_row(pre_row, row)
                if res is not None:
                    self.insert_row_value(res)
                pre_row = row
            if pre_row is not None:
                res = self.parse_row(pre_row, None)
                if res is not None:
                    self.insert_row_value(res)

        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
def parse_file(self, url):
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')
            headers = next(reader)
            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return
            expected_h = self.expected_csv_headers
            assert (
                set(expected_h).issubset(headers) == True
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, headers
            )
            pre_row = None
            for row in reader:
                res = self.parse_row(pre_row, row)
                if res is not None:
                    self.insert_row_value(res)
                pre_row = row
            if pre_row is not None:
                res = self.parse_row(pre_row, None)
                if res is not None:
                    self.insert_row_value(res)
def parse_file ( self , url ) : print ( "Getting data from {}" . format ( url ) ) with closing ( self . get ( url , stream = True ) ) as r : f = ( line . decode ( "utf-8" ) for line in r . iter_lines ( ) ) reader = csv . reader ( f , delimiter = "," , quotechar = '"' ) headers = next ( reader ) if headers [ 0 ] == "404: Not Found" : print ( "  Unable to get file contents, received {}." . format ( headers ) ) return expected_h = self . expected_csv_headers assert ( set ( expected_h ) . issubset ( headers ) == True ) , "CSV headers have changed (expected {}, got {}). We may need to update the ETL code" . format ( expected_h , headers ) pre_row = None for row in reader : res = self . parse_row ( pre_row , row ) if res is not None : self . insert_row_value ( res ) pre_row = row if pre_row is not None : res = self . parse_row ( pre_row , None ) if res is not None : self . insert_row_value ( res )
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
"""
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        """
self
url
URL at which the CSV file is available
URL at which the CSV file is available