covid19-etl/etl/jhu_to_s3.py
def parse_file_to_nested_dict(self, file_type, data_type, url):
        """
        Converts a CSV file to self.nested_dict in the format described on
        top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        """
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')

            headers = next(reader)

            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return

            expected_h = self.expected_csv_headers[file_type]
            if isinstance(expected_h, dict):
                expected_h = expected_h[data_type]
            obtained_h = headers[: len(expected_h)]
            assert (
                obtained_h == expected_h
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, obtained_h
            )

            file_latest_date = get_unified_date_format(headers[-1])
            if not self.latest_date or datetime.strptime(
                self.latest_date, "%Y-%m-%d"
            ) > datetime.strptime(file_latest_date, "%Y-%m-%d"):
                self.latest_date = file_latest_date

            for row in reader:
                self.parse_row(file_type, data_type, headers, row)

        Converts a CSV file to self.nested_dict in the format described on
        top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        
Converts a CSV file to self.nested_dict in the format described on
top of this file.
Converts a CSV file to self . nested_dict in the format described on top of this file .
def parse_file_to_nested_dict(self, file_type, data_type, url):
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')
            headers = next(reader)
            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return
            expected_h = self.expected_csv_headers[file_type]
            if isinstance(expected_h, dict):
                expected_h = expected_h[data_type]
            obtained_h = headers[: len(expected_h)]
            assert (
                obtained_h == expected_h
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, obtained_h
            )
            file_latest_date = get_unified_date_format(headers[-1])
            if not self.latest_date or datetime.strptime(
                self.latest_date, "%Y-%m-%d"
            ) > datetime.strptime(file_latest_date, "%Y-%m-%d"):
                self.latest_date = file_latest_date
            for row in reader:
                self.parse_row(file_type, data_type, headers, row)
def parse_file_to_nested_dict ( self , file_type , data_type , url ) : print ( "Getting data from {}" . format ( url ) ) with closing ( self . get ( url , stream = True ) ) as r : f = ( line . decode ( "utf-8" ) for line in r . iter_lines ( ) ) reader = csv . reader ( f , delimiter = "," , quotechar = '"' ) headers = next ( reader ) if headers [ 0 ] == "404: Not Found" : print ( "  Unable to get file contents, received {}." . format ( headers ) ) return expected_h = self . expected_csv_headers [ file_type ] if isinstance ( expected_h , dict ) : expected_h = expected_h [ data_type ] obtained_h = headers [ : len ( expected_h ) ] assert ( obtained_h == expected_h ) , "CSV headers have changed (expected {}, got {}). We may need to update the ETL code" . format ( expected_h , obtained_h ) file_latest_date = get_unified_date_format ( headers [ - 1 ] ) if not self . latest_date or datetime . strptime ( self . latest_date , "%Y-%m-%d" ) > datetime . strptime ( file_latest_date , "%Y-%m-%d" ) : self . latest_date = file_latest_date for row in reader : self . parse_row ( file_type , data_type , headers , row )
Converts a CSV file to self.nested_dict in the format described on
top of this file.
Converts a CSV file to self . nested_dict in the format described on top of this file .
"""
        Converts a CSV file to self.nested_dict in the format described on
        top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        """
self
file_type
type of this file - one
of ["global", "US_counties"]
type of this file - one of [ " global " " US_counties " ]
data_type
type of the data in this file - one
of ["confirmed", "deaths", "recovered"]
type of the data in this file - one of [ " confirmed " " deaths " " recovered " ]
url
URL at which the CSV file is available
URL at which the CSV file is available