covid19-etl/etl/jhu_to_s3.py
def parse_row(self, file_type, data_type, headers, row):
        """
        Converts a row of a CSV file to self.nested_dict in the format
        described on top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data
        """
        if not row:  # ignore empty rows
            return

        header_to_column = self.header_to_column[file_type]
        if "country" not in header_to_column:
            header_to_column = header_to_column[data_type]

        country = row[header_to_column["country"]]
        latitude = row[header_to_column["latitude"]] or "0"
        longitude = row[header_to_column["longitude"]] or "0"

        # Data with "Out of <state>" or "Unassigned" county value have
        # unknown coordinates of (0,0). We keep them to have accurate counts,
        # but they should not be displayed on the map: remove coordinates.
        if int(float(latitude)) == 0 and int(float(longitude)) == 0:
            latitude = None
            longitude = None

        codes = get_codes_for_country_name(self.codes_dict, country)
        iso3 = codes["iso3"]
        if iso3 not in self.nested_dict:
            # add this country if it's not already there
            self.nested_dict[iso3] = {
                "country_region": country,
                "latitude": latitude,
                "longitude": longitude,
                "iso2": codes["iso2"],
                "iso3": iso3,
                "provinces": {},
                "time_series": defaultdict(dict),
            }

        province = row[header_to_column["province"]]
        if not province:
            # the country may have been added using a province's coordinates.
            # when we find the country-level data, update the coordinates
            self.nested_dict[iso3]["latitude"] = latitude
            self.nested_dict[iso3]["longitude"] = longitude
        elif province not in self.nested_dict[iso3]["provinces"]:
            # add this province if it's not already there
            self.nested_dict[iso3]["provinces"][province] = {
                "province_state": province,
                "latitude": latitude,
                "longitude": longitude,
                "counties": {},
                "time_series": defaultdict(dict),
            }

        fips = None
        if file_type == "US_counties":
            fips = row[header_to_column["FIPS"]]
            county = row[header_to_column["county"]]
            iso2 = row[header_to_column["iso2"]]
            county_iso3 = row[header_to_column["iso3"]]
            code3 = row[header_to_column["code3"]]
            if fips:
                fips = int(float(fips))
                if (
                    fips
                    not in self.nested_dict[iso3]["provinces"][province]["counties"]
                ):
                    # add this county if it's not already there
                    self.nested_dict[iso3]["provinces"][province]["counties"][fips] = {
                        "fips": fips,
                        "county": county or None,
                        "latitude": latitude,
                        "longitude": longitude,
                        "iso2": iso2 or None,
                        "iso3": county_iso3 or None,
                        "code3": int(code3) or None,
                        "time_series": defaultdict(dict),
                    }

        # find the "time_series" dict we should add this data to
        tmp_dict = self.nested_dict[iso3]
        if province:
            tmp_dict = tmp_dict["provinces"][province]
            if fips:
                tmp_dict = tmp_dict["counties"][fips]
        time_series = tmp_dict["time_series"]

        dates_start = header_to_column["dates_start"]
        for i in range(dates_start, len(headers)):
            date = headers[i]
            date = get_unified_date_format(date)

            if row[i] == "":  # ignore empty values
                continue
            try:
                val = int(float(row[i]))
            except ValueError:
                print(
                    'Unable to convert {} to int for "{}", "{}" at {}'.format(
                        row[i], province, country, date
                    )
                )
                raise
            time_series[date][data_type] = val

        Converts a row of a CSV file to self.nested_dict in the format
        described on top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data
        
Converts a row of a CSV file to self.nested_dict in the format
described on top of this file.
Converts a row of a CSV file to self . nested_dict in the format described on top of this file .
def parse_row(self, file_type, data_type, headers, row):
        if not row:  
            return
        header_to_column = self.header_to_column[file_type]
        if "country" not in header_to_column:
            header_to_column = header_to_column[data_type]
        country = row[header_to_column["country"]]
        latitude = row[header_to_column["latitude"]] or "0"
        longitude = row[header_to_column["longitude"]] or "0"
        if int(float(latitude)) == 0 and int(float(longitude)) == 0:
            latitude = None
            longitude = None
        codes = get_codes_for_country_name(self.codes_dict, country)
        iso3 = codes["iso3"]
        if iso3 not in self.nested_dict:
            self.nested_dict[iso3] = {
                "country_region": country,
                "latitude": latitude,
                "longitude": longitude,
                "iso2": codes["iso2"],
                "iso3": iso3,
                "provinces": {},
                "time_series": defaultdict(dict),
            }
        province = row[header_to_column["province"]]
        if not province:
            self.nested_dict[iso3]["latitude"] = latitude
            self.nested_dict[iso3]["longitude"] = longitude
        elif province not in self.nested_dict[iso3]["provinces"]:
            self.nested_dict[iso3]["provinces"][province] = {
                "province_state": province,
                "latitude": latitude,
                "longitude": longitude,
                "counties": {},
                "time_series": defaultdict(dict),
            }
        fips = None
        if file_type == "US_counties":
            fips = row[header_to_column["FIPS"]]
            county = row[header_to_column["county"]]
            iso2 = row[header_to_column["iso2"]]
            county_iso3 = row[header_to_column["iso3"]]
            code3 = row[header_to_column["code3"]]
            if fips:
                fips = int(float(fips))
                if (
                    fips
                    not in self.nested_dict[iso3]["provinces"][province]["counties"]
                ):
                    self.nested_dict[iso3]["provinces"][province]["counties"][fips] = {
                        "fips": fips,
                        "county": county or None,
                        "latitude": latitude,
                        "longitude": longitude,
                        "iso2": iso2 or None,
                        "iso3": county_iso3 or None,
                        "code3": int(code3) or None,
                        "time_series": defaultdict(dict),
                    }
        tmp_dict = self.nested_dict[iso3]
        if province:
            tmp_dict = tmp_dict["provinces"][province]
            if fips:
                tmp_dict = tmp_dict["counties"][fips]
        time_series = tmp_dict["time_series"]
        dates_start = header_to_column["dates_start"]
        for i in range(dates_start, len(headers)):
            date = headers[i]
            date = get_unified_date_format(date)
            if row[i] == "":  
                continue
            try:
                val = int(float(row[i]))
            except ValueError:
                print(
                    'Unable to convert {} to int for "{}", "{}" at {}'.format(
                        row[i], province, country, date
                    )
                )
                raise
            time_series[date][data_type] = val
def parse_row ( self , file_type , data_type , headers , row ) : if not row : return header_to_column = self . header_to_column [ file_type ] if "country" not in header_to_column : header_to_column = header_to_column [ data_type ] country = row [ header_to_column [ "country" ] ] latitude = row [ header_to_column [ "latitude" ] ] or "0" longitude = row [ header_to_column [ "longitude" ] ] or "0" if int ( float ( latitude ) ) == 0 and int ( float ( longitude ) ) == 0 : latitude = None longitude = None codes = get_codes_for_country_name ( self . codes_dict , country ) iso3 = codes [ "iso3" ] if iso3 not in self . nested_dict : self . nested_dict [ iso3 ] = { "country_region" : country , "latitude" : latitude , "longitude" : longitude , "iso2" : codes [ "iso2" ] , "iso3" : iso3 , "provinces" : { } , "time_series" : defaultdict ( dict ) , } province = row [ header_to_column [ "province" ] ] if not province : self . nested_dict [ iso3 ] [ "latitude" ] = latitude self . nested_dict [ iso3 ] [ "longitude" ] = longitude elif province not in self . nested_dict [ iso3 ] [ "provinces" ] : self . nested_dict [ iso3 ] [ "provinces" ] [ province ] = { "province_state" : province , "latitude" : latitude , "longitude" : longitude , "counties" : { } , "time_series" : defaultdict ( dict ) , } fips = None if file_type == "US_counties" : fips = row [ header_to_column [ "FIPS" ] ] county = row [ header_to_column [ "county" ] ] iso2 = row [ header_to_column [ "iso2" ] ] county_iso3 = row [ header_to_column [ "iso3" ] ] code3 = row [ header_to_column [ "code3" ] ] if fips : fips = int ( float ( fips ) ) if ( fips not in self . nested_dict [ iso3 ] [ "provinces" ] [ province ] [ "counties" ] ) : self . nested_dict [ iso3 ] [ "provinces" ] [ province ] [ "counties" ] [ fips ] = { "fips" : fips , "county" : county or None , "latitude" : latitude , "longitude" : longitude , "iso2" : iso2 or None , "iso3" : county_iso3 or None , "code3" : int ( code3 ) or None , "time_series" : defaultdict ( dict ) , } tmp_dict = self . nested_dict [ iso3 ] if province : tmp_dict = tmp_dict [ "provinces" ] [ province ] if fips : tmp_dict = tmp_dict [ "counties" ] [ fips ] time_series = tmp_dict [ "time_series" ] dates_start = header_to_column [ "dates_start" ] for i in range ( dates_start , len ( headers ) ) : date = headers [ i ] date = get_unified_date_format ( date ) if row [ i ] == "" : continue try : val = int ( float ( row [ i ] ) ) except ValueError : print ( 'Unable to convert {} to int for "{}", "{}" at {}' . format ( row [ i ] , province , country , date ) ) raise time_series [ date ] [ data_type ] = val
Converts a row of a CSV file to self.nested_dict in the format
described on top of this file.
Converts a row of a CSV file to self . nested_dict in the format described on top of this file .
"""
        Converts a row of a CSV file to self.nested_dict in the format
        described on top of this file.

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data
        """ # ignore empty rows # Data with "Out of <state>" or "Unassigned" county value have # unknown coordinates of (0,0). We keep them to have accurate counts, # but they should not be displayed on the map: remove coordinates. # add this country if it's not already there # the country may have been added using a province's coordinates. # when we find the country-level data, update the coordinates # add this province if it's not already there # add this county if it's not already there # find the "time_series" dict we should add this data to # ignore empty values
self
file_type
type of this file - one
of ["global", "US_counties"]
type of this file - one of [ " global " " US_counties " ]
data_type
type of the data in this file - one
of ["confirmed", "deaths", "recovered"]
type of the data in this file - one of [ " confirmed " " deaths " " recovered " ]
headers
CSV file headers (first row of the file)
CSV file headers ( first row of the file )
row
row of data
row of data