covid19-etl/etl/jhu.py
def parse_row(self, file_type, data_type, headers, row):
        """
        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        """
        header_to_column = self.header_to_column[file_type]
        if "country" not in header_to_column:
            header_to_column = header_to_column[data_type]

        country = row[header_to_column["country"]]
        province = row[header_to_column["province"]]
        latitude = row[header_to_column["latitude"]] or "0"
        longitude = row[header_to_column["longitude"]] or "0"

        if country == "US" and province == "":
            # We are using US data by state instead of global
            return None, None

        if int(float(latitude)) == 0 and int(float(longitude)) == 0:
            # Data with "Out of <state>" or "Unassigned" county value have
            # unknown coordinates of (0,0). We don't submit them for now
            return None, None

        submitter_id = format_location_submitter_id(country, province)
        location = {
            "country_region": country,
            "latitude": latitude,
            "longitude": longitude,
            "projects": [{"code": self.project_code}],
        }
        if province:
            location["province_state"] = province
        if file_type == "US_counties":
            county = row[header_to_column["county"]]
            iso2 = row[header_to_column["iso2"]]
            iso3 = row[header_to_column["iso3"]]
            code3 = row[header_to_column["code3"]]
            fips = row[header_to_column["FIPS"]]
            if county:
                location["county"] = county
                submitter_id = format_location_submitter_id(country, province, county)
            if iso2:
                location["iso2"] = iso2
            if iso3:
                location["iso3"] = iso3
            if code3:
                location["code3"] = int(code3)
            if fips:
                location["FIPS"] = int(float(fips))
        location["submitter_id"] = submitter_id

        date_to_value = {}
        dates_start = header_to_column["dates_start"]
        dates_indices = range(dates_start, len(headers))
        if LAST_DATE_ONLY:
            dates_indices = [len(headers) - 1]
        for i in dates_indices:
            date = headers[i]
            date = get_unified_date_format(date)

            if row[i] == "":  # ignore empty values
                continue
            try:
                val = int(float(row[i]))
            except ValueError:
                print(
                    'Unable to convert {} to int for "{}", "{}" at {}'.format(
                        row[i], province, country, date
                    )
                )
                raise
            date_to_value[date] = val

        return location, date_to_value

        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        
Converts a row of a CSV file to data we can submit via Sheepdog
Converts a row of a CSV file to data we can submit via Sheepdog
def parse_row(self, file_type, data_type, headers, row):
        header_to_column = self.header_to_column[file_type]
        if "country" not in header_to_column:
            header_to_column = header_to_column[data_type]
        country = row[header_to_column["country"]]
        province = row[header_to_column["province"]]
        latitude = row[header_to_column["latitude"]] or "0"
        longitude = row[header_to_column["longitude"]] or "0"
        if country == "US" and province == "":
            return None, None
        if int(float(latitude)) == 0 and int(float(longitude)) == 0:
            return None, None
        submitter_id = format_location_submitter_id(country, province)
        location = {
            "country_region": country,
            "latitude": latitude,
            "longitude": longitude,
            "projects": [{"code": self.project_code}],
        }
        if province:
            location["province_state"] = province
        if file_type == "US_counties":
            county = row[header_to_column["county"]]
            iso2 = row[header_to_column["iso2"]]
            iso3 = row[header_to_column["iso3"]]
            code3 = row[header_to_column["code3"]]
            fips = row[header_to_column["FIPS"]]
            if county:
                location["county"] = county
                submitter_id = format_location_submitter_id(country, province, county)
            if iso2:
                location["iso2"] = iso2
            if iso3:
                location["iso3"] = iso3
            if code3:
                location["code3"] = int(code3)
            if fips:
                location["FIPS"] = int(float(fips))
        location["submitter_id"] = submitter_id
        date_to_value = {}
        dates_start = header_to_column["dates_start"]
        dates_indices = range(dates_start, len(headers))
        if LAST_DATE_ONLY:
            dates_indices = [len(headers) - 1]
        for i in dates_indices:
            date = headers[i]
            date = get_unified_date_format(date)
            if row[i] == "":  
                continue
            try:
                val = int(float(row[i]))
            except ValueError:
                print(
                    'Unable to convert {} to int for "{}", "{}" at {}'.format(
                        row[i], province, country, date
                    )
                )
                raise
            date_to_value[date] = val
        return location, date_to_value
def parse_row ( self , file_type , data_type , headers , row ) : header_to_column = self . header_to_column [ file_type ] if "country" not in header_to_column : header_to_column = header_to_column [ data_type ] country = row [ header_to_column [ "country" ] ] province = row [ header_to_column [ "province" ] ] latitude = row [ header_to_column [ "latitude" ] ] or "0" longitude = row [ header_to_column [ "longitude" ] ] or "0" if country == "US" and province == "" : return None , None if int ( float ( latitude ) ) == 0 and int ( float ( longitude ) ) == 0 : return None , None submitter_id = format_location_submitter_id ( country , province ) location = { "country_region" : country , "latitude" : latitude , "longitude" : longitude , "projects" : [ { "code" : self . project_code } ] , } if province : location [ "province_state" ] = province if file_type == "US_counties" : county = row [ header_to_column [ "county" ] ] iso2 = row [ header_to_column [ "iso2" ] ] iso3 = row [ header_to_column [ "iso3" ] ] code3 = row [ header_to_column [ "code3" ] ] fips = row [ header_to_column [ "FIPS" ] ] if county : location [ "county" ] = county submitter_id = format_location_submitter_id ( country , province , county ) if iso2 : location [ "iso2" ] = iso2 if iso3 : location [ "iso3" ] = iso3 if code3 : location [ "code3" ] = int ( code3 ) if fips : location [ "FIPS" ] = int ( float ( fips ) ) location [ "submitter_id" ] = submitter_id date_to_value = { } dates_start = header_to_column [ "dates_start" ] dates_indices = range ( dates_start , len ( headers ) ) if LAST_DATE_ONLY : dates_indices = [ len ( headers ) - 1 ] for i in dates_indices : date = headers [ i ] date = get_unified_date_format ( date ) if row [ i ] == "" : continue try : val = int ( float ( row [ i ] ) ) except ValueError : print ( 'Unable to convert {} to int for "{}", "{}" at {}' . format ( row [ i ] , province , country , date ) ) raise date_to_value [ date ] = val return location , date_to_value
Converts a row of a CSV file to data we can submit via Sheepdog
Converts a row of a CSV file to data we can submit via Sheepdog
"""
        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            headers (list(str)): CSV file headers (first row of the file)
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        """ # We are using US data by state instead of global # Data with "Out of <state>" or "Unassigned" county value have # unknown coordinates of (0,0). We don't submit them for now # ignore empty values
(dict, dict) tuple:
location data, in a format ready to be submitted to Sheepdog
{ "date1": , "date2":  } from the row data
( dict dict ) tuple : location data in a format ready to be submitted to Sheepdog { " date1 " : " date2 " : } from the row data
self
file_type
type of this file - one
of ["global", "US_counties"]
type of this file - one of [ " global " " US_counties " ]
data_type
type of the data in this file - one
of ["confirmed", "deaths", "recovered"]
type of the data in this file - one of [ " confirmed " " deaths " " recovered " ]
headers
CSV file headers (first row of the file)
CSV file headers ( first row of the file )
row
row of data
row of data