covid19-etl/etl/ctp.py
def parse_row(self, row):
        """
        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        """

        date = row[self.header_to_column["date"]]
        date = datetime.strptime(date, "%Y%m%d").date()
        date = date.strftime("%Y-%m-%d")

        country = "US"
        state = row[self.header_to_column["state"]]
        summary_location_submitter_id = format_location_submitter_id(country, state)

        summary_location = {
            "country_region": country,
            "submitter_id": summary_location_submitter_id,
            "projects": [{"code": self.project_code}],
            "province_state": state,
        }

        fips = row[self.header_to_column["fips"]]
        if fips:
            summary_location["FIPS"] = int(fips)

        summary_clinical_submitter_id = format_summary_clinical_submitter_id(
            summary_location_submitter_id, date
        )
        summary_clinical = {
            "date": date,
            "submitter_id": summary_clinical_submitter_id,
            "summary_locations": [{"submitter_id": summary_location_submitter_id}],
        }

        map_csv_fields = {
            "confirmed": "positive",
            "negative": "negative",
            "pending": "pending",
            "hospitalizedCurrently": "hospitalizedCurrently",
            "hospitalizedCumulative": "hospitalizedCumulative",
            "inIcuCurrently": "inIcuCurrently",
            "inIcuCumulative": "inIcuCumulative",
            "onVentilatorCurrently": "onVentilatorCurrently",
            "recovered": "recovered",
            "totalTestsViral": "totalTestsViral",
            "positiveTestsViral": "positiveTestsViral",
            "negativeTestsViral": "negativeTestsViral",
            "positiveCasesViral": "positiveCasesViral",
            "positiveIncrease": "positiveIncrease",
            "negativeIncrease": "negativeIncrease",
            "totalTestResultsIncrease": "totalTestResultsIncrease",
            "deathIncrease": "deathIncrease",
            "hospitalizedIncrease": "hospitalizedIncrease",
            "race_white_count": "Cases_White",
            "race_black_count": "Cases_Black",
            "race_hispanic_count": "Cases_Latinx",
            "race_asian_count": "Cases_Asian",
            "race_ai_an_count": "Cases_AIAN",
            "race_nh_pi_count": "Cases_NHPI",
            "race_multiracial_count": "Cases_Multiracial",
            "race_other_count": "Cases_Other",
            "race_left_blank_count": "Cases_Unknown",
            "ethnicity_hispanic_count": "Cases_Ethnicity_Hispanic",
            "ethnicity_nonhispanic_count": "Cases_Ethnicity_NonHispanic",
            "ethnicity_unknown_count": "Cases_Ethnicity_Unknown",
            "deaths": "Deaths_Total",
            "race_white_deaths": "Deaths_White",
            "race_black_deaths": "Deaths_Black",
            "race_hispanic_deaths": "Deaths_Latinx",
            "race_asian_deaths": "Deaths_Asian",
            "race_ai_an_deaths": "Deaths_AIAN",
            "race_nh_pi_deaths": "Deaths_NHPI",
            "race_multiracial_deaths": "Deaths_Multiracial",
            "race_other_deaths": "Deaths_Other",
            "race_left_blank_deaths": "Deaths_Unknown",
            "ethnicity_hispanic_deaths": "Deaths_Ethnicity_Hispanic",
            "ethnicity_nonhispanic_deaths": "Deaths_Ethnicity_NonHispanic",
            "ethnicity_unknown_deaths": "Deaths_Ethnicity_Unknown",
        }

        for k, v in map_csv_fields.items():
            value = row[self.header_to_column[v]]
            if value and value.lower() not in ["nan", "n/a"]:
                try:
                    summary_clinical[k] = int(value.replace(",", ""))
                except:
                    pass

        dataQualityGrade = row[self.header_to_column["dataQualityGrade"]]
        if dataQualityGrade:
            summary_clinical["dataQualityGrade"] = dataQualityGrade

        lastUpdateEt = row[self.header_to_column["lastUpdateEt"]]
        if lastUpdateEt:
            summary_clinical["lastUpdateEt"] = lastUpdateEt

        return summary_location, summary_clinical

        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        
Converts a row of a CSV file to data we can submit via Sheepdog
Converts a row of a CSV file to data we can submit via Sheepdog
def parse_row(self, row):
        date = row[self.header_to_column["date"]]
        date = datetime.strptime(date, "%Y%m%d").date()
        date = date.strftime("%Y-%m-%d")
        country = "US"
        state = row[self.header_to_column["state"]]
        summary_location_submitter_id = format_location_submitter_id(country, state)
        summary_location = {
            "country_region": country,
            "submitter_id": summary_location_submitter_id,
            "projects": [{"code": self.project_code}],
            "province_state": state,
        }
        fips = row[self.header_to_column["fips"]]
        if fips:
            summary_location["FIPS"] = int(fips)
        summary_clinical_submitter_id = format_summary_clinical_submitter_id(
            summary_location_submitter_id, date
        )
        summary_clinical = {
            "date": date,
            "submitter_id": summary_clinical_submitter_id,
            "summary_locations": [{"submitter_id": summary_location_submitter_id}],
        }
        map_csv_fields = {
            "confirmed": "positive",
            "negative": "negative",
            "pending": "pending",
            "hospitalizedCurrently": "hospitalizedCurrently",
            "hospitalizedCumulative": "hospitalizedCumulative",
            "inIcuCurrently": "inIcuCurrently",
            "inIcuCumulative": "inIcuCumulative",
            "onVentilatorCurrently": "onVentilatorCurrently",
            "recovered": "recovered",
            "totalTestsViral": "totalTestsViral",
            "positiveTestsViral": "positiveTestsViral",
            "negativeTestsViral": "negativeTestsViral",
            "positiveCasesViral": "positiveCasesViral",
            "positiveIncrease": "positiveIncrease",
            "negativeIncrease": "negativeIncrease",
            "totalTestResultsIncrease": "totalTestResultsIncrease",
            "deathIncrease": "deathIncrease",
            "hospitalizedIncrease": "hospitalizedIncrease",
            "race_white_count": "Cases_White",
            "race_black_count": "Cases_Black",
            "race_hispanic_count": "Cases_Latinx",
            "race_asian_count": "Cases_Asian",
            "race_ai_an_count": "Cases_AIAN",
            "race_nh_pi_count": "Cases_NHPI",
            "race_multiracial_count": "Cases_Multiracial",
            "race_other_count": "Cases_Other",
            "race_left_blank_count": "Cases_Unknown",
            "ethnicity_hispanic_count": "Cases_Ethnicity_Hispanic",
            "ethnicity_nonhispanic_count": "Cases_Ethnicity_NonHispanic",
            "ethnicity_unknown_count": "Cases_Ethnicity_Unknown",
            "deaths": "Deaths_Total",
            "race_white_deaths": "Deaths_White",
            "race_black_deaths": "Deaths_Black",
            "race_hispanic_deaths": "Deaths_Latinx",
            "race_asian_deaths": "Deaths_Asian",
            "race_ai_an_deaths": "Deaths_AIAN",
            "race_nh_pi_deaths": "Deaths_NHPI",
            "race_multiracial_deaths": "Deaths_Multiracial",
            "race_other_deaths": "Deaths_Other",
            "race_left_blank_deaths": "Deaths_Unknown",
            "ethnicity_hispanic_deaths": "Deaths_Ethnicity_Hispanic",
            "ethnicity_nonhispanic_deaths": "Deaths_Ethnicity_NonHispanic",
            "ethnicity_unknown_deaths": "Deaths_Ethnicity_Unknown",
        }
        for k, v in map_csv_fields.items():
            value = row[self.header_to_column[v]]
            if value and value.lower() not in ["nan", "n/a"]:
                try:
                    summary_clinical[k] = int(value.replace(",", ""))
                except:
                    pass
        dataQualityGrade = row[self.header_to_column["dataQualityGrade"]]
        if dataQualityGrade:
            summary_clinical["dataQualityGrade"] = dataQualityGrade
        lastUpdateEt = row[self.header_to_column["lastUpdateEt"]]
        if lastUpdateEt:
            summary_clinical["lastUpdateEt"] = lastUpdateEt
        return summary_location, summary_clinical
def parse_row ( self , row ) : date = row [ self . header_to_column [ "date" ] ] date = datetime . strptime ( date , "%Y%m%d" ) . date ( ) date = date . strftime ( "%Y-%m-%d" ) country = "US" state = row [ self . header_to_column [ "state" ] ] summary_location_submitter_id = format_location_submitter_id ( country , state ) summary_location = { "country_region" : country , "submitter_id" : summary_location_submitter_id , "projects" : [ { "code" : self . project_code } ] , "province_state" : state , } fips = row [ self . header_to_column [ "fips" ] ] if fips : summary_location [ "FIPS" ] = int ( fips ) summary_clinical_submitter_id = format_summary_clinical_submitter_id ( summary_location_submitter_id , date ) summary_clinical = { "date" : date , "submitter_id" : summary_clinical_submitter_id , "summary_locations" : [ { "submitter_id" : summary_location_submitter_id } ] , } map_csv_fields = { "confirmed" : "positive" , "negative" : "negative" , "pending" : "pending" , "hospitalizedCurrently" : "hospitalizedCurrently" , "hospitalizedCumulative" : "hospitalizedCumulative" , "inIcuCurrently" : "inIcuCurrently" , "inIcuCumulative" : "inIcuCumulative" , "onVentilatorCurrently" : "onVentilatorCurrently" , "recovered" : "recovered" , "totalTestsViral" : "totalTestsViral" , "positiveTestsViral" : "positiveTestsViral" , "negativeTestsViral" : "negativeTestsViral" , "positiveCasesViral" : "positiveCasesViral" , "positiveIncrease" : "positiveIncrease" , "negativeIncrease" : "negativeIncrease" , "totalTestResultsIncrease" : "totalTestResultsIncrease" , "deathIncrease" : "deathIncrease" , "hospitalizedIncrease" : "hospitalizedIncrease" , "race_white_count" : "Cases_White" , "race_black_count" : "Cases_Black" , "race_hispanic_count" : "Cases_Latinx" , "race_asian_count" : "Cases_Asian" , "race_ai_an_count" : "Cases_AIAN" , "race_nh_pi_count" : "Cases_NHPI" , "race_multiracial_count" : "Cases_Multiracial" , "race_other_count" : "Cases_Other" , "race_left_blank_count" : "Cases_Unknown" , "ethnicity_hispanic_count" : "Cases_Ethnicity_Hispanic" , "ethnicity_nonhispanic_count" : "Cases_Ethnicity_NonHispanic" , "ethnicity_unknown_count" : "Cases_Ethnicity_Unknown" , "deaths" : "Deaths_Total" , "race_white_deaths" : "Deaths_White" , "race_black_deaths" : "Deaths_Black" , "race_hispanic_deaths" : "Deaths_Latinx" , "race_asian_deaths" : "Deaths_Asian" , "race_ai_an_deaths" : "Deaths_AIAN" , "race_nh_pi_deaths" : "Deaths_NHPI" , "race_multiracial_deaths" : "Deaths_Multiracial" , "race_other_deaths" : "Deaths_Other" , "race_left_blank_deaths" : "Deaths_Unknown" , "ethnicity_hispanic_deaths" : "Deaths_Ethnicity_Hispanic" , "ethnicity_nonhispanic_deaths" : "Deaths_Ethnicity_NonHispanic" , "ethnicity_unknown_deaths" : "Deaths_Ethnicity_Unknown" , } for k , v in map_csv_fields . items ( ) : value = row [ self . header_to_column [ v ] ] if value and value . lower ( ) not in [ "nan" , "n/a" ] : try : summary_clinical [ k ] = int ( value . replace ( "," , "" ) ) except : pass dataQualityGrade = row [ self . header_to_column [ "dataQualityGrade" ] ] if dataQualityGrade : summary_clinical [ "dataQualityGrade" ] = dataQualityGrade lastUpdateEt = row [ self . header_to_column [ "lastUpdateEt" ] ] if lastUpdateEt : summary_clinical [ "lastUpdateEt" ] = lastUpdateEt return summary_location , summary_clinical
Converts a row of a CSV file to data we can submit via Sheepdog
Converts a row of a CSV file to data we can submit via Sheepdog
"""
        Converts a row of a CSV file to data we can submit via Sheepdog

        Args:
            row (list(str)): row of data

        Returns:
            (dict, dict) tuple:
                - location data, in a format ready to be submitted to Sheepdog
                - { "date1": <value>, "date2": <value> } from the row data
        """
(dict, dict) tuple:
location data, in a format ready to be submitted to Sheepdog
{ "date1": , "date2":  } from the row data
( dict dict ) tuple : location data in a format ready to be submitted to Sheepdog { " date1 " : " date2 " : } from the row data
self
row
row of data
row of data