covid19-etl/etl/jhu.py
def parse_file(self, file_type, data_type, url):
        """
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        """
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')

            headers = next(reader)

            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return

            expected_h = self.expected_csv_headers[file_type]
            if isinstance(expected_h, dict):
                expected_h = expected_h[data_type]
            obtained_h = headers[: len(expected_h)]
            assert (
                obtained_h == expected_h
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, obtained_h
            )

            first_date_i = [i for i, h in enumerate(headers) if h.endswith("/20")][0]
            last_date = headers[-1]
            print(
                "  First date: {}; last date: {}".format(
                    headers[first_date_i], last_date
                )
            )

            for row in reader:
                if not row:  # ignore empty rows
                    continue
                location, date_to_value = self.parse_row(
                    file_type, data_type, headers, row
                )
                if not location:
                    # We are using US data by state instead of global
                    continue

                location_submitter_id = location["submitter_id"]
                if (
                    location_submitter_id not in self.location_data
                    # do not re-submit location data that already exist
                    and location_submitter_id not in self.existing_summary_locations
                ):
                    self.location_data[location_submitter_id] = location

                for date, value in date_to_value.items():
                    # do not re-submit summary_clinical data that
                    # already exist. Assume anything older than the last
                    # submitted date has already been submitted
                    if (
                        time_series_date_to_string(date)
                        > time_series_date_to_string(self.last_date)
                        or LAST_DATE_ONLY
                    ):
                        self.time_series_data[location_submitter_id][date][
                            data_type
                        ] = value

        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
def parse_file(self, file_type, data_type, url):
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')
            headers = next(reader)
            if headers[0] == "404: Not Found":
                print("  Unable to get file contents, received {}.".format(headers))
                return
            expected_h = self.expected_csv_headers[file_type]
            if isinstance(expected_h, dict):
                expected_h = expected_h[data_type]
            obtained_h = headers[: len(expected_h)]
            assert (
                obtained_h == expected_h
            ), "CSV headers have changed (expected {}, got {}). We may need to update the ETL code".format(
                expected_h, obtained_h
            )
            first_date_i = [i for i, h in enumerate(headers) if h.endswith("/20")][0]
            last_date = headers[-1]
            print(
                "  First date: {}; last date: {}".format(
                    headers[first_date_i], last_date
                )
            )
            for row in reader:
                if not row:  
                    continue
                location, date_to_value = self.parse_row(
                    file_type, data_type, headers, row
                )
                if not location:
                    continue
                location_submitter_id = location["submitter_id"]
                if (
                    location_submitter_id not in self.location_data
                    and location_submitter_id not in self.existing_summary_locations
                ):
                    self.location_data[location_submitter_id] = location
                for date, value in date_to_value.items():
                    if (
                        time_series_date_to_string(date)
                        > time_series_date_to_string(self.last_date)
                        or LAST_DATE_ONLY
                    ):
                        self.time_series_data[location_submitter_id][date][
                            data_type
                        ] = value
def parse_file ( self , file_type , data_type , url ) : print ( "Getting data from {}" . format ( url ) ) with closing ( self . get ( url , stream = True ) ) as r : f = ( line . decode ( "utf-8" ) for line in r . iter_lines ( ) ) reader = csv . reader ( f , delimiter = "," , quotechar = '"' ) headers = next ( reader ) if headers [ 0 ] == "404: Not Found" : print ( "  Unable to get file contents, received {}." . format ( headers ) ) return expected_h = self . expected_csv_headers [ file_type ] if isinstance ( expected_h , dict ) : expected_h = expected_h [ data_type ] obtained_h = headers [ : len ( expected_h ) ] assert ( obtained_h == expected_h ) , "CSV headers have changed (expected {}, got {}). We may need to update the ETL code" . format ( expected_h , obtained_h ) first_date_i = [ i for i , h in enumerate ( headers ) if h . endswith ( "/20" ) ] [ 0 ] last_date = headers [ - 1 ] print ( "  First date: {}; last date: {}" . format ( headers [ first_date_i ] , last_date ) ) for row in reader : if not row : continue location , date_to_value = self . parse_row ( file_type , data_type , headers , row ) if not location : continue location_submitter_id = location [ "submitter_id" ] if ( location_submitter_id not in self . location_data and location_submitter_id not in self . existing_summary_locations ) : self . location_data [ location_submitter_id ] = location for date , value in date_to_value . items ( ) : if ( time_series_date_to_string ( date ) > time_series_date_to_string ( self . last_date ) or LAST_DATE_ONLY ) : self . time_series_data [ location_submitter_id ] [ date ] [ data_type ] = value
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
"""
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            file_type (str): type of this file - one
                of ["global", "US_counties"]
            data_type (str): type of the data in this file - one
                of ["confirmed", "deaths", "recovered"]
            url (str): URL at which the CSV file is available
        """ # ignore empty rows # We are using US data by state instead of global # do not re-submit location data that already exist # do not re-submit summary_clinical data that # already exist. Assume anything older than the last # submitted date has already been submitted
self
file_type
type of this file - one
of ["global", "US_counties"]
type of this file - one of [ " global " " US_counties " ]
data_type
type of the data in this file - one
of ["confirmed", "deaths", "recovered"]
type of the data in this file - one of [ " confirmed " " deaths " " recovered " ]
url
URL at which the CSV file is available
URL at which the CSV file is available