covid19-etl/etl/ctp.py
def parse_file(self, url):
        """
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        """
        races, race_headers = self.extract_races()
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')

            headers = next(reader)

            assert (
                headers[0] != "404: Not Found"
            ), "Unable to get file contents, received {}.".format(headers)

            assert self.expected_file_headers.issubset(
                set(headers)
            ), "CSV headers have changed (expected {} is a subset of {}). We may need to update the ETL code".format(
                self.expected_file_headers, headers
            )

            headers = headers + race_headers[3:]

            for i in range(0, len(headers)):
                self.header_to_column[headers[i]] = i

            summary_location_list = []

            for row in reader:
                if (row[0], row[1], row[2]) in races:
                    [row.append(k) for k in races[(row[0], row[1], row[2])]]
                else:
                    [row.append("") for _ in range(len(self.expected_race_headers) - 3)]

                summary_location, summary_clinical = self.parse_row(row)

                summary_location_submitter_id = summary_location["submitter_id"]
                if summary_location_submitter_id not in summary_location_list:
                    self.summary_locations.append(summary_location)
                    summary_location_list.append(summary_location_submitter_id)

                self.summary_clinicals.append(summary_clinical)

        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
def parse_file(self, url):
        races, race_headers = self.extract_races()
        print("Getting data from {}".format(url))
        with closing(self.get(url, stream=True)) as r:
            f = (line.decode("utf-8") for line in r.iter_lines())
            reader = csv.reader(f, delimiter=",", quotechar='"')
            headers = next(reader)
            assert (
                headers[0] != "404: Not Found"
            ), "Unable to get file contents, received {}.".format(headers)
            assert self.expected_file_headers.issubset(
                set(headers)
            ), "CSV headers have changed (expected {} is a subset of {}). We may need to update the ETL code".format(
                self.expected_file_headers, headers
            )
            headers = headers + race_headers[3:]
            for i in range(0, len(headers)):
                self.header_to_column[headers[i]] = i
            summary_location_list = []
            for row in reader:
                if (row[0], row[1], row[2]) in races:
                    [row.append(k) for k in races[(row[0], row[1], row[2])]]
                else:
                    [row.append("") for _ in range(len(self.expected_race_headers) - 3)]
                summary_location, summary_clinical = self.parse_row(row)
                summary_location_submitter_id = summary_location["submitter_id"]
                if summary_location_submitter_id not in summary_location_list:
                    self.summary_locations.append(summary_location)
                    summary_location_list.append(summary_location_submitter_id)
                self.summary_clinicals.append(summary_clinical)
def parse_file ( self , url ) : races , race_headers = self . extract_races ( ) print ( "Getting data from {}" . format ( url ) ) with closing ( self . get ( url , stream = True ) ) as r : f = ( line . decode ( "utf-8" ) for line in r . iter_lines ( ) ) reader = csv . reader ( f , delimiter = "," , quotechar = '"' ) headers = next ( reader ) assert ( headers [ 0 ] != "404: Not Found" ) , "Unable to get file contents, received {}." . format ( headers ) assert self . expected_file_headers . issubset ( set ( headers ) ) , "CSV headers have changed (expected {} is a subset of {}). We may need to update the ETL code" . format ( self . expected_file_headers , headers ) headers = headers + race_headers [ 3 : ] for i in range ( 0 , len ( headers ) ) : self . header_to_column [ headers [ i ] ] = i summary_location_list = [ ] for row in reader : if ( row [ 0 ] , row [ 1 ] , row [ 2 ] ) in races : [ row . append ( k ) for k in races [ ( row [ 0 ] , row [ 1 ] , row [ 2 ] ) ] ] else : [ row . append ( "" ) for _ in range ( len ( self . expected_race_headers ) - 3 ) ] summary_location , summary_clinical = self . parse_row ( row ) summary_location_submitter_id = summary_location [ "submitter_id" ] if summary_location_submitter_id not in summary_location_list : self . summary_locations . append ( summary_location ) summary_location_list . append ( summary_location_submitter_id ) self . summary_clinicals . append ( summary_clinical )
Converts a CSV file to data we can submit via Sheepdog.
Converts a CSV file to data we can submit via Sheepdog .
"""
        Converts a CSV file to data we can submit via Sheepdog. Stores the
        records to submit in `self.location_data` and `self.time_series_data`.
        Ignores any records that are already in Sheepdog (relies on unique
        `submitter_id` to check)

        Args:
            url (str): URL at which the CSV file is available
        """
self
url
URL at which the CSV file is available
URL at which the CSV file is available