covid19-etl/etl/jhu_to_s3.py
def files_to_submissions(self):
        """
        Reads CSV files and converts them to the formats described on top
        of this file.
        """
        urls = {
            "global": {
                "confirmed": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv",
                "deaths": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv",
                "recovered": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv",
                # "testing": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_testing_global.csv",
            },
            "US_counties": {
                "confirmed": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_US.csv",
                "deaths": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_US.csv",
            },
        }

        for file_type in ["global", "US_counties"]:
            for data_type, url in urls[file_type].items():
                self.parse_file_to_nested_dict(file_type, data_type, url)

        # create data folders
        data_folders = [
            os.path.join(CURRENT_DIR, MAP_DATA_FOLDER),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "country"),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "state"),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "county"),
        ]
        for path in data_folders:
            pathlib.Path(path).mkdir(exist_ok=True)

        # generate data files
        self.nested_dict_to_geojson()
        self.nested_dict_to_data_by_level()
        self.nested_dict_to_time_series_by_level()
        self.nested_dict_to_data_by_time()

        print("Latest date: {}".format(self.latest_date))

        Reads CSV files and converts them to the formats described on top
        of this file.
        
Reads CSV files and converts them to the formats described on top
of this file.
Reads CSV files and converts them to the formats described on top of this file .
def files_to_submissions(self):
        urls = {
            "global": {
                "confirmed": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv",
                "deaths": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv",
                "recovered": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv",
            },
            "US_counties": {
                "confirmed": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_US.csv",
                "deaths": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_US.csv",
            },
        }
        for file_type in ["global", "US_counties"]:
            for data_type, url in urls[file_type].items():
                self.parse_file_to_nested_dict(file_type, data_type, url)
        data_folders = [
            os.path.join(CURRENT_DIR, MAP_DATA_FOLDER),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "country"),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "state"),
            os.path.join(CURRENT_DIR, TIME_SERIES_DATA_FOLDER, "county"),
        ]
        for path in data_folders:
            pathlib.Path(path).mkdir(exist_ok=True)
        self.nested_dict_to_geojson()
        self.nested_dict_to_data_by_level()
        self.nested_dict_to_time_series_by_level()
        self.nested_dict_to_data_by_time()
        print("Latest date: {}".format(self.latest_date))
def files_to_submissions ( self ) : urls = { "global" : { "confirmed" : "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv" , "deaths" : "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv" , "recovered" : "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv" , } , "US_counties" : { "confirmed" : "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_US.csv" , "deaths" : "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_US.csv" , } , } for file_type in [ "global" , "US_counties" ] : for data_type , url in urls [ file_type ] . items ( ) : self . parse_file_to_nested_dict ( file_type , data_type , url ) data_folders = [ os . path . join ( CURRENT_DIR , MAP_DATA_FOLDER ) , os . path . join ( CURRENT_DIR , TIME_SERIES_DATA_FOLDER ) , os . path . join ( CURRENT_DIR , TIME_SERIES_DATA_FOLDER , "country" ) , os . path . join ( CURRENT_DIR , TIME_SERIES_DATA_FOLDER , "state" ) , os . path . join ( CURRENT_DIR , TIME_SERIES_DATA_FOLDER , "county" ) , ] for path in data_folders : pathlib . Path ( path ) . mkdir ( exist_ok = True ) self . nested_dict_to_geojson ( ) self . nested_dict_to_data_by_level ( ) self . nested_dict_to_time_series_by_level ( ) self . nested_dict_to_data_by_time ( ) print ( "Latest date: {}" . format ( self . latest_date ) )
Reads CSV files and converts them to the formats described on top
of this file.
Reads CSV files and converts them to the formats described on top of this file .
"""
        Reads CSV files and converts them to the formats described on top
        of this file.
        """ # "testing": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_testing_global.csv", # create data folders # generate data files
self