covid19-etl/etl/jhu_to_s3.py
def nested_dict_to_data_by_level(self):
        """
        See `nested_dict_to_geojson` docstring for details on the aggregation.
        """
        print("Generating {}...".format(JSON_BY_LEVEL_FILENAME))
        LATEST_DATE_ONLY = True  # enabling this would break a bunch of things
        js = {
            "country": {},  # aggregated data for all countries
            "state": {},  # US only
            "county": {},  # US only
        }
        for country_data in self.nested_dict.values():
            iso3 = country_data["iso3"]
            if iso3 not in js["country"]:
                # add this country if it's not already there
                js["country"][iso3] = {
                    "confirmed": 0,
                    "deaths": 0,
                    "recovered": 0,
                    "country_region": country_data["country_region"],
                }

            # country-level time_series data
            for date, ts1 in country_data["time_series"].items():
                if LATEST_DATE_ONLY and date != self.latest_date:
                    continue
                if country_data["country_region"] not in ["US", "Canada"]:
                    js["country"][iso3]["confirmed"] += ts1.get("confirmed", 0)
                    js["country"][iso3]["deaths"] += ts1.get("deaths", 0)
                    js["country"][iso3]["recovered"] += ts1.get("recovered", 0)

            for province_data in country_data.get("provinces", {}).values():
                state_name = province_data["province_state"]
                if (
                    country_data["country_region"] == "US"
                    and state_name not in js["state"]
                ):
                    # add this US state if it's not already there
                    js["state"][state_name] = {
                        "confirmed": 0,
                        "deaths": 0,
                        "recovered": 0,
                        "country_region": country_data["country_region"],
                        "province_state": state_name,
                    }

                # add province-level time_series data for all countries + US states
                for date, ts in province_data["time_series"].items():
                    if LATEST_DATE_ONLY and date != self.latest_date:
                        continue
                    js["country"][iso3]["confirmed"] += ts.get("confirmed", 0)
                    js["country"][iso3]["deaths"] += ts.get("deaths", 0)
                    js["country"][iso3]["recovered"] += ts.get("recovered", 0)
                    if country_data["country_region"] == "US":
                        js["state"][state_name]["confirmed"] += ts.get("confirmed", 0)
                        js["state"][state_name]["deaths"] += ts.get("deaths", 0)
                        js["state"][state_name]["recovered"] += ts.get("recovered", 0)

                for county_data in province_data.get("counties", {}).values():
                    county_fips = county_data["fips"]
                    # add county-level time_series data for all countries + US states + US counties
                    for date, ts in county_data["time_series"].items():
                        if LATEST_DATE_ONLY and date != self.latest_date:
                            continue
                        js["country"][iso3]["confirmed"] += ts.get("confirmed", 0)
                        js["country"][iso3]["deaths"] += ts.get("deaths", 0)
                        js["country"][iso3]["recovered"] += ts.get("recovered", 0)
                        if country_data["country_region"] == "US":
                            js["state"][state_name]["confirmed"] += ts.get(
                                "confirmed", 0
                            )
                            js["state"][state_name]["deaths"] += ts.get("deaths", 0)
                            js["state"][state_name]["recovered"] += ts.get(
                                "recovered", 0
                            )

                            # add this US county. it shouldn't already be there
                            js["county"][county_fips] = {
                                "confirmed": ts.get("confirmed", 0),
                                "deaths": ts.get("deaths", 0),
                                "recovered": ts.get("recovered", 0),
                                "country_region": country_data["country_region"],
                                "province_state": state_name,
                                "county": county_data["county"],
                            }

            # (countries) if the original count is greater than the
            # aggregated count we calculated, use the original count
            for key in ["confirmed", "deaths", "recovered"]:
                original_count = ts1.get(key, 0)
                aggregated_count = js["country"][iso3][key]
                if original_count > aggregated_count:
                    print(
                        "  Country {}: Using global {} count ({}) rather than smaller aggregated count ({})".format(
                            country_data["country_region"],
                            key,
                            original_count,
                            aggregated_count,
                        )
                    )
                    js["country"][iso3][key] = original_count

        # remove values smaller than the threshold
        for data_level, locations in js.items():
            for location_id, data in locations.items():
                js[data_level][location_id] = replace_small_counts(data, data_level)

        # add last updated date
        js["last_updated"] = self.latest_date
        with open(
            os.path.join(CURRENT_DIR, MAP_DATA_FOLDER, JSON_BY_LEVEL_FILENAME), "w"
        ) as f:
            f.write(
                json.dumps(
                    js,
                    separators=(",", ":"),
                )
            )

        See `nested_dict_to_geojson` docstring for details on the aggregation.
        
See `nested_dict_to_geojson` docstring for details on the aggregation.
See ` nested_dict_to_geojson ` docstring for details on the aggregation .
def nested_dict_to_data_by_level(self):
        print("Generating {}...".format(JSON_BY_LEVEL_FILENAME))
        LATEST_DATE_ONLY = True  
        js = {
            "country": {},  
            "state": {},  
            "county": {},  
        }
        for country_data in self.nested_dict.values():
            iso3 = country_data["iso3"]
            if iso3 not in js["country"]:
                js["country"][iso3] = {
                    "confirmed": 0,
                    "deaths": 0,
                    "recovered": 0,
                    "country_region": country_data["country_region"],
                }
            for date, ts1 in country_data["time_series"].items():
                if LATEST_DATE_ONLY and date != self.latest_date:
                    continue
                if country_data["country_region"] not in ["US", "Canada"]:
                    js["country"][iso3]["confirmed"] += ts1.get("confirmed", 0)
                    js["country"][iso3]["deaths"] += ts1.get("deaths", 0)
                    js["country"][iso3]["recovered"] += ts1.get("recovered", 0)
            for province_data in country_data.get("provinces", {}).values():
                state_name = province_data["province_state"]
                if (
                    country_data["country_region"] == "US"
                    and state_name not in js["state"]
                ):
                    js["state"][state_name] = {
                        "confirmed": 0,
                        "deaths": 0,
                        "recovered": 0,
                        "country_region": country_data["country_region"],
                        "province_state": state_name,
                    }
                for date, ts in province_data["time_series"].items():
                    if LATEST_DATE_ONLY and date != self.latest_date:
                        continue
                    js["country"][iso3]["confirmed"] += ts.get("confirmed", 0)
                    js["country"][iso3]["deaths"] += ts.get("deaths", 0)
                    js["country"][iso3]["recovered"] += ts.get("recovered", 0)
                    if country_data["country_region"] == "US":
                        js["state"][state_name]["confirmed"] += ts.get("confirmed", 0)
                        js["state"][state_name]["deaths"] += ts.get("deaths", 0)
                        js["state"][state_name]["recovered"] += ts.get("recovered", 0)
                for county_data in province_data.get("counties", {}).values():
                    county_fips = county_data["fips"]
                    for date, ts in county_data["time_series"].items():
                        if LATEST_DATE_ONLY and date != self.latest_date:
                            continue
                        js["country"][iso3]["confirmed"] += ts.get("confirmed", 0)
                        js["country"][iso3]["deaths"] += ts.get("deaths", 0)
                        js["country"][iso3]["recovered"] += ts.get("recovered", 0)
                        if country_data["country_region"] == "US":
                            js["state"][state_name]["confirmed"] += ts.get(
                                "confirmed", 0
                            )
                            js["state"][state_name]["deaths"] += ts.get("deaths", 0)
                            js["state"][state_name]["recovered"] += ts.get(
                                "recovered", 0
                            )
                            js["county"][county_fips] = {
                                "confirmed": ts.get("confirmed", 0),
                                "deaths": ts.get("deaths", 0),
                                "recovered": ts.get("recovered", 0),
                                "country_region": country_data["country_region"],
                                "province_state": state_name,
                                "county": county_data["county"],
                            }
            for key in ["confirmed", "deaths", "recovered"]:
                original_count = ts1.get(key, 0)
                aggregated_count = js["country"][iso3][key]
                if original_count > aggregated_count:
                    print(
                        "  Country {}: Using global {} count ({}) rather than smaller aggregated count ({})".format(
                            country_data["country_region"],
                            key,
                            original_count,
                            aggregated_count,
                        )
                    )
                    js["country"][iso3][key] = original_count
        for data_level, locations in js.items():
            for location_id, data in locations.items():
                js[data_level][location_id] = replace_small_counts(data, data_level)
        js["last_updated"] = self.latest_date
        with open(
            os.path.join(CURRENT_DIR, MAP_DATA_FOLDER, JSON_BY_LEVEL_FILENAME), "w"
        ) as f:
            f.write(
                json.dumps(
                    js,
                    separators=(",", ":"),
                )
            )
def nested_dict_to_data_by_level ( self ) : print ( "Generating {}..." . format ( JSON_BY_LEVEL_FILENAME ) ) LATEST_DATE_ONLY = True js = { "country" : { } , "state" : { } , "county" : { } , } for country_data in self . nested_dict . values ( ) : iso3 = country_data [ "iso3" ] if iso3 not in js [ "country" ] : js [ "country" ] [ iso3 ] = { "confirmed" : 0 , "deaths" : 0 , "recovered" : 0 , "country_region" : country_data [ "country_region" ] , } for date , ts1 in country_data [ "time_series" ] . items ( ) : if LATEST_DATE_ONLY and date != self . latest_date : continue if country_data [ "country_region" ] not in [ "US" , "Canada" ] : js [ "country" ] [ iso3 ] [ "confirmed" ] += ts1 . get ( "confirmed" , 0 ) js [ "country" ] [ iso3 ] [ "deaths" ] += ts1 . get ( "deaths" , 0 ) js [ "country" ] [ iso3 ] [ "recovered" ] += ts1 . get ( "recovered" , 0 ) for province_data in country_data . get ( "provinces" , { } ) . values ( ) : state_name = province_data [ "province_state" ] if ( country_data [ "country_region" ] == "US" and state_name not in js [ "state" ] ) : js [ "state" ] [ state_name ] = { "confirmed" : 0 , "deaths" : 0 , "recovered" : 0 , "country_region" : country_data [ "country_region" ] , "province_state" : state_name , } for date , ts in province_data [ "time_series" ] . items ( ) : if LATEST_DATE_ONLY and date != self . latest_date : continue js [ "country" ] [ iso3 ] [ "confirmed" ] += ts . get ( "confirmed" , 0 ) js [ "country" ] [ iso3 ] [ "deaths" ] += ts . get ( "deaths" , 0 ) js [ "country" ] [ iso3 ] [ "recovered" ] += ts . get ( "recovered" , 0 ) if country_data [ "country_region" ] == "US" : js [ "state" ] [ state_name ] [ "confirmed" ] += ts . get ( "confirmed" , 0 ) js [ "state" ] [ state_name ] [ "deaths" ] += ts . get ( "deaths" , 0 ) js [ "state" ] [ state_name ] [ "recovered" ] += ts . get ( "recovered" , 0 ) for county_data in province_data . get ( "counties" , { } ) . values ( ) : county_fips = county_data [ "fips" ] for date , ts in county_data [ "time_series" ] . items ( ) : if LATEST_DATE_ONLY and date != self . latest_date : continue js [ "country" ] [ iso3 ] [ "confirmed" ] += ts . get ( "confirmed" , 0 ) js [ "country" ] [ iso3 ] [ "deaths" ] += ts . get ( "deaths" , 0 ) js [ "country" ] [ iso3 ] [ "recovered" ] += ts . get ( "recovered" , 0 ) if country_data [ "country_region" ] == "US" : js [ "state" ] [ state_name ] [ "confirmed" ] += ts . get ( "confirmed" , 0 ) js [ "state" ] [ state_name ] [ "deaths" ] += ts . get ( "deaths" , 0 ) js [ "state" ] [ state_name ] [ "recovered" ] += ts . get ( "recovered" , 0 ) js [ "county" ] [ county_fips ] = { "confirmed" : ts . get ( "confirmed" , 0 ) , "deaths" : ts . get ( "deaths" , 0 ) , "recovered" : ts . get ( "recovered" , 0 ) , "country_region" : country_data [ "country_region" ] , "province_state" : state_name , "county" : county_data [ "county" ] , } for key in [ "confirmed" , "deaths" , "recovered" ] : original_count = ts1 . get ( key , 0 ) aggregated_count = js [ "country" ] [ iso3 ] [ key ] if original_count > aggregated_count : print ( "  Country {}: Using global {} count ({}) rather than smaller aggregated count ({})" . format ( country_data [ "country_region" ] , key , original_count , aggregated_count , ) ) js [ "country" ] [ iso3 ] [ key ] = original_count for data_level , locations in js . items ( ) : for location_id , data in locations . items ( ) : js [ data_level ] [ location_id ] = replace_small_counts ( data , data_level ) js [ "last_updated" ] = self . latest_date with open ( os . path . join ( CURRENT_DIR , MAP_DATA_FOLDER , JSON_BY_LEVEL_FILENAME ) , "w" ) as f : f . write ( json . dumps ( js , separators = ( "," , ":" ) , ) )
See `nested_dict_to_geojson` docstring for details on the aggregation.
See ` nested_dict_to_geojson ` docstring for details on the aggregation .
"""
        See `nested_dict_to_geojson` docstring for details on the aggregation.
        """ # enabling this would break a bunch of things # aggregated data for all countries # US only # US only # add this country if it's not already there # country-level time_series data # add this US state if it's not already there # add province-level time_series data for all countries + US states # add county-level time_series data for all countries + US states + US counties # add this US county. it shouldn't already be there # (countries) if the original count is greater than the # aggregated count we calculated, use the original count # remove values smaller than the threshold # add last updated date
self