# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# ## Configuration
# _Initial steps to get the notebook ready to play nice with our repository. Do not delete this section._

# Code formatting with [black](https://pypi.org/project/nb-black/).

# %load_ext lab_black

# +
import os
import pytz
import glob
import pathlib

this_dir = pathlib.Path(os.path.abspath(""))
data_dir = this_dir / "data"
# -

import requests
import pandas as pd
import json
from datetime import datetime
from slugify import slugify

# ## Download

# Retrieve the page

url = "https://services7.arcgis.com/zaLZMEOGUnUT78nG/ArcGIS/rest/services/COVID-19%20Public%20Dashboard%20Data%20V3/FeatureServer/11/query?where=1%3D1&objectIds=&time=&resultType=none&outFields=*&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&sqlFormat=none&f=pjson&token="

r = requests.get(url)

data = r.json()

# ## Parse

# Get latest from timeseries

latest = data["features"][-1]

df = (
    pd.DataFrame.from_dict(
        latest["attributes"], orient="index", columns=["confirmed_cases"]
    )
    .reset_index()
    .rename(columns={"index": "area"})
)

# Create list of uncleaned city names and remove unneeded ones

cities_list = list(df[df["area"].str.contains("P_")]["area"])
cities_list.remove("P_Total")
cities_list.remove("P_Daily_Cases")
cities_list.remove("P_Unk")


# Trim down the dataframe based on the list

def prep_df(df, list):
    df = df[df["area"].isin(list)]
    df["area"] = df["area"].astype(str)
    df["area"] = df["area"].str[2:]
    return df


trim_df = prep_df(df, cities_list)


# Convert camel case to regular

def change_case(str):
    res = [str[0]]
    for c in str[1:]:
        if c in ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
            res.append(" ")
            res.append(c)
        else:
            res.append(c)

    return "".join(res)


trim_df["area"] = trim_df["area"].apply(change_case)


# Fix truncated names 

def clean_city_names(s):
    if s in df:
        return s.strip()
    else:
        s = s.replace("Green V Lake", "Green Valley Lake")
        s = s.replace("Lucerne V", "Lucerne Valley")
        s = s.replace("Newberry S", "Newberry Springs")
        s = s.replace("Pinon Hills", "Piñon Hills")
        s = s.replace("R C", "Rancho Cucamonga")
        s = s.replace("R Springs", "Running Springs")
        s = s.replace("San B", "San Bernardino")
        s = s.replace("Twentynine P", "Twentynine Palms")
        return s.strip()


trim_df["area"] = trim_df["area"].apply(clean_city_names)

# Get timestamp

timestamp = latest["attributes"]["Date"]

timestamp = datetime.fromtimestamp((timestamp / 1000))

latest_date = pd.to_datetime(timestamp).date()

trim_df["county_date"] = latest_date

trim_df.insert(0, "county", "San Bernardino")

# ## Vet

try:
    assert not len(trim_df) < 64
except AssertionError:
    raise AssertionError("San Bernardino County's scraper is missing rows")

try:
    assert not len(trim_df) > 64
except AssertionError:
    raise AssertionError("San Bernardino County's scraper has more rows than before")

# ## Export

# Set date

tz = pytz.timezone("America/Los_Angeles")

today = datetime.now(tz).date()

slug = "san-bernardino"

trim_df.to_csv(data_dir / slug / f"{today}.csv", index=False)

# ## Combine

csv_list = [
    i
    for i in glob.glob(str(data_dir / slug / "*.csv"))
    if not str(i).endswith("timeseries.csv")
]

df_list = []
for csv in csv_list:
    if "manual" in csv:
        df = pd.read_csv(csv, parse_dates=["date"])
    else:
        file_date = csv.split("/")[-1].replace(".csv", "")
        df = pd.read_csv(csv, parse_dates=["county_date"])
        df["date"] = file_date
    df_list.append(df)

df = pd.concat(df_list).sort_values(["date", "area"])

df.to_csv(data_dir / slug / "timeseries.csv", index=False)

StarcoderdataJupyterScriptsDedupFiltered