# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
# %reload_ext autoreload
# %autoreload 2

import numpy as np
import pandas as pd
import sys
sys.path.append("data_resolution")
from resolution_helpers import invalid_fips, remove_cols
# -

pm_data_path = "~/ASR-PM2.5/datasets/input_datasets/pm2.5/2015pm2.5dataset.csv"
pm_data = pd.read_csv(pm_data_path, dtype = {"fips": str}).dropna()

pm_data_updated = pm_data.rename(columns = {"fips": "FIPS"})

pm_data_updated

pm_fips = pm_data_updated["FIPS"]

pm_fips

invalid_pm_fips = invalid_fips(pm_fips)
assert len(invalid_pm_fips) == 0

keep_cols = ["FIPS", "pred_wght"]

remove_cols(pm_data_updated, keep_cols)

pm_data_updated.to_csv("~/ASR-PM2.5/datasets/intermediate_datasets/resultforpm2.5dataset.csv")

StarcoderdataJupyterScriptsDedupFiltered