# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
from io import BytesIO
from zipfile import ZipFile
from urllib.request import urlopen

import numpy as np

import pandas as pd
from pandas import DataFrame, Series

def get_filename(file_list):
    match = [s for s in file_list if "_rv" in s]
    answer = file_list[0]
    if len(match) > 0:
        answer = match[0]
    return(answer)

def net_load_info(url):
    resp = urlopen(url)
    zipfile = ZipFile(BytesIO(resp.read()))
    files = zipfile.namelist()
    return(files)

def net_load_data(url):
    resp = urlopen(url)
    zipfile = ZipFile(BytesIO(resp.read()))
    file_name = get_filename(zipfile.namelist())
    data_file = zipfile.open(file_name)
    answer = pd.read_csv(data_file,
                         low_memory = False,
                         encoding = "iso-8859-1")
    
    return(answer)

def fix_cols(dat):
    for col in dat:
        col = col.lower()
        dat[col] = pd.to_numeric(dat[col], errors = "coerce", downcast = "integer")
    return(dat)

def read_file(year, keepers):
    file_spec = np.where(year < 2014, "ic" + str(year) + ".zip", "adm" + str(year) + ".zip")
    answer = net_load_data("https://nces.ed.gov/ipeds/datacenter/data/" + str(file_spec))
    answer.columns = [colname.lower() for colname in list(answer.columns.values)]
    answer = fix_cols(answer)
    answer = answer[keepers]
    return(answer)

# value columns
cols = ["unitid", "applcn", "applcnm", "applcnw", "admssn", "admssnm", "admssnw","enrlt", "enrlm", "enrlw"]

# years to pull
years = range(2007, 2009)
# -

for year in years:
    file_spec = np.where(year < 2014, "ic" + str(year) + ".zip", "adm" + str(year) + ".zip")
    contents = net_load_info("https://nces.ed.gov/ipeds/datacenter/data/" + str(file_spec))
    for item in contents:
        print(item)

for year in years:
    file_spec = np.where(year < 2014, "ic" + str(year) + ".zip", "adm" + str(year) + ".zip")
    contents = net_load_info("https://nces.ed.gov/ipeds/datacenter/data/" + str(file_spec))
    file_name = get_filename(contents)
    print(file_name)

# +
df = DataFrame()

for year in years:
    temp = read_file(year, cols)
    temp = fix_cols(temp)
    df = pd.concat([df, temp],
                   sort = True)
    temp = None

df = df.fillna(0)

df.head(20)
# -

df.dtypes



StarcoderdataJupyterScriptsDedupFiltered