# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Kotzly/DS4H_Course/blob/main/notebooks/colab_sinasc_download.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="wrhbqcV-Dp8n" colab={"base_uri": "https://localhost:8080/"} outputId="e23efb09-5ec5-4bd7-8b90-73072bd8bfa2"
# !pip install simpledbf

# + id="7ShGotOt9xf7"
import shutil
import urllib.request as request
from contextlib import closing
import zipfile
import shutil
import sys
import os
from pathlib import Path
import pandas as pd
from simpledbf import Dbf5
import platform

def download(url, save_filepath):
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(save_filepath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192): 
                # If you have chunk encoded response uncomment if
                # and set chunk_size parameter to None.
                #if chunk: 
                f.write(chunk)

def download_ftp(url, savepath):
    with closing(request.urlopen(url)) as r:
        with open(savepath, 'wb') as f:
            shutil.copyfileobj(r, f)

root = Path(".")

# + id="VnoIvGBO9xgB"
dbc_save_folder = root / "SINASC_DBC"
dbf_save_folder = root / "SINASC_DBF"
csv_save_folder = root / "SINASC_CSV"

dbc_save_folder.mkdir(exist_ok=True)
dbf_save_folder.mkdir(exist_ok=True)
csv_save_folder.mkdir(exist_ok=True)


# + id="Z2EpgNZu9xgC" colab={"base_uri": "https://localhost:8080/"} outputId="7888ccd9-e85e-4b2c-89ab-97a065bd5480"
SINASC_URLS = [
    f"ftp://ftp.datasus.gov.br/dissemin/publicos/SINASC/1996_/Dados/DNRES/DNSP{year}.dbc"
    for year in range(2010, 2019 + 1)
]



for url in SINASC_URLS:
    print("Downloading", url.split("/")[-1])
    save_path = dbc_save_folder / url.split("/")[-1]
    download_ftp(url, save_path)

# + id="whWXpLVzFYnP"
# %load_ext rpy2.ipython

# + id="un_pBWn7F3Dr" colab={"base_uri": "https://localhost:8080/"} outputId="a926243d-dca3-4c05-c36d-420734918871" language="R"
# # Using read.dbc package
# # https://pt.linkedin.com/pulse/datasus-conhe%C3%A7a-nova-ferramenta-para-ler-arquivos-dbc-petruzalek
#
# install.packages("read.dbc")
# library("read.dbc")
#

# + id="S9IFq3KYK0YV"
# !ls SINASC_CSV

# + id="GPSxZANrHBGT" colab={"base_uri": "https://localhost:8080/"} outputId="15347d58-6e1a-4706-ed7d-4408cfa8f57c" language="R"
# files = list.files("SINASC_DBC")
# for (file in files){
#     
#     print(file)
#
#     filename = strsplit(file, ".dbc")[1]
#     
#     dbc_filepath = paste("./SINASC_DBC/", file, sep="")
#
#     csv_filepath = paste("./SINASC_CSV/", filename, ".csv", sep="")
#     
#     df <- read.dbc(dbc_filepath)
#
#     write.csv(df, csv_filepath)
# }

# + id="WgbL2yLu9xgG" colab={"base_uri": "https://localhost:8080/"} outputId="76a471a8-d260-4510-b1f0-efae412de843"
union_df = None
dfs = []
cols = ["DTNASC", "QTDFILVIVO", "SEXO", "IDADEMAE", "PESO", "GRAVIDEZ", "CONSULTAS", "RACACOR", "CODMUNNASC", "ESTCIVMAE", "ESCMAE", "PARTO", "IDANOMAL"]
union_csv_filename = "union.csv"
for csv_filepath in csv_save_folder.glob("*.csv"):
    if csv_filepath.name == union_csv_filename:
        continue
    df = pd.read_csv(csv_filepath, usecols=cols)
    print(csv_filepath, len(df.columns))
    sorted_cols = list(sorted(df.columns))
    df = df[cols]
    dfs += [df]
    if union_df is None:
        union_df = df
    else:
        union_df = pd.concat([union_df, df])
union_df.to_csv(csv_save_folder / union_csv_filename)

# + id="e6GwCGtO9xgH" colab={"base_uri": "https://localhost:8080/", "height": 435} outputId="1a307b55-80d5-4e22-802c-5a2b9e6c944b"
import pandas as pd

pd.read_csv(csv_save_folder / "union.csv")

# + id="TmzZLlnlZdQS" outputId="ce367955-067b-4f97-9a03-c3b36ec1d661" colab={"base_uri": "https://localhost:8080/"}
df = pd.read_csv(csv_save_folder / "union.csv")

# + id="-hPlLbcRbmpk" outputId="2406772b-3f3e-4120-bb8b-daf029aca381" colab={"base_uri": "https://localhost:8080/"}
cities = {
  3502101: 'Andradina',
  3502804: 'Araãatuba',
  3505708: 'Barueri',
  3514403: 'Dracena',
  3517406: 'Guaãra',
  3524808: 'Jales',
  3546801: 'Santa Isabel',
  3548500: 'Santos',
  3548807: 'São Caetano do Sul',
  3549805: 'São Josã do Rio Preto',
  3550308: 'São Paulo'
}
cities = {(cod // 10): name for cod, name in cities.items()}
cities


# + id="h6vwLDvTZQf4"
def parse_day(d):
  d = str(d)
  day, month, year = int(d[-8:-6]), int(d[-6:-4]), int(d[:-4])
  return day
def parse_month(d):
  d = str(d)
  day, month, year = int(d[-8:-6]), int(d[-6:-4]), int(d[:-4])
  return month
def parse_year(d):
  d = str(d)
  day, month, year = int(d[-8:-6]), int(d[-6:-4]), int(d[-4:])
  if year < 2000:
    year += 2000
  return year

df = df[df.CODMUNNASC.apply(lambda x: x in cities.keys())]

df["YEAR"], df["MONTH"], df["DAY"] = 0, 0, 0
df["DAY"] = df.DTNASC.apply(parse_day)
df["MONTH"] = df.DTNASC.apply(parse_month)
df["YEAR"] = df.DTNASC.apply(parse_year)
df["CITYNAME"] = df.CODMUNNASC.apply(lambda x: cities[x])

# + id="adkM7HaSa43s" outputId="79fc6a6d-b954-4fe3-ec06-9e60aa80d818" colab={"base_uri": "https://localhost:8080/", "height": 573}
df

# + id="nXork2Yeg8Yw" outputId="812f4bd9-27e8-4743-f68d-d6c3f1017d06" colab={"base_uri": "https://localhost:8080/", "height": 1000}
pd.set_option('display.max_rows', None)

count_df = df.groupby(["YEAR", "MONTH", "CITYNAME", "CODMUNNASC"]).count()[["DTNASC"]]

StarcoderdataJupyterScriptsDedupFiltered