# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: adsi_ass_2
#     language: python
#     name: adsi_ass_2
# ---

# + [markdown] pycharm={"name": "#%% md\n"}
# # Introduction
#
# This notebook performs:
# 1. dropping of unused features
# 1. profile reports of the original and the subset data
# 1. splitting into training, testing and validation sets
# 1. saving the data into `/data/processed`

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
# %load_ext autoreload
# %autoreload 2

# +
import pandas as pd
import numpy as np
from pathlib import Path
from dotenv import find_dotenv, load_dotenv
from pandas_profiling import ProfileReport
from sklearn.model_selection import train_test_split

from src.data.sets import save_sets, load_sets, split_sets_random
# -

# # Introduction

# + pycharm={"name": "#%%\n"}
project_dir = Path(find_dotenv()).parent
data_dir = project_dir / 'data'
raw_data_dir = data_dir / 'raw'
interim_data_dir = data_dir / 'interim'
processed_data_dir = data_dir / 'processed'
report_dir = project_dir / 'reports'

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Load data
#

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
path = raw_data_dir / 'beer_reviews.csv'
df = pd.read_csv(path)
df.info()

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
df

# + [markdown] pycharm={"name": "#%% md\n"}
# # Profile report

# + pycharm={"name": "#%% raw\n"} active=""
#
# profile_report = ProfileReport(df,
#                                explorative=True)
# path = report_dir / 'pr_beer_reviews_raw'
# profile_report.to_file(path.with_suffix('.html'))
#

# + [markdown] pycharm={"name": "#%% md\n"}
# # Subset data
#
# The target variable is `beer_style`.
#
# The required parameters for the API are:
# * `brewery_name`
# * `review_aroma`
# * `review_appearance`
# * `review_palate`
# * `review_taste`
#
# So the model will be trained on this subset of the data.

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
target = 'beer_style'
col_list = [target] + [
    'brewery_name',
    'review_aroma',
    'review_appearance',
    'review_palate',
    'review_taste'
]
df_subset = df[col_list]

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
df_subset.describe()

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
df_subset.info()

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
path = processed_data_dir / 'subset'
df_subset.to_csv(path.with_suffix('.csv'), index=False)
df_subset.to_parquet(path.with_suffix('.parquet'), index=False)

# + [markdown] pycharm={"name": "#%% md\n"}
# # Profile report (subset)

# + pycharm={"name": "#%% raw\n"} active=""
# profile_report = ProfileReport(df_subset,
#                                explorative=True)
# path = report_dir / 'pr_beer_reviews_raw_subset'
# profile_report.to_file(path.with_suffix('.html'))

# + [markdown] pycharm={"name": "#%% md\n"}
# ## Split data

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
X_train, y_train, X_val, y_val, X_test, y_test = split_sets_random(
    df=df_subset,
    target_col=target,
    test_ratio=0.2,
    to_numpy=False
)

# + jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
save_sets(X_train=X_train,
          X_test=X_test,
          X_val=X_val,
          y_train=y_train,
          y_test=y_test,
          y_val=y_val,
          path=processed_data_dir)

StarcoderdataJupyterScriptsDedupFiltered