"""
# Reduce the Dataset size by 76.9%


The first of your worries start when loading the data - the time it takes to read the dataset into your working environment can be as long as you train a model. At this stage, don't use pandas - there are much faster alternatives available.Its one of the datatable package which can read data up to 10 times faster.

As an example, we will load 1000000 rows Kaggle TPS OCT 2021 dataset with both datatable and pandas and compare the speeds:
"""
"""
### Import Libraries
"""
import datatable as dt  # pip install datatable
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
"""
### Loading data using datatable
"""
%%time
train_dt = dt.fread("../input/tabular-playground-series-oct-2021/train.csv").to_pandas()
%%time
test_dt = dt.fread("../input/tabular-playground-series-oct-2021/test.csv").to_pandas()
"""
### Loading data using pandas
"""
%%time
train_pd = pd.read_csv("../input/tabular-playground-series-oct-2021/train.csv")
%%time
test_pd = pd.read_csv("../input/tabular-playground-series-oct-2021/test.csv")
"""
### Load time comparision
"""
"""
|            | Datatable  | Pandas   |
|------------| -----------| ---------|
| Train data | 19.1s      | 49.6s    |
| Test data  | 8.91s      | 23.4s    |

"""
"""
### Memory usage of train and test data
"""
memory_usage_train = train_pd.memory_usage(deep=True) / 1024 ** 2
memory_usage_test = test_pd.memory_usage(deep=True) / 1024 ** 2

memory_usage_train.head(7)
memory_usage_train.sum()
"""
### Reduce the memory of train and test data which is loaded using pandas
"""
def reduce_memory_usage(df, verbose=True):
    numerics = ["int8", "int16", "int32", "int64", "float16", "float32", "float64"]
    start_mem = df.memory_usage().sum() / 1024 ** 2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == "int":
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if (
                    c_min > np.finfo(np.float16).min
                    and c_max < np.finfo(np.float16).max
                ):
                    df[col] = df[col].astype(np.float16)
                elif (
                    c_min > np.finfo(np.float32).min
                    and c_max < np.finfo(np.float32).max
                ):
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
    end_mem = df.memory_usage().sum() / 1024 ** 2
    if verbose:
        print(
            "Mem. usage decreased to {:.2f} Mb ({:.1f}% reduction)".format(
                end_mem, 100 * (start_mem - end_mem) / start_mem
            )
        )
    return df
reduced_train_pd = reduce_memory_usage(train_pd, verbose=True)
reduced_test_pd = reduce_memory_usage(test_pd, verbose=True)
"""
 <center><img src="http://ift.tt/2lRnGIX" alt="centered image"> </center>
 <h1 style=' border:0; color:#FFA500 '><center> If you find this notebook usefull support me with giving ⬆️!</left></h1> 
"""
"""
Reference : https://www.kaggle.com/bextuychiev/how-to-work-w-million-row-datasets-like-a-pro
"""