! pip install numba numexpr

import math
import time
import warnings
from dateutil.parser import parse

import janitor
import numpy as np
import pandas as pd
from numba import jit
from sklearn import datasets
from pandas.api.types import is_datetime64_any_dtype as is_datetime

warnings.filterwarnings("ignore", category=pd.errors.DtypeWarning)
pd.options.display.max_columns = 999

path = 'https://raw.githubusercontent.com/FBosler/you-datascientist/master/invoices.csv'

def load_dataset(naivly=False):
    df = (pd.concat([pd.read_csv(path)
                       .clean_names()
                       .remove_columns(["meal_id", "company_id"]) 
                     for i in range(20)])
            .assign(meal_tip=lambda x: x.meal_price.map(lambda x: x * 0.2))
            .astype({"meal_price": int})
            .rename(columns={"meal_price": "meal_price_with_tip"}))

    if naivly:
        for col in df.columns:
            df[col] = df[col].astype(object)
    return df

df = load_dataset()
df.head()

"""
Explanation: Practical Optimisations for Pandas 🐼
Eyal Trabelsi
About Me 🙈


Software Engineer at Salesforce 👷


Big passion for python, data and performance optimisations 🐍🤖


Online at medium | twitter 🌐


Optimizing Your Pandas is not Rocket Science 🚀
Optimization ?! Why ?🤨


Fast is better than slow 🐇


latency response time 200 milliseconds client roundtrip


throughput successful traffic flow of 200 requests per seconds


Memory efficiency is good 💾


Saving money is awesome 💸


Hardware will only take you so far 💻


Ok now that i have got you attention, the next question i want to tackle is when should we optimize our code


Before We Optimize ⏰

It's actually needed 🚔

remember optimized code is:

harder to write and read
less maintainable
buggier, more brittle

Optimize when

gather requirements, there are some parts you won't be able to touch

establish percentile SLAs: 50, 95, 99 max


Our code is well tested 💯


Focus on the bottlenecks 🍾


Dataset 📉
End of explanation
"""


import warnings
warnings.filterwarnings("ignore")

def iterrows_original_meal_price(df):
    for i, row in df.iterrows():
        df.loc[i]["original_meal_price"] = row["meal_price_with_tip"] - row["meal_tip"]
    return df

%%timeit -r 1 -n 1
iterrows_original_meal_price(df)

"""
Explanation: How 👀
Use What You Need 🧑


Keep needed columns only


Keep needed rows only


Dont Reinvent the Wheel 🎡


Vast ecosystem


Use existing solutions


Fewer bugs


Highly optimized


Avoid Loops ♾
Bad Option 😈
End of explanation
"""


def apply_original_meal_price(df):
    df["original_meal_price"] = df.apply(lambda x: x['meal_price_with_tip'] - x['meal_tip'], axis=1)
    return df

%%timeit 
apply_original_meal_price(df)

"""
Explanation: Better Option 🤵
End of explanation
"""


def vectorized_original_meal_price(df):
    df["original_meal_price"] = df["meal_price_with_tip"] - df["meal_tip"] 
    return df

%%timeit 
vectorized_original_meal_price(df)

"""
Explanation: 150x Improvement In Execution Time ⌛
Best Option 👼
End of explanation
"""


ones = np.ones(shape=5000)
ones

types = ['object', 'complex128', 'float64', 'int64', 'int32', 'int16', 'int8', 'bool']
df = pd.DataFrame(dict([(t, ones.astype(t)) for t in types]))
df.memory_usage(index=False, deep=True)

"""
Explanation: Another 2000x Improvement In Execution Time ⌛
Picking the Right Type 🌈
Motivation 🏆
End of explanation
"""


df = load_dataset(naivly=True)

df.memory_usage(deep=True).sum()

df.memory_usage(deep=True)

df.dtypes

"""
Explanation: Supported Types 🌈


int64 / float64


bool


objects


datetime64


timedelta


Category


Sparse Types


Nullable Integer/Nullable Bolean


Your Own Types


Open Sourced Types like cyberpandas and geopandas 


Where We Stand 🌈
End of explanation
"""


optimized_df = df.astype({'order_id': 'category',
                          'date': 'category',
                          'date_of_meal': 'category',
                          'participants': 'category',
                          'meal_price_with_tip': 'int16',
                          'type_of_meal': 'category',
                          'heroes_adjustment': 'bool',
                          'meal_tip': 'float32'})

optimized_df.memory_usage(deep=True).sum()

"""
Explanation: Optimized Types 🌈
End of explanation
"""


%%timeit
df["meal_price_with_tip"].astype(object).mean()

%%timeit
df["meal_price_with_tip"].astype(float).mean()

"""
Explanation: 13x Improvement In Memory ⌛
Optimized Types 🌈

Mathmatical operation performance 🧮
End of explanation
"""


def foo(N):
    accumulator = 0
    for i in range(N):
        accumulator = accumulator + i
    return accumulator

%%timeit
df.meal_price_with_tip.map(foo)

"""
Explanation: 2.5x Performance Improvement⌛
Recommended Installation  👨‍🏫


numexpr - uses multiple cores as well as smart chunking and caching to achieve large speedups.


bottleneck -  uses specialized nan aware Cython routines to achieve large speedups. 


Better for medium to big datasets


Compiled Code 🤯


Python dynamic nature 


No compilation optimization


Pure Python can be slow
End of explanation
"""


%load_ext Cython

"""
Explanation: Cython and Numba for the rescue 👨‍🚒
Cython 🤯


Up to 100x speedup from pure python 👍


Learning Curve 👎


Separated Compilation Step 👎 👍
End of explanation
"""


%%cython
def cython_foo(long N):
    cdef long accumulator
    accumulator = 0

    cdef long i
    for i in range(N):
        accumulator += i

    return accumulator

%%timeit
df.meal_price_with_tip.map(cython_foo)

"""
Explanation: Example
End of explanation
"""


@jit(nopython=True)
def numba_foo(N):
    accumulator = 0
    for i in range(N):
        accumulator = accumulator + i
    return accumulator

%%timeit
df.meal_price_with_tip.map(numba_foo)

"""
Explanation: 100x  Performance Improvement⌛
Numba 🤯


Up to 200x speedup from pure python  👍


Easy 👍


using numba is really easy its simply adding a decorator to a method


Highly Configurable  👍


Mostly Numeric 👎 


Example
End of explanation
"""


def another_foo(data):
    return data * 2

def foo(data):
    return data + 10


"""
Explanation: 65x  Performance Improvement⌛
1️⃣ Vectorized methods
2️⃣ Numba
3️⃣ Cython
General Python Optimizations 🐍
Caching 🏎


Avoid unnecessary work/computation.


Faster code 


Intermediate Variables👩‍👩‍👧‍👧


Intermediate calculations 


Memory foot print of both objects


Smarter variables allocation
End of explanation
"""


%reload_ext memory_profiler

def load_data():
    return np.ones((2 ** 30), dtype=np.uint8)

%%memit
def proccess():
    data = load_data()
    return another_foo(foo(data))

proccess()

%%memit
def proccess():
    data = load_data()
    data = foo(data)
    data = another_foo(data)
    return data

proccess()

"""
Explanation: Example
End of explanation
"""


