!pip install modin[all] 

"""
Explanation: <center><h2>Scale your pandas workflows by changing one line of code</h2>
Getting Started
To install the most recent stable release for Modin run the following code on your command line:
End of explanation
"""


import modin.pandas as pd
import pandas

#############################################
### For the purpose of timing comparisons ###
#############################################
import time
import ray
ray.init()
from IPython.display import Markdown, display
def printmd(string):
    display(Markdown(string))

"""
Explanation: For further instructions on how to install Modin with conda or for specific platforms or engines, see our detailed installation guide.
Modin acts as a drop-in replacement for pandas so you can simply change a single line of import to speed up your pandas workflows. To use Modin, you simply have to replace the import of pandas with the import of Modin, as follows.
End of explanation
"""


# This may take a few minutes to download
import urllib.request
s3_path = "https://modin-test.s3.us-west-1.amazonaws.com/yellow_tripdata_2015-01.csv"
urllib.request.urlretrieve(s3_path, "taxi.csv")  

"""
Explanation: Dataset: NYC taxi trip data
Link to raw dataset: https://modin-test.s3.us-west-1.amazonaws.com/yellow_tripdata_2015-01.csv (Size: ~200MB)
End of explanation
"""


start = time.time()

pandas_df = pandas.read_csv("taxi.csv", parse_dates=["tpep_pickup_datetime", "tpep_dropoff_datetime"], quoting=3)

end = time.time()
pandas_duration = end - start
print("Time to read with pandas: {} seconds".format(round(pandas_duration, 3)))

start = time.time()

modin_df = pd.read_csv("taxi.csv", parse_dates=["tpep_pickup_datetime", "tpep_dropoff_datetime"], quoting=3)

end = time.time()
modin_duration = end - start
print("Time to read with Modin: {} seconds".format(round(modin_duration, 3)))

printmd("## Modin is {}x faster than pandas at `read_csv`!".format(round(pandas_duration / modin_duration, 2)))

"""
Explanation: Faster Data Loading with Modin's read_csv
End of explanation
"""


pandas_df

modin_df

"""
Explanation: You can quickly check that the result from pandas and Modin is exactly the same.
End of explanation
"""


N_copies= 100
start = time.time()

big_pandas_df = pandas.concat([pandas_df for _ in range(N_copies)])

end = time.time()
pandas_duration = end - start
print("Time to concat with pandas: {} seconds".format(round(pandas_duration, 3)))

start = time.time()

big_modin_df = pd.concat([modin_df for _ in range(N_copies)])

end = time.time()
modin_duration = end - start
print("Time to concat with Modin: {} seconds".format(round(modin_duration, 3)))

printmd("### Modin is {}x faster than pandas at `concat`!".format(round(pandas_duration / modin_duration, 2)))

"""
Explanation: Faster Append with Modin's concat
Our previous read_csv example operated on a relatively small dataframe. In the following example, we duplicate the same taxi dataset 100 times and then concatenate them together.
End of explanation
"""


big_modin_df.info()

"""
Explanation: The result dataset is around 19GB in size.
End of explanation
"""


start = time.time()
rounded_trip_distance_pandas = big_pandas_df["trip_distance"].apply(round)

end = time.time()
pandas_duration = end - start
print("Time to apply with pandas: {} seconds".format(round(pandas_duration, 3)))

start = time.time()

rounded_trip_distance_modin = big_modin_df["trip_distance"].apply(round)

end = time.time()
modin_duration = end - start
print("Time to apply with Modin: {} seconds".format(round(modin_duration, 3)))

printmd("### Modin is {}x faster than pandas at `apply` on one column!".format(round(pandas_duration / modin_duration, 2)))

"""
Explanation: Faster apply over a single column
The performance benefits of Modin becomes aparent when we operate on large gigabyte-scale datasets. For example, let's say that we want to round up the number across a single column via the apply operation.
End of explanation
"""


