# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernel_info:
#     name: python3-azureml
#   kernelspec:
#     display_name: Python 3.6 - AzureML
#     language: python
#     name: python3-azureml
# ---

# + [markdown] nteract={"transient": {"deleting": false}}
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# Licensed under the MIT License.

# + [markdown] nteract={"transient": {"deleting": false}}
# # Prepare Dataset
#
# This notebook file is for preparing the dataset that will be used for the [dataset_search_tool.ipynb](./dataset_search_tool.ipynb). 
# In this notebook, you will use two different NYC Taxi datasets. You need to register and update dataset with tag information.
#
# This tutorial includes the following tasks:
# * Prepare data
# * Download data locally and then upload to Azure Blob
# * Create tag filter method
# * Search tag using predefined method 

# + [markdown] nteract={"transient": {"deleting": false}}
# ## Prerequisite
#
# If you are using an Azure Machine Learning Notebook VM, you are all set. Otherwise, make sure you go through the configuration Notebook located at https://github.com/Azure/MachineLearningNotebooks first if you haven't. This sets you up with a working config file that has information on your workspace, subscription id, etc.

# + [markdown] nteract={"transient": {"deleting": false}}
# ## Prepare data
#
# We will leverage the convenience of Azure Open Datasets along with the power of Azure Machine Learning service to create a regression model to predict NYC taxi fare prices. Perform `pip install azureml-opendatasets` to get the open dataset package. The Open Datasets package contains a class representing each data source (NycTlcGreen and NycTlcYellow) to easily filter date parameters before downloading.

# + [markdown] nteract={"transient": {"deleting": false}}
# ### Load data
#
#

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906292740}
import azureml.core
# Check core SDK version number
print("SDK version:", azureml.core.VERSION)

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906396839}
from azureml.opendatasets import NycTlcGreen, NycTlcYellow
import pandas as pd
from datetime import datetime
from dateutil.relativedelta import relativedelta

green_df_raw = pd.DataFrame([])
start = datetime.strptime("1/1/2016","%m/%d/%Y")
end = datetime.strptime("1/31/2016","%m/%d/%Y")

number_of_months = 1
sample_size = 5000

for sample_month in range(number_of_months):
    temp_df_green = NycTlcGreen(start + relativedelta(months=sample_month), end + relativedelta(months=sample_month)) \
        .to_pandas_dataframe()
    green_df_raw = green_df_raw.append(temp_df_green.sample(sample_size))

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906451475}
yellow_df_raw = pd.DataFrame([])
start = datetime.strptime("1/1/2016","%m/%d/%Y")
end = datetime.strptime("1/31/2016","%m/%d/%Y")

sample_size = 500

for sample_month in range(number_of_months):
    temp_df_yellow = NycTlcYellow(start + relativedelta(months=sample_month), end + relativedelta(months=sample_month)) \
        .to_pandas_dataframe()
    yellow_df_raw = yellow_df_raw.append(temp_df_yellow.sample(sample_size))

# + [markdown] nteract={"transient": {"deleting": false}}
# ### See the data

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906488982}
from IPython.display import display

display(green_df_raw.head(5))
display(yellow_df_raw.head(5))

# + [markdown] nteract={"transient": {"deleting": false}}
# ## Download data locally and then upload to Azure Blob

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906496159}
import os
dataDir = "data"

if not os.path.exists(dataDir):
    os.mkdir(dataDir)

greenDir = dataDir + "/green"
yelloDir = dataDir + "/yellow"

if not os.path.exists(greenDir):
    os.mkdir(greenDir)
    
if not os.path.exists(yelloDir):
    os.mkdir(yelloDir)
    
greenTaxiData = greenDir + "/unprepared.parquet"
yellowTaxiData = yelloDir + "/unprepared.parquet"

green_df_raw.to_csv(greenTaxiData, index=False)
yellow_df_raw.to_csv(yellowTaxiData, index=False)

print("Data written to local folder.")

# + [markdown] nteract={"transient": {"deleting": false}}
# ## Create and register datasets

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906618982}

from azureml.core import Workspace

ws = Workspace.from_config()
print("Workspace: " + ws.name, "Region: " + ws.location, sep = '\n')

# Default datastore
default_store = ws.get_default_datastore() 

default_store.upload_files([greenTaxiData], 
                           target_path = 'green', 
                           overwrite = True, 
                           show_progress = True)

default_store.upload_files([yellowTaxiData], 
                           target_path = 'yellow', 
                           overwrite = True, 
                           show_progress = True)

print("Upload calls completed.")

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906655337}
from azureml.core import Dataset
green_taxi_data = Dataset.Tabular.from_delimited_files(default_store.path('green/unprepared.parquet'))
yellow_taxi_data = Dataset.Tabular.from_delimited_files(default_store.path('yellow/unprepared.parquet'))

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614906665036}
green_taxi_data = green_taxi_data.register(ws, 'green_taxi_data')
yellow_taxi_data = yellow_taxi_data.register(ws, 'yellow_taxi_data')

# + [markdown] nteract={"transient": {"deleting": false}}
# ## Update dataset with tag information

# + jupyter={"source_hidden": false, "outputs_hidden": false} nteract={"transient": {"deleting": false}} gather={"logged": 1614907517125}
green_taxi_data.update(tags={'version': 'original', 'type': 'green'})
yellow_taxi_data.update(tags={'version': 'original', 'type': 'yellow'})

StarcoderdataJupyterScriptsDedupFiltered