import functools

import numpy as np
import tensorflow as tf

print("TensorFlow version: ", tf.version.VERSION)

TRAIN_DATA_URL = "https://storage.googleapis.com/tf-datasets/titanic/train.csv"
TEST_DATA_URL = "https://storage.googleapis.com/tf-datasets/titanic/eval.csv"

train_file_path = tf.keras.utils.get_file("train.csv", TRAIN_DATA_URL)
test_file_path = tf.keras.utils.get_file("eval.csv", TEST_DATA_URL)

# Make numpy values easier to read.
np.set_printoptions(precision=3, suppress=True)

"""
Explanation: How to Load CSV and Numpy File Types in TensorFlow 2.0
Learning Objectives

Load a CSV file into a tf.data.Dataset. 
Load Numpy data

Introduction
In this lab, you load CSV data from a file into a tf.data.Dataset.  This tutorial provides an example of loading data from NumPy arrays into a tf.data.Dataset you also load text data.
Each learning objective will correspond to a #TODO in the student lab notebook -- try to complete that notebook first before reviewing this solution notebook.
Load necessary libraries
We will start by importing the necessary libraries for this lab.
End of explanation
"""


!head {train_file_path}

"""
Explanation: Load data
This section provides an example of how to load CSV data from a file into a tf.data.Dataset.  The data used in this tutorial are taken from the Titanic passenger list. The model will predict the likelihood a passenger survived based on characteristics like age, gender, ticket class, and whether the person was traveling alone.
To start, let's look at the top of the CSV file to see how it is formatted.
End of explanation
"""


# TODO 1
LABEL_COLUMN = "survived"
LABELS = [0, 1]

"""
Explanation: You can load this using pandas, and pass the NumPy arrays to TensorFlow. If you need to scale up to a large set of files, or need a loader that integrates with TensorFlow and tf.data then use the tf.data.experimental.make_csv_dataset function:
The only column you need to identify explicitly is the one with the value that the model is intended to predict.
End of explanation
"""


def get_dataset(file_path, **kwargs):
    # TODO 2
    dataset = tf.data.experimental.make_csv_dataset(
        file_path,
        batch_size=5,  # Artificially small to make examples easier to show.
        label_name=LABEL_COLUMN,
        na_value="?",
        num_epochs=1,
        ignore_errors=True,
        **kwargs,
    )
    return dataset


raw_train_data = get_dataset(train_file_path)
raw_test_data = get_dataset(test_file_path)

def show_batch(dataset):
    for batch, label in dataset.take(1):
        for key, value in batch.items():
            print(f"{key:20s}: {value.numpy()}")

"""
Explanation: Now read the CSV data from the file and create a dataset. 
(For the full documentation, see tf.data.experimental.make_csv_dataset)
End of explanation
"""


show_batch(raw_train_data)

"""
Explanation: Each item in the dataset is a batch, represented as a tuple of (many examples, many labels). The data from the examples is organized in column-based tensors (rather than row-based tensors), each with as many elements as the batch size (5 in this case).
It might help to see this yourself.
End of explanation
"""


CSV_COLUMNS = [
    "survived",
    "sex",
    "age",
    "n_siblings_spouses",
    "parch",
    "fare",
    "class",
    "deck",
    "embark_town",
    "alone",
]

temp_dataset = get_dataset(train_file_path, column_names=CSV_COLUMNS)

show_batch(temp_dataset)

"""
Explanation: As you can see, the columns in the CSV are named. The dataset constructor will pick these names up automatically. If the file you are working with does not contain the column names in the first line, pass them in a list of strings to  the column_names argument in the make_csv_dataset function.
End of explanation
"""


SELECT_COLUMNS = [
    "survived",
    "age",
    "n_siblings_spouses",
    "class",
    "deck",
    "alone",
]

temp_dataset = get_dataset(train_file_path, select_columns=SELECT_COLUMNS)

show_batch(temp_dataset)

"""
Explanation: This example is going to use all the available columns. If you need to omit some columns from the dataset, create a list of just the columns you plan to use, and pass it into the (optional) select_columns argument of the constructor.
End of explanation
"""


SELECT_COLUMNS = ["survived", "age", "n_siblings_spouses", "parch", "fare"]
DEFAULTS = [0, 0.0, 0.0, 0.0, 0.0]
temp_dataset = get_dataset(
    train_file_path, select_columns=SELECT_COLUMNS, column_defaults=DEFAULTS
)

show_batch(temp_dataset)

example_batch, labels_batch = next(iter(temp_dataset))

"""
Explanation: Data preprocessing
A CSV file can contain a variety of data types. Typically you want to convert from those mixed types to a fixed length vector before feeding the data into your model.
TensorFlow has a built-in system for describing common input conversions: tf.feature_column, see this tutorial for details.
You can preprocess your data using any tool you like (like nltk or sklearn), and just pass the processed output to TensorFlow. 
The primary advantage of doing the preprocessing inside your model is that when you export the model it includes the preprocessing. This way you can pass the raw data directly to your model.
Continuous data
If your data is already in an appropriate numeric format, you can pack the data into a vector before passing it off to the model:
End of explanation
"""


def pack(features, label):
    return tf.stack(list(features.values()), axis=-1), label

"""
Explanation: Here's a simple function that will pack together all the columns:
End of explanation
"""


packed_dataset = temp_dataset.map(pack)

for features, labels in packed_dataset.take(1):
    print(features.numpy())
    print()
    print(labels.numpy())

"""
Explanation: Apply this to each element of the dataset:
End of explanation
"""


show_batch(raw_train_data)

example_batch, labels_batch = next(iter(temp_dataset))

"""
Explanation: If you have mixed datatypes you may want to separate out these simple-numeric fields. The tf.feature_column api can handle them, but this incurs some overhead and should be avoided unless really necessary. Switch back to the mixed dataset:
End of explanation
"""


class PackNumericFeatures:
    def __init__(self, names):
        self.names = names

    def __call__(self, features, labels):
        numeric_features = [features.pop(name) for name in self.names]
        numeric_features = [
            tf.cast(feat, tf.float32) for feat in numeric_features
        ]
        numeric_features = tf.stack(numeric_features, axis=-1)
        features["numeric"] = numeric_features

        return features, labels

NUMERIC_FEATURES = ["age", "n_siblings_spouses", "parch", "fare"]

packed_train_data = raw_train_data.map(PackNumericFeatures(NUMERIC_FEATURES))

packed_test_data = raw_test_data.map(PackNumericFeatures(NUMERIC_FEATURES))

show_batch(packed_train_data)

example_batch, labels_batch = next(iter(packed_train_data))

"""
Explanation: So define a more general preprocessor that selects a list of numeric features and packs them into a single column:
End of explanation
"""


import pandas as pd

desc = pd.read_csv(train_file_path)[NUMERIC_FEATURES].describe()
desc

# TODO 1
MEAN = np.array(desc.T["mean"])
STD = np.array(desc.T["std"])

def normalize_numeric_data(data, mean, std):
    # TODO 2
    # Center the data
    return (data - mean) / std

print(MEAN, STD)

"""
Explanation: Data Normalization
Continuous data should always be normalized.
End of explanation
"""


# See what you just created.
normalizer = functools.partial(normalize_numeric_data, mean=MEAN, std=STD)

numeric_column = tf.feature_column.numeric_column(
    "numeric", normalizer_fn=normalizer, shape=[len(NUMERIC_FEATURES)]
)
numeric_columns = [numeric_column]
numeric_column

"""
Explanation: Now create a numeric column. The tf.feature_columns.numeric_column API accepts a normalizer_fn argument, which will be run on each batch.
Bind the MEAN and STD to the normalizer fn using functools.partial.
End of explanation
"""


example_batch["numeric"]

numeric_layer = tf.keras.layers.DenseFeatures(numeric_columns)
numeric_layer(example_batch).numpy()

"""
Explanation: When you train the model, include this feature column to select and center this block of numeric data:
End of explanation
"""


CATEGORIES = {
    "sex": ["male", "female"],
    "class": ["First", "Second", "Third"],
    "deck": ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"],
    "embark_town": ["Cherbourg", "Southhampton", "Queenstown"],
    "alone": ["y", "n"],
}

categorical_columns = []
for feature, vocab in CATEGORIES.items():
    cat_col = tf.feature_column.categorical_column_with_vocabulary_list(
        key=feature, vocabulary_list=vocab
    )
    categorical_columns.append(tf.feature_column.indicator_column(cat_col))

# See what you just created.
categorical_columns

categorical_layer = tf.keras.layers.DenseFeatures(categorical_columns)
print(categorical_layer(example_batch).numpy()[0])

"""
Explanation: The mean based normalization used here requires knowing the means of each column ahead of time.
Categorical data
Some of the columns in the CSV data are categorical columns. That is, the content should be one of a limited set of options.
Use the tf.feature_column API to create a collection with a tf.feature_column.indicator_column for each categorical column.
End of explanation
"""


# TODO 1
preprocessing_layer = tf.keras.layers.DenseFeatures(
    categorical_columns + numeric_columns
)

print(preprocessing_layer(example_batch).numpy()[0])

"""
Explanation: This will be become part of a data processing input later when you build the model.
Combined preprocessing layer
Add the two feature column collections and pass them to a tf.keras.layers.DenseFeatures to create an input layer that will extract and preprocess both input types:
End of explanation
"""


import numpy as np
import tensorflow as tf

print("TensorFlow version: ", tf.version.VERSION)

"""
Explanation: Next Step
A next step would be to build a build a tf.keras.Sequential, starting with the preprocessing_layer, which is beyond the scope of this lab.  We will cover the Keras Sequential API in the next Lesson.
Load NumPy data
Load necessary libraries
First, restart the Kernel.  Then, we will start by importing the necessary libraries for this lab.
End of explanation
"""


DATA_URL = (
    "https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz"
)

path = tf.keras.utils.get_file("mnist.npz", DATA_URL)
with np.load(path) as data:
    # TODO 1
    train_examples = data["x_train"]
    train_labels = data["y_train"]
    test_examples = data["x_test"]
    test_labels = data["y_test"]

"""
Explanation: Load data from .npz file
We use the MNIST dataset in Keras.
End of explanation
"""


# TODO 2
train_dataset = tf.data.Dataset.from_tensor_slices(
    (train_examples, train_labels)
)
test_dataset = tf.data.Dataset.from_tensor_slices((test_examples, test_labels))

"""
Explanation: Load NumPy arrays with tf.data.Dataset
Assuming you have an array of examples and a corresponding array of labels, pass the two arrays as a tuple into tf.data.Dataset.from_tensor_slices to create a tf.data.Dataset.
End of explanation
"""


