# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
print(tf.__version__)

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()

print("x_train.shape = {}".format(x_train.shape))
print("y_train.shape = {}".format(y_train.shape))
print("x_test.shape = {}".format(x_test.shape))
print("y_test.shape = {}".format(y_test.shape))


# ### Get just cars.

def get_cifar_cars(features, labels, class_idx):
    """Returns features of only chosen class and features of other classes.

    Args:
        features: np.array, array of features.
        labels: np.array, array of int labels.
        class_idx: int, index of chosen class.
    Returns:
        Numpy arrays of chosen class features and other class features.
    """
    in_class = features[labels.flatten() == class_idx, :, :, :]
    out_class = features[labels.flatten() != class_idx, :, :, :]

    sample_indices = np.random.choice(
        a=np.arange(out_class.shape[0]), size=in_class.shape[0], replace=False
    )
    out_class = out_class[sample_indices, :, :, :]

    return in_class, out_class



car_train, anom_train = get_cifar_cars(
    features=x_train, labels=y_train, class_idx=1
)
car_test, anom_test = get_cifar_cars(
    features=x_test, labels=y_test, class_idx=1
)

print("car_train.shape = {}".format(car_train.shape))
print("anom_train.shape = {}".format(anom_train.shape))
print("car_test.shape = {}".format(car_test.shape))
print("anom_test.shape = {}".format(anom_test.shape))


def plot_images(images):
    num_images = len(images)

    plt.figure(figsize=(20, 20))
    for i in range(num_images):
        plt.subplot(1, num_images, i + 1)
        plt.xticks([])
        plt.yticks([])
        plt.grid(False)
        plt.imshow(images[i], cmap=plt.cm.binary)
    plt.show()


plot_images(images=car_train[:10])

plot_images(images=car_test[:10])

plot_images(images=anom_train[:10])

plot_images(images=anom_test[:10])


# ## Write

def write_tf_record(filepath, data):
    """Writes TFRecord files of given data to given filepath.

    Args:
        filepath: str, filepath location.
        data: dict, np.arrays of feature and label data.
    """
    with tf.io.TFRecordWriter(path=filepath) as ofp:
        for i in range(len(data["image_raw"])):
            example = tf.train.Example(
                features=tf.train.Features(
                    feature={
                        "image_raw": tf.train.Feature(
                            bytes_list=tf.train.BytesList(
                                value=[data["image_raw"][i].tobytes()]
                            )
                        ),
                        "label": tf.train.Feature(
                            int64_list=tf.train.Int64List(
                                value=[data["label"][i]]
                            )
                        )
                    }
                )
            )
            ofp.write(example.SerializeToString())



def create_tf_record_shards(num_shards, filepath, data):
    """Creates TFRecord shards of given data at given filepath.

    Args:
        num_shards: int, number of file shards to split data into.
        filepath: str, filepath location.
        data: dict, np.arrays of feature and label data.
    """
    dataset_len = data[list(data.keys())[0]].shape[0]
    num_shards = min(num_shards, dataset_len)
    shard_size = dataset_len // num_shards
    remainder = dataset_len % num_shards
    loop_limit = num_shards if remainder == 0 else num_shards - 1
    for i in range(loop_limit):
        write_tf_record(
            filepath="{}_{}.tfrecord".format(filepath, i),
            data={
                key: value[shard_size * i:shard_size * (i + 1)]
                for key, value in data.items()
            }
        )

    if remainder > 0:
        write_tf_record(
            filepath="{}_{}.tfrecord".format(filepath, num_shards - 1),
            data={
                key: value[shard_size * (num_shards - 1):shard_size * num_shards + remainder]
                for key, value in data.items()
            }
        )



# +
# Set number of shards to split dataset into.
num_shards = 10

# Loop through each progressive resolution image size.
for i in range(4):
    img_sz = 4 * 2 ** i
    print("\nWriting {0}_{0} TF Records!".format(img_sz))

    # Cars
    create_tf_record_shards(
        num_shards=num_shards,
        filepath="data/cifar10_car/train_{0}x{0}".format(img_sz),
        data={
            "image_raw": tf.image.resize(
                images=car_train, size=(img_sz, img_sz), method="nearest"
            ).numpy(),
            "label": np.ones(shape=[car_train.shape[0]], dtype=np.int32)
        }
    )
    create_tf_record_shards(
        num_shards=num_shards,
        filepath="data/cifar10_car/test_{0}x{0}".format(img_sz),
        data={
            "image_raw": tf.image.resize(
                images=car_test, size=(img_sz, img_sz), method="nearest"
            ).numpy(),
            "label": np.ones(shape=[car_test.shape[0]], dtype=np.int32)
        }
    )

    # Non-cars
    create_tf_record_shards(
        num_shards=num_shards,
        filepath="data/cifar10_non_car/train_{0}x{0}".format(img_sz),
        data={
            "image_raw": tf.image.resize(
                images=anom_train, size=(img_sz, img_sz), method="nearest"
            ).numpy(),
            "label": np.ones(shape=[car_train.shape[0]], dtype=np.int32)
        }
    )
    create_tf_record_shards(
        num_shards=num_shards,
        filepath="data/cifar10_non_car/test_{0}x{0}".format(img_sz),
        data={
            "image_raw": tf.image.resize(
                images=anom_test, size=(img_sz, img_sz), method="nearest"
            ).numpy(),
            "label": np.ones(shape=[car_test.shape[0]], dtype=np.int32)
        }
    )


# -

# ## Read

def print_obj(function_name, object_name, object_value):
    """Prints enclosing function, object name, and object value.

    Args:
        function_name: str, name of function.
        object_name: str, name of object.
        object_value: object, value of passed object.
    """
#     pass
    print("{}: {} = {}".format(function_name, object_name, object_value))


# +
def decode_example(protos, params):
    """Decodes TFRecord file into tensors.

    Given protobufs, decode into image and label tensors.

    Args:
        protos: protobufs from TFRecord file.
        params: dict, user passed parameters.

    Returns:
        Image and label tensors.
    """
    # Create feature schema map for protos.
    features = {
        "image_raw": tf.io.FixedLenFeature(shape=[], dtype=tf.string),
        "label": tf.io.FixedLenFeature(shape=[], dtype=tf.int64)
    }

    # Parse features from tf.Example.
    parsed_features = tf.io.parse_single_example(
        serialized=protos, features=features
    )
    print_obj("\ndecode_example", "features", features)

    # Convert from a scalar string tensor (whose single string has
    # length height * width * depth) to a uint8 tensor with shape
    # [height * width * depth].
    image = tf.io.decode_raw(
        input_bytes=parsed_features["image_raw"], out_type=tf.uint8
    )
    print_obj("decode_example", "image", image)

    # Reshape flattened image back into normal dimensions.
    image = tf.reshape(
        tensor=image,
        shape=[params["height"], params["width"], params["depth"]]
    )
    print_obj("decode_example", "image", image)

    # Convert from [0, 255] -> [-1.0, 1.0] floats.
    image_scaled = tf.cast(x=image, dtype=tf.float32) * (2. / 255) - 1.0
    print_obj("decode_example", "image", image)

    # Convert label from a scalar uint8 tensor to an int32 scalar.
    label = tf.cast(x=parsed_features["label"], dtype=tf.int32)
    print_obj("decode_example", "label", label)

    return {"image": image, "image_scaled": image_scaled}, label

def read_dataset(filename, mode, batch_size, params):
    """Reads CSV time series data using tf.data, doing necessary preprocessing.

    Given filename, mode, batch size, and other parameters, read CSV dataset
    using Dataset API, apply necessary preprocessing, and return an input
    function to the Estimator API.

    Args:
        filename: str, file pattern that to read into our tf.data dataset.
        mode: The estimator ModeKeys. Can be TRAIN or EVAL.
        batch_size: int, number of examples per batch.
        params: dict, dictionary of user passed parameters.

    Returns:
        An input function.
    """
    def _input_fn():
        """Wrapper input function used by Estimator API to get data tensors.

        Returns:
            Batched dataset object of dictionary of feature tensors and label
                tensor.
        """
        # Create list of files that match pattern.
        file_list = tf.io.gfile.glob(pattern=filename)

        # Create dataset from file list.
        dataset = tf.data.TFRecordDataset(
            filenames=file_list, num_parallel_reads=40
        )

        # Shuffle and repeat if training with fused op.
        if mode == tf.estimator.ModeKeys.TRAIN:
            dataset = dataset.apply(
                tf.contrib.data.shuffle_and_repeat(
                    buffer_size=50 * batch_size,
                    count=None  # indefinitely
                )
            )

        # Decode example into a features dictionary of tensors, then batch.
        dataset = dataset.map(
            map_func=lambda x: decode_example(
                protos=x,
                params=params
            ),
            num_parallel_calls=4
        )

        dataset = dataset.batch(batch_size=batch_size)

        # Prefetch data to improve latency.
        dataset = dataset.prefetch(buffer_size=2)

        return dataset
    return _input_fn



# -

def try_out_input_function(arguments, print_out=False):
    """Trys out input function for testing purposes.

    Args:
        arguments: dict, user passed parameters.
    """
    fn = read_dataset(
        filename=arguments["filename"],
        mode=tf.estimator.ModeKeys.EVAL,
        batch_size=8,
        params=arguments
    )

    dataset = fn()

    batches = [batch for batch in dataset]
    features, labels = batches[0]
    print("features[image].shape = {}".format(features["image"].shape))
    print("labels.shape = {}".format(labels.shape))
#         print("features = \n{}".format(features))
    print("labels = \n{}".format(labels))

    return features, labels



def plot_all_resolutions(filepath, starting_size, num_doublings, depth):
    """Plots all resolutions of images.

    Args:
        filepath: str, filepath location.
        starting_size: int, smallest/initial size of image.
        num_doublings: int, number of times to double height/width of original
            image size.
        depth: int, number of image channels.
    """
    for i in range(num_doublings):
        img_sz = starting_size * 2 ** i
        print("\nImage size = {0}x{0}".format(img_sz))

        features, labels = try_out_input_function(
            arguments={
                "filename": "{0}_{1}x{1}_0.tfrecord".format(
                    filepath, img_sz
                ),
                "height": img_sz,
                "width": img_sz,
                "depth": depth
            },
            print_out=True
        )
        plot_images(features["image"])



plot_all_resolutions(
    filepath="data/cifar10_car/train",
    starting_size=4,
    num_doublings=4,
    depth=3
)

plot_all_resolutions(
    filepath="data/cifar10_car/test",
    starting_size=4,
    num_doublings=4,
    depth=3
)

plot_all_resolutions(
    filepath="data/cifar10_non_car/train",
    starting_size=4,
    num_doublings=4,
    depth=3
)

plot_all_resolutions(
    filepath="data/cifar10_non_car/test",
    starting_size=4,
    num_doublings=4,
    depth=3
)



StarcoderdataJupyterScriptsDedupFiltered