text: __author__ = 'Charlie'
  # Converting udacity tensorflow tutorial code such that notMNIST can be accessed with ease
  #Note that the taining data set size is set to 20k - The data set has 500k in large and 20k in small
  import numpy as np
  import os
  import sys
  import tarfile
  from IPython.display import display, Image
  from scipy import ndimage
  from six.moves.urllib.request import urlretrieve
  from six.moves import cPickle as pickle
  
  url = 'http://commondatastorage.googleapis.com/books1000/'
  
  num_classes = 10
  np.random.seed(133)
  
  image_size = 28  # Pixel width and height.
  pixel_depth = 255.0  # Number of levels per pixel.
  
  
  def maybe_download(dir_path, filename, expected_bytes, force=False):
      """Download a file if not present, and make sure it's the right size."""
      file_path = os.path.join(dir_path, filename)
      if force or not os.path.exists(file_path):
          file_path, _ = urlretrieve(url + filename, file_path)
      statinfo = os.stat(file_path)
      if statinfo.st_size == expected_bytes:
          print('Found and verified', filename)
      else:
          raise Exception(
              'Failed to verify ' + filename + '. Can you get to it with a browser?')
      return filename
  
  
  def maybe_extract(dir_path, filename, force=False):
      root = os.path.join(dir_path, os.path.splitext(os.path.splitext(filename)[0])[0])  # remove .tar.gz
      if os.path.isdir(root) and not force:
          # You may override by setting force=True.
          print('%s already present - Skipping extraction of %s.' % (root, filename))
      else:
          print('Extracting data for %s. This may take a while. Please wait.' % root)
          tar = tarfile.open(os.path.join(dir_path, filename))
          sys.stdout.flush()
          tar.extractall(dir_path)
          tar.close()
      data_folders = [
          os.path.join(root, d) for d in sorted(os.listdir(root))
          if os.path.isdir(os.path.join(root, d))]
      if len(data_folders) != num_classes:
          raise Exception(
              'Expected %d folders, one per class. Found %d instead.' % (
                  num_classes, len(data_folders)))
      print(data_folders)
      return data_folders
  
  
  def load_letter(folder, min_num_images):
      """Load the data for a single letter label."""
      image_files = os.listdir(folder)
      dataset = np.ndarray(shape=(len(image_files), image_size, image_size),
                           dtype=np.float32)
      image_index = 0
      print(folder)
      for image in os.listdir(folder):
          image_file = os.path.join(folder, image)
          try:
              image_data = (ndimage.imread(image_file).astype(float) -
                            pixel_depth / 2) / pixel_depth
              if image_data.shape != (image_size, image_size):
                  raise Exception('Unexpected image shape: %s' % str(image_data.shape))
              dataset[image_index, :, :] = image_data
              image_index += 1
          except IOError as e:
              print('Could not read:', image_file, ':', e, '- it\'s ok, skipping.')
  
      num_images = image_index
      dataset = dataset[0:num_images, :, :]
      if num_images < min_num_images:
          raise Exception('Many fewer images than expected: %d < %d' %
                          (num_images, min_num_images))
  
      print('Full dataset tensor:', dataset.shape)
      print('Mean:', np.mean(dataset))
      print('Standard deviation:', np.std(dataset))
      return dataset
  
  
  def maybe_pickle(data_folders, min_num_images_per_class, force=False):
      dataset_names = []
      for folder in data_folders:
          set_filename = folder + '.pickle'
          dataset_names.append(set_filename)
          if os.path.exists(set_filename) and not force:
              # You may override by setting force=True.
              print('%s already present - Skipping pickling.' % set_filename)
          else:
              print('Pickling %s.' % set_filename)
              dataset = load_letter(folder, min_num_images_per_class)
              try:
                  with open(set_filename, 'wb') as f:
                      pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
              except Exception as e:
                  print('Unable to save data to', set_filename, ':', e)
  
      return dataset_names
  
  
  def make_arrays(nb_rows, img_size):
      if nb_rows:
          dataset = np.ndarray((nb_rows, img_size, img_size), dtype=np.float32)
          labels = np.ndarray(nb_rows, dtype=np.int32)
      else:
          dataset, labels = None, None
      return dataset, labels
  
  
  def merge_datasets(pickle_files, train_size, valid_size=0):
      num_classes = len(pickle_files)
      valid_dataset, valid_labels = make_arrays(valid_size, image_size)
      train_dataset, train_labels = make_arrays(train_size, image_size)
      vsize_per_class = valid_size // num_classes
      tsize_per_class = train_size // num_classes
  
      start_v, start_t = 0, 0
      end_v, end_t = vsize_per_class, tsize_per_class
      end_l = vsize_per_class + tsize_per_class
      for label, pickle_file in enumerate(pickle_files):
          try:
              with open(pickle_file, 'rb') as f:
                  letter_set = pickle.load(f)
                  # let's shuffle the letters to have random validation and training set
                  np.random.shuffle(letter_set)
                  if valid_dataset is not None:
                      valid_letter = letter_set[:vsize_per_class, :, :]
                      valid_dataset[start_v:end_v, :, :] = valid_letter
                      valid_labels[start_v:end_v] = label
                      start_v += vsize_per_class
                      end_v += vsize_per_class
  
                  train_letter = letter_set[vsize_per_class:end_l, :, :]
                  train_dataset[start_t:end_t, :, :] = train_letter
                  train_labels[start_t:end_t] = label
                  start_t += tsize_per_class
                  end_t += tsize_per_class
          except Exception as e:
              print('Unable to process data from', pickle_file, ':', e)
              raise
  
      return valid_dataset, valid_labels, train_dataset, train_labels
  
  
  def randomize(dataset, labels):
      permutation = np.random.permutation(labels.shape[0])
      shuffled_dataset = dataset[permutation, :, :]
      shuffled_labels = labels[permutation]
      return shuffled_dataset, shuffled_labels
  
  
  def reformat(dataset, labels):
      dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
      # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]
      labels = (np.arange(num_classes) == labels[:, None]).astype(np.float32)
      return dataset, labels
  
  
  def initialize_read(data_dir):
      train_filename = maybe_download(data_dir, 'notMNIST_large.tar.gz', 247336696)
      test_filename = maybe_download(data_dir, 'notMNIST_small.tar.gz', 8458043)
  
      train_folders = maybe_extract(data_dir, train_filename)
      test_folders = maybe_extract(data_dir, test_filename)
  
      train_datasets = maybe_pickle(train_folders, 45000)
      test_datasets = maybe_pickle(test_folders, 1800)
  
      train_size = 200000
      valid_size = 10000
      test_size = 10000
  
      valid_dataset, valid_labels, train_dataset, train_labels = merge_datasets(
          train_datasets, train_size, valid_size)
      _, _, test_dataset, test_labels = merge_datasets(test_datasets, test_size)
  
      print('Training:', train_dataset.shape, train_labels.shape)
      print('Validation:', valid_dataset.shape, valid_labels.shape)
      print('Testing:', test_dataset.shape, test_labels.shape)
  
      train_dataset, train_labels = randomize(train_dataset, train_labels)
      test_dataset, test_labels = randomize(test_dataset, test_labels)
      valid_dataset, valid_labels = randomize(valid_dataset, valid_labels)
  
      pickle_file = os.path.join(data_dir, 'notMNIST.pickle')
      if not os.path.exists(pickle_file):
          try:
              f = open(pickle_file, 'wb')
              save = {
                  'train_dataset': train_dataset,
                  'train_labels': train_labels,
                  'valid_dataset': valid_dataset,
                  'valid_labels': valid_labels,
                  'test_dataset': test_dataset,
                  'test_labels': test_labels,
              }
              pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
              f.close()
          except Exception as e:
              print('Unable to save data to', pickle_file, ':', e)
              raise
  
      statinfo = os.stat(pickle_file)
      print('Compressed pickle size:', statinfo.st_size)
  
  
  def get_notMNISTData(data_dir):
      initialize_read(data_dir)
  
      pickle_file = os.path.join(data_dir, 'notMNIST.pickle')
  
      with open(pickle_file, 'rb') as f:
          save = pickle.load(f)
          train_dataset = save['train_dataset']
          train_labels = save['train_labels']
          valid_dataset = save['valid_dataset']
          valid_labels = save['valid_labels']
          test_dataset = save['test_dataset']
          test_labels = save['test_labels']
          del save  # hint to help gc free up memory
          print('Training set', train_dataset.shape, train_labels.shape)
          print('Validation set', valid_dataset.shape, valid_labels.shape)
          print('Test set', test_dataset.shape, test_labels.shape)
  
      train_dataset, train_labels = reformat(train_dataset, train_labels)
      valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
      test_dataset, test_labels = reformat(test_dataset, test_labels)
      print('Training set', train_dataset.shape, train_labels.shape)
      print('Validation set', valid_dataset.shape, valid_labels.shape)
      print('Test set', test_dataset.shape, test_labels.shape)
  
      return train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels
  