text: #!/usr/bin/env python
  
  # Copyright 2019 Google LLC
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     https://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """Script to create SSL splits from a dataset.
  """
  
  import json
  import os
  from collections import defaultdict
  
  import numpy as np
  import tensorflow.compat.v1 as tf
  from absl import app
  from absl import flags
  from tqdm import trange, tqdm
  
  from libml import data as libml_data
  from libml import utils
  
  import pickle
  
  flags.DEFINE_integer('seed', 0, 'Random seed to use, 0 for no shuffling.')
  flags.DEFINE_integer('size', 0, 'Size of labelled set.')
  
  FLAGS = flags.FLAGS
  
  
  def get_class(serialized_example):
      return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.int64)})['label']
  
  def get_class_fuzzy(serialized_example):
      return tf.parse_single_example(serialized_example, features={'label': tf.FixedLenFeature([], tf.string)})['label']
  
  
  def main(argv):
      # Specifies datasets with scheme (name:string, count:int, soft labels:bool) for quick setup.
      datasets = {('cifar10', 1, False)}
      for current_set in datasets:
          for i in range(current_set[1]):
              for seed in range(1,6):
                  FLAGS.seed = seed
                  FLAGS.size = 1000
                  argv = []
                  if current_set[1] == 1 or None:
                      argv.append(os.path.join('datasets', 'SSL2', current_set[0]))
                      argv.append(os.path.join('datasets', current_set[0] + '-train.tfrecord'))
                  else:
                      argv.append(os.path.join('datasets', 'SSL2', current_set[0] + str(i)))
                      argv.append(os.path.join('datasets', current_set[0] + str(i) + '-train.tfrecord'))
  
                  if any(not tf.gfile.Exists(f) for f in argv[1:]):
                      raise FileNotFoundError(argv[1:])
                  target = '%s.%d@%d' % (argv[0], FLAGS.seed, FLAGS.size)
                  if tf.gfile.Exists(target):
                      raise FileExistsError('For safety overwriting is not allowed', target)
                  input_files = argv[1:]
                  count = 0
                  id_class = []
                  class_id = defaultdict(list)
                  print('Computing class distribution')
                  if current_set[2]:
                      dataset = tf.data.TFRecordDataset(input_files).map(get_class_fuzzy, 4).batch(1 << 10)
                  else:
                      dataset = tf.data.TFRecordDataset(input_files).map(get_class, 4).batch(1 << 10)
                  it = dataset.make_one_shot_iterator().get_next()
                  try:
                      with tf.Session() as session, tqdm(leave=False) as t:
                          while 1:
                              old_count = count
                              for i in session.run(it):
                                  if current_set[2]:
                                      label = pickle.loads(i)
                                      i = np.argmax(label)
                                  id_class.append(i)
                                  class_id[i].append(count)
                                  count += 1
                              t.update(count - old_count)
                  except tf.errors.OutOfRangeError:
                      pass
                  print('%d records found' % count)
                  nclass = len(class_id)
                  assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1)
                  train_stats = np.array([len(class_id[i]) for i in range(nclass)], np.float64)
                  train_stats /= train_stats.max()
                  if 'stl10' in argv[1]:
                      # All of the unlabeled data is given label 0, but we know that
                      # STL has equally distributed data among the 10 classes.
                      train_stats[:] = 1
  
                  print('  Stats', ' '.join(['%.2f' % (100 * x) for x in train_stats]))
                  assert min(class_id.keys()) == 0 and max(class_id.keys()) == (nclass - 1)
                  class_id = [np.array(class_id[i], dtype=np.int64) for i in range(nclass)]
                  if FLAGS.seed:
                      np.random.seed(FLAGS.seed)
                      for i in range(nclass):
                          np.random.shuffle(class_id[i])
  
                  # Distribute labels to match the input distribution.
                  npos = np.zeros(nclass, np.int64)
                  label = []
                  for i in range(FLAGS.size):
                      c = np.argmax(train_stats - npos / max(npos.max(), 1))
                      label.append(class_id[c][npos[c]])
                      npos[c] += 1
  
                  del npos, class_id
                  label = frozenset([int(x) for x in label])
                  if 'stl10' in argv[1] and FLAGS.size == 1000:
                      data = tf.gfile.Open(os.path.join(libml_data.DATA_DIR, 'stl10_fold_indices.txt'), 'r').read()
                      label = frozenset(list(map(int, data.split('\n')[FLAGS.seed].split())))
  
                  print('Creating split in %s' % target)
                  tf.gfile.MakeDirs(os.path.dirname(target))
                  with tf.python_io.TFRecordWriter(target + '-label.tfrecord') as writer_label:
                      pos, loop = 0, trange(count, desc='Writing records')
                      for input_file in input_files:
                          for record in tf.python_io.tf_record_iterator(input_file):
                              if pos in label:
                                  writer_label.write(record)
                              pos += 1
                              loop.update()
                      loop.close()
                  with tf.gfile.Open(target + '-label.json', 'w') as writer:
                      writer.write(json.dumps(dict(distribution=train_stats.tolist(), label=sorted(label)), indent=2, sort_keys=True))
  
  
  if __name__ == '__main__':
      tf.disable_v2_behavior()
      utils.setup_tf()
      app.run(main)
  