Spaces:
Running
Running
File size: 2,988 Bytes
6a67483 832f54f 9a1c210 832f54f 9a1c210 832f54f 9a1c210 d46a843 6a67483 9a1c210 8a4d47d 9a1c210 6a67483 9a1c210 e7a4071 9a1c210 832f54f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
import os
import gradio as gr
import random
import time
from datasets import load_dataset, Dataset
"""
This code is designed to read in the ImageNet 1K ILSVRC dataset from the Hugging Face Hub,
then create a new version of this dataset with {percentage} lines with random labels between 0-9,
then upload this new version of the Hugging Face Hub, in the Data Composition organization:
https://huggingface.co/datasets/datacomp
"""
# The number of examples/instances in this dataset is copied from the model card:
# https://huggingface.co/datasets/ILSVRC/imagenet-1k
NUM_EXAMPLES = 1281167
# Arbitrary small number, only using in devv'ing (uncomment #.take(DEV_AMOUNT) below to use it).
DEV_AMOUNT = 10
GATED_IMAGENET = os.environ.get("GATED_IMAGENET")
def main(percentage=10):
global randomize_subset
# Just for timing how long this takes.
start = time.time()
percentage = float(percentage)
print("Randomizing %d percent of the data." % percentage)
# Set the random seed, based on the percentage, so that our random changes are reproducible.
random.seed(percentage)
# Load the dataset from the HF hub. Use streaming so as not to load the entire dataset at once.
# Use the .take(DEV_AMOUNT) to only grab a small chunk of instances to develop with.
dataset = load_dataset("ILSVRC/imagenet-1k", split="train", streaming=True,
trust_remote_code=True, token=GATED_IMAGENET).take(DEV_AMOUNT)
# Create a set of indices that are randomly chosen, to change their labels.
# Specifically, randomly choose NUM_EXAMPLES/percentage indices.
randomize_subset = set(random.sample(range(0, NUM_EXAMPLES), round(
NUM_EXAMPLES / float(percentage))))
# Update the dataset so that the labels are randomized
updated_dataset = dataset.map(randomize_labels, with_indices=True,
features=dataset.features, batched=True)
# Upload the new version of the dataset (this will take awhile)
Dataset.from_generator(updated_dataset.__iter__).push_to_hub(
"datacomp/imagenet-1k-random" + str(percentage), token=GATED_IMAGENET)
end = time.time()
print("That took %d seconds" % (end - start))
def randomize_labels(examples, indices):
# What set of examples should be randomized in this batch?
# This is the intersection of the batch indices and the indices we randomly selected to change the labels of.
batch_subset = list(set(indices) & randomize_subset)
# If this batch has indices that we're changing the label of....
if batch_subset != []:
# Change the label to a random integer between 0 and 9
for n in range(len(indices)):
index = indices[n]
examples["label"][n] = random.randint(0,
9) if index in batch_subset else \
examples["label"][n]
return examples
demo = gr.Interface(fn=main, inputs="text", outputs="text")
demo.launch()
|