Update app.py

app.py

@@ -17,6 +17,7 @@ https://huggingface.co/datasets/datacomp
 # https://huggingface.co/datasets/ILSVRC/imagenet-1k
 NUM_EXAMPLES = 1281167
 # Arbitrary small number, only using in devv'ing (uncomment #.take(DEV_AMOUNT) below to use it).
+DEV = False
 DEV_AMOUNT = 10
 GATED_IMAGENET = os.environ.get("GATED_IMAGENET")
 
@@ -33,8 +34,12 @@ def main(percentage=10):
 
     # Load the dataset from the HF hub. Use streaming so as not to load the entire dataset at once.
     # Use the .take(DEV_AMOUNT) to only grab a small chunk of instances to develop with.
-    dataset = load_dataset("ILSVRC/imagenet-1k", split="train", streaming=True,
+    if DEV:
+        dataset = load_dataset("ILSVRC/imagenet-1k", split="train", streaming=True,
                            trust_remote_code=True, token=GATED_IMAGENET).take(DEV_AMOUNT)
+    else:
+        dataset = load_dataset("ILSVRC/imagenet-1k", split="train", streaming=True,
+                           trust_remote_code=True, token=GATED_IMAGENET)
 
     # Create a set of indices that are randomly chosen, to change their labels.
     # Specifically, randomly choose NUM_EXAMPLES/percentage indices.
@@ -46,8 +51,13 @@ def main(percentage=10):
                                   features=dataset.features, batched=True)
 
     # Upload the new version of the dataset (this will take awhile)
-    Dataset.from_generator(updated_dataset.__iter__).push_to_hub(
-        "datacomp/imagenet-1k-random-debug" + str(percentage), token=GATED_IMAGENET)
+    if DEV:
+        Dataset.from_generator(updated_dataset.__iter__).push_to_hub(
+            "datacomp/imagenet-1k-random-debug" + str(percentage), token=GATED_IMAGENET)
+    else:
+        Dataset.from_generator(updated_dataset.__iter__).push_to_hub(
+            "datacomp/imagenet-1k-random" + str(percentage), token=GATED_IMAGENET)
+
 
     end = time.time()
     print("That took %d seconds" % (end - start))