aashraychegu commited on
Commit
e86180f
·
1 Parent(s): 4500ac5

Upload semanticallysegmentdeezglaciers.ipynb

Browse files
Files changed (1) hide show
  1. semanticallysegmentdeezglaciers.ipynb +351 -81
semanticallysegmentdeezglaciers.ipynb CHANGED
@@ -2,7 +2,7 @@
2
  "cells": [
3
  {
4
  "cell_type": "code",
5
- "execution_count": 1,
6
  "metadata": {
7
  "colab": {
8
  "base_uri": "https://localhost:8080/"
@@ -10,24 +10,7 @@
10
  "id": "c0C76YvrvDbu",
11
  "outputId": "526c8200-e257-45d7-89ec-6c4d6f30d5d0"
12
  },
13
- "outputs": [
14
- {
15
- "name": "stdout",
16
- "output_type": "stream",
17
- "text": [
18
- "1\n",
19
- "2\n",
20
- "3\n",
21
- "4\n",
22
- "5\n",
23
- "5.1\n",
24
- "5.2\n",
25
- "5.3\n",
26
- "6\n",
27
- "7\n"
28
- ]
29
- }
30
- ],
31
  "source": [
32
  "import torch\n",
33
  "import matplotlib.pyplot as plt\n",
@@ -68,7 +51,7 @@
68
  },
69
  {
70
  "cell_type": "code",
71
- "execution_count": 22,
72
  "metadata": {
73
  "colab": {
74
  "base_uri": "https://localhost:8080/",
@@ -156,43 +139,12 @@
156
  "id": "kOiKU_-vvDb1",
157
  "outputId": "531092ef-a3b9-4156-9d9c-a1835feece0a"
158
  },
159
- "outputs": [
160
- {
161
- "name": "stderr",
162
- "output_type": "stream",
163
- "text": [
164
- "Found cached dataset parquet (C:/Users/aashr/.cache/huggingface/datasets/glacierscopessegmentation___parquet/glacierscopessegmentation--secondleg-718284968c2f234c/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)\n"
165
- ]
166
- },
167
- {
168
- "data": {
169
- "application/vnd.jupyter.widget-view+json": {
170
- "model_id": "593aa8d59e094d338a2fc5cf0121e1db",
171
- "version_major": 2,
172
- "version_minor": 0
173
- },
174
- "text/plain": [
175
- " 0%| | 0/1 [00:00<?, ?it/s]"
176
- ]
177
- },
178
- "metadata": {},
179
- "output_type": "display_data"
180
- },
181
- {
182
- "data": {
183
- "text/plain": [
184
- "(8033, 423)"
185
- ]
186
- },
187
- "execution_count": 22,
188
- "metadata": {},
189
- "output_type": "execute_result"
190
- }
191
- ],
192
  "source": [
193
- "ds = load_dataset(\"glacierscopessegmentation/secondleg\")\n",
 
194
  "\n",
195
- "ds = ds[\"train\"].train_test_split(.05)\n",
196
  "train_ds = ds[\"train\"]\n",
197
  "test_ds = ds[\"test\"]\n",
198
  "\n",
@@ -211,7 +163,33 @@
211
  },
212
  {
213
  "cell_type": "code",
214
- "execution_count": 17,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
215
  "metadata": {
216
  "colab": {
217
  "base_uri": "https://localhost:8080/",
@@ -233,20 +211,7 @@
233
  "id": "PAvIJWo1vDb3",
234
  "outputId": "06c909f3-8500-49f6-bca7-b475b1d86885"
235
  },
236
- "outputs": [
237
- {
238
- "name": "stderr",
239
- "output_type": "stream",
240
- "text": [
241
- "c:\\Program Files\\Python310\\lib\\site-packages\\transformers\\models\\segformer\\image_processing_segformer.py:99: FutureWarning: The `reduce_labels` parameter is deprecated and will be removed in a future version. Please use `do_reduce_labels` instead.\n",
242
- " warnings.warn(\n",
243
- "C:\\Users\\aashr\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\storage.py:315: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.\n",
244
- " warnings.warn(message, UserWarning)\n",
245
- "Some weights of SegformerForSemanticSegmentation were not initialized from the model checkpoint at nvidia/MiT-b0 and are newly initialized: ['decode_head.linear_c.3.proj.weight', 'decode_head.linear_fuse.weight', 'decode_head.classifier.weight', 'decode_head.batch_norm.bias', 'decode_head.linear_c.1.proj.bias', 'decode_head.linear_c.0.proj.bias', 'decode_head.linear_c.1.proj.weight', 'decode_head.linear_c.0.proj.weight', 'decode_head.batch_norm.running_mean', 'decode_head.linear_c.3.proj.bias', 'decode_head.linear_c.2.proj.weight', 'decode_head.batch_norm.num_batches_tracked', 'decode_head.batch_norm.running_var', 'decode_head.classifier.bias', 'decode_head.linear_c.2.proj.bias', 'decode_head.batch_norm.weight']\n",
246
- "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
247
- ]
248
- }
249
- ],
250
  "source": [
251
  "# Define the checkpoint from which to load the pre-trained model preprocessor\n",
252
  "checkpoint = \"nvidia/MiT-b0\" # We need to use this processor for resizing the images from the dataset to the size expected by the model; the main problem with this is the output scaling for training and testing, so using the right prepreocessor is important\n",
@@ -267,13 +232,15 @@
267
  "test_image_processor = SegformerImageProcessor.from_pretrained(checkpoint)\n",
268
  "\n",
269
  "# Create a Segformer model for semantic segmentation using the test configuration and move it to the GPU\n",
270
- "test_model = SegformerForSemanticSegmentation(test_config).to(\"cuda:0\")\n",
271
- "\n"
 
 
272
  ]
273
  },
274
  {
275
  "cell_type": "code",
276
- "execution_count": 13,
277
  "metadata": {},
278
  "outputs": [],
279
  "source": [
@@ -291,7 +258,7 @@
291
  },
292
  {
293
  "cell_type": "code",
294
- "execution_count": 24,
295
  "metadata": {
296
  "id": "L-Eojv9VvDb3"
297
  },
@@ -337,15 +304,21 @@
337
  " # This is input that has gone through the model's forward pass\n",
338
  " logits, labels = eval_pred\n",
339
  " logits_tensor = torch.from_numpy(logits)\n",
 
 
 
340
  " # this can lead to very high ram usage for the upscaling\n",
341
  " logits_tensor = nn.functional.interpolate(\n",
342
  " logits_tensor,\n",
343
- " size=labels.shape[-2:],\n",
344
  " mode=\"bilinear\",\n",
345
  " align_corners=False,\n",
346
  " )\n",
 
 
 
 
347
  " # Take the argmax of the logits tensor along dimension 1 to get the predicted labels\n",
348
- " logits_tensor = logits_tensor.argmax(dim=1)\n",
349
  " # Detach the predicted labels from the computation graph and move them to the CPU \n",
350
  " # (although they are already on the CPU) to save memory and to use numpy features like the metrics module\n",
351
  " pred_labels = logits_tensor.detach().cpu().numpy()\n",
@@ -374,15 +347,15 @@
374
  "training_args = TrainingArguments(\n",
375
  " output_dir=\"glacformer\", # The output directory for the model predictions and checkpoints\n",
376
  " learning_rate=6e-5, # The initial learning rate for Adam\n",
377
- " num_train_epochs=1, # Total number of training epochs to perform\n",
378
  " auto_find_batch_size=True, # Whether to automatically find an appropriate batch size\n",
379
  " save_total_limit=3, # Limit the total amount of checkpoints and delete the older checkpoints\n",
380
- " eval_accumulation_steps=0, # Number of steps to accumulate gradients before performing a backward/update pass\n",
381
  " evaluation_strategy=\"epoch\", # The evaluation strategy to adopt during training\n",
382
  " save_strategy=\"epoch\", # The checkpoint save strategy to adopt during training\n",
383
  " save_steps=1, # Number of updates steps before two checkpoint saves\n",
384
  " eval_steps=1, # Number of update steps before two evaluations\n",
385
- " logging_steps=30, # Number of update steps before logging learning rate and other metrics\n",
386
  " remove_unused_columns=False, # Whether to remove columns not used by the model when using a dataset\n",
387
  " fp16=True, # Whether to use 16-bit float precision instead of 32-bit for saving memory\n",
388
  " tf32=True, # Whether to use tf32 precision instead of 32-bit for saving memory\n",
@@ -400,6 +373,13 @@
400
  ")"
401
  ]
402
  },
 
 
 
 
 
 
 
403
  {
404
  "cell_type": "code",
405
  "execution_count": null,
@@ -419,12 +399,21 @@
419
  "trainer.model.save_pretrained(\"glacformer\")\n",
420
  "\n",
421
  "# Create a repository object for the specified repository on Hugging Face's hub, cloning from the specified source\n",
422
- "repo = huggingface_hub.Repository(\"glacformer\", clone_from=hf_model_name)\n",
 
 
423
  "\n",
424
  "repo.git_pull()\n",
425
  "repo.push_to_hub()"
426
  ]
427
  },
 
 
 
 
 
 
 
428
  {
429
  "cell_type": "code",
430
  "execution_count": null,
@@ -486,6 +475,287 @@
486
  "\n",
487
  "glacformer.display(display)"
488
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  }
490
  ],
491
  "metadata": {
@@ -508,7 +778,7 @@
508
  "name": "python",
509
  "nbconvert_exporter": "python",
510
  "pygments_lexer": "ipython3",
511
- "version": "3.10.11"
512
  },
513
  "widgets": {
514
  "application/vnd.jupyter.widget-state+json": {
 
2
  "cells": [
3
  {
4
  "cell_type": "code",
5
+ "execution_count": null,
6
  "metadata": {
7
  "colab": {
8
  "base_uri": "https://localhost:8080/"
 
10
  "id": "c0C76YvrvDbu",
11
  "outputId": "526c8200-e257-45d7-89ec-6c4d6f30d5d0"
12
  },
13
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  "source": [
15
  "import torch\n",
16
  "import matplotlib.pyplot as plt\n",
 
51
  },
52
  {
53
  "cell_type": "code",
54
+ "execution_count": null,
55
  "metadata": {
56
  "colab": {
57
  "base_uri": "https://localhost:8080/",
 
139
  "id": "kOiKU_-vvDb1",
140
  "outputId": "531092ef-a3b9-4156-9d9c-a1835feece0a"
141
  },
142
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  "source": [
144
+ "ds = load_dataset(\"glacierscopessegmentation/scopes\",keep_in_memory=True,)\n",
145
+ "ds = datasets.concatenate_datasets((ds[\"test\"], ds[\"train\"]))\n",
146
  "\n",
147
+ "ds = ds.train_test_split(.05)\n",
148
  "train_ds = ds[\"train\"]\n",
149
  "test_ds = ds[\"test\"]\n",
150
  "\n",
 
163
  },
164
  {
165
  "cell_type": "code",
166
+ "execution_count": null,
167
+ "metadata": {},
168
+ "outputs": [],
169
+ "source": [
170
+ "ds = load_dataset(\"glacierscopessegmentation/scopes\",keep_in_memory=True,)\n",
171
+ "# ds = datasets.concatenate_datasets((ds[\"test\"], ds[\"train\"]))\n",
172
+ "\n",
173
+ "# ds = ds.train_test_split(.05)\n",
174
+ "# train_ds = ds[\"train\"]\n",
175
+ "# test_ds = ds[\"test\"]\n",
176
+ "\n",
177
+ "id2label = {\n",
178
+ " \"0\": \"sky\", # This is given by the rgb value of 00 00 00 for the mask\n",
179
+ " \"1\": \"surface-to-bed\", # This is given by the rgb value of 01 01 01 for the mask\n",
180
+ " \"2\": \"bed-to-bottom\", # This is given by the rgb value of 02 02 02 for the mask\n",
181
+ "}\n",
182
+ "\n",
183
+ "id2label = {int(k): v for k, v in id2label.items()}\n",
184
+ "label2id = {v: k for k, v in id2label.items()}\n",
185
+ "num_labels = len(id2label)\n",
186
+ "\n",
187
+ "len(train_ds), len(test_ds)\n"
188
+ ]
189
+ },
190
+ {
191
+ "cell_type": "code",
192
+ "execution_count": null,
193
  "metadata": {
194
  "colab": {
195
  "base_uri": "https://localhost:8080/",
 
211
  "id": "PAvIJWo1vDb3",
212
  "outputId": "06c909f3-8500-49f6-bca7-b475b1d86885"
213
  },
214
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
215
  "source": [
216
  "# Define the checkpoint from which to load the pre-trained model preprocessor\n",
217
  "checkpoint = \"nvidia/MiT-b0\" # We need to use this processor for resizing the images from the dataset to the size expected by the model; the main problem with this is the output scaling for training and testing, so using the right prepreocessor is important\n",
 
232
  "test_image_processor = SegformerImageProcessor.from_pretrained(checkpoint)\n",
233
  "\n",
234
  "# Create a Segformer model for semantic segmentation using the test configuration and move it to the GPU\n",
235
+ "\n",
236
+ "# The line below initializes a blank model, while the new line initializes the model from the huggingface hub\n",
237
+ "# test_model = SegformerForSemanticSegmentation(test_config).to(\"cuda:0\")\n",
238
+ "test_model = SegformerForSemanticSegmentation.from_pretrained(hf_model_name,id2label = id2label, label2id = label2id).to(\"cuda:0\")\n"
239
  ]
240
  },
241
  {
242
  "cell_type": "code",
243
+ "execution_count": null,
244
  "metadata": {},
245
  "outputs": [],
246
  "source": [
 
258
  },
259
  {
260
  "cell_type": "code",
261
+ "execution_count": null,
262
  "metadata": {
263
  "id": "L-Eojv9VvDb3"
264
  },
 
304
  " # This is input that has gone through the model's forward pass\n",
305
  " logits, labels = eval_pred\n",
306
  " logits_tensor = torch.from_numpy(logits)\n",
307
+ " \n",
308
+ " logits_tensor = logits_tensor.argmax(dim=1)\n",
309
+ " logits_tensor = logits_tensor.unsqueeze(1).to(float)\n",
310
  " # this can lead to very high ram usage for the upscaling\n",
311
  " logits_tensor = nn.functional.interpolate(\n",
312
  " logits_tensor,\n",
313
+ " size = labels.shape[-2:],\n",
314
  " mode=\"bilinear\",\n",
315
  " align_corners=False,\n",
316
  " )\n",
317
+ "\n",
318
+ " # logits_tensor = logits_tensor.argmax(dim=1)\n",
319
+ " logits_tensor = torch.squeeze(logits_tensor,dim = 1)\n",
320
+ "\n",
321
  " # Take the argmax of the logits tensor along dimension 1 to get the predicted labels\n",
 
322
  " # Detach the predicted labels from the computation graph and move them to the CPU \n",
323
  " # (although they are already on the CPU) to save memory and to use numpy features like the metrics module\n",
324
  " pred_labels = logits_tensor.detach().cpu().numpy()\n",
 
347
  "training_args = TrainingArguments(\n",
348
  " output_dir=\"glacformer\", # The output directory for the model predictions and checkpoints\n",
349
  " learning_rate=6e-5, # The initial learning rate for Adam\n",
350
+ " num_train_epochs=6, # Total number of training epochs to perform\n",
351
  " auto_find_batch_size=True, # Whether to automatically find an appropriate batch size\n",
352
  " save_total_limit=3, # Limit the total amount of checkpoints and delete the older checkpoints\n",
353
+ " # eval_accumulation_steps=1, # Number of steps to accumulate gradients before performing a backward/update pass\n",
354
  " evaluation_strategy=\"epoch\", # The evaluation strategy to adopt during training\n",
355
  " save_strategy=\"epoch\", # The checkpoint save strategy to adopt during training\n",
356
  " save_steps=1, # Number of updates steps before two checkpoint saves\n",
357
  " eval_steps=1, # Number of update steps before two evaluations\n",
358
+ " # logging_steps=30, # Number of update steps before logging learning rate and other metrics\n",
359
  " remove_unused_columns=False, # Whether to remove columns not used by the model when using a dataset\n",
360
  " fp16=True, # Whether to use 16-bit float precision instead of 32-bit for saving memory\n",
361
  " tf32=True, # Whether to use tf32 precision instead of 32-bit for saving memory\n",
 
373
  ")"
374
  ]
375
  },
376
+ {
377
+ "cell_type": "code",
378
+ "execution_count": null,
379
+ "metadata": {},
380
+ "outputs": [],
381
+ "source": []
382
+ },
383
  {
384
  "cell_type": "code",
385
  "execution_count": null,
 
399
  "trainer.model.save_pretrained(\"glacformer\")\n",
400
  "\n",
401
  "# Create a repository object for the specified repository on Hugging Face's hub, cloning from the specified source\n",
402
+ "repo = huggingface_hub.Repository(\"glacformer1\", clone_from=hf_model_name)\n",
403
+ "\n",
404
+ "! cp -r glacformer/* glacformer1/\n",
405
  "\n",
406
  "repo.git_pull()\n",
407
  "repo.push_to_hub()"
408
  ]
409
  },
410
+ {
411
+ "cell_type": "code",
412
+ "execution_count": null,
413
+ "metadata": {},
414
+ "outputs": [],
415
+ "source": []
416
+ },
417
  {
418
  "cell_type": "code",
419
  "execution_count": null,
 
475
  "\n",
476
  "glacformer.display(display)"
477
  ]
478
+ },
479
+ {
480
+ "cell_type": "code",
481
+ "execution_count": 13,
482
+ "metadata": {},
483
+ "outputs": [
484
+ {
485
+ "data": {
486
+ "application/vnd.jupyter.widget-view+json": {
487
+ "model_id": "032abac7315f4925b32b07a1dd8e8db3",
488
+ "version_major": 2,
489
+ "version_minor": 0
490
+ },
491
+ "text/plain": [
492
+ "Map: 0%| | 0/1848 [00:00<?, ? examples/s]"
493
+ ]
494
+ },
495
+ "metadata": {},
496
+ "output_type": "display_data"
497
+ },
498
+ {
499
+ "data": {
500
+ "application/vnd.jupyter.widget-view+json": {
501
+ "model_id": "34a7142f36734fff959971c8c335b1c0",
502
+ "version_major": 2,
503
+ "version_minor": 0
504
+ },
505
+ "text/plain": [
506
+ "Pushing dataset shards to the dataset hub: 0%| | 0/1 [00:00<?, ?it/s]"
507
+ ]
508
+ },
509
+ "metadata": {},
510
+ "output_type": "display_data"
511
+ },
512
+ {
513
+ "data": {
514
+ "application/vnd.jupyter.widget-view+json": {
515
+ "model_id": "8be902f9846b426c82d9c2ea0bccf561",
516
+ "version_major": 2,
517
+ "version_minor": 0
518
+ },
519
+ "text/plain": [
520
+ "Creating parquet from Arrow format: 0%| | 0/19 [00:00<?, ?ba/s]"
521
+ ]
522
+ },
523
+ "metadata": {},
524
+ "output_type": "display_data"
525
+ },
526
+ {
527
+ "data": {
528
+ "application/vnd.jupyter.widget-view+json": {
529
+ "model_id": "607b7ae0e6d345cfbec7ebf10c6e38ac",
530
+ "version_major": 2,
531
+ "version_minor": 0
532
+ },
533
+ "text/plain": [
534
+ "Deleting unused files from dataset repository: 0%| | 0/6 [00:00<?, ?it/s]"
535
+ ]
536
+ },
537
+ "metadata": {},
538
+ "output_type": "display_data"
539
+ },
540
+ {
541
+ "data": {
542
+ "application/vnd.jupyter.widget-view+json": {
543
+ "model_id": "e5d424e575d54d5abfb984ef40d16fb7",
544
+ "version_major": 2,
545
+ "version_minor": 0
546
+ },
547
+ "text/plain": [
548
+ "Map: 0%| | 0/5851 [00:00<?, ? examples/s]"
549
+ ]
550
+ },
551
+ "metadata": {},
552
+ "output_type": "display_data"
553
+ },
554
+ {
555
+ "data": {
556
+ "application/vnd.jupyter.widget-view+json": {
557
+ "model_id": "09796dba22ff47d5ba3204dbe0c79ae1",
558
+ "version_major": 2,
559
+ "version_minor": 0
560
+ },
561
+ "text/plain": [
562
+ "Pushing dataset shards to the dataset hub: 0%| | 0/6 [00:00<?, ?it/s]"
563
+ ]
564
+ },
565
+ "metadata": {},
566
+ "output_type": "display_data"
567
+ },
568
+ {
569
+ "data": {
570
+ "application/vnd.jupyter.widget-view+json": {
571
+ "model_id": "c98ca8c9d2264ac6bf8cc2a48e0ed268",
572
+ "version_major": 2,
573
+ "version_minor": 0
574
+ },
575
+ "text/plain": [
576
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
577
+ ]
578
+ },
579
+ "metadata": {},
580
+ "output_type": "display_data"
581
+ },
582
+ {
583
+ "data": {
584
+ "application/vnd.jupyter.widget-view+json": {
585
+ "model_id": "520f907923fe4f8e9d52e274eca484c6",
586
+ "version_major": 2,
587
+ "version_minor": 0
588
+ },
589
+ "text/plain": [
590
+ "Map: 0%| | 0/5850 [00:00<?, ? examples/s]"
591
+ ]
592
+ },
593
+ "metadata": {},
594
+ "output_type": "display_data"
595
+ },
596
+ {
597
+ "data": {
598
+ "application/vnd.jupyter.widget-view+json": {
599
+ "model_id": "304c1699e4e547e49b99b98efb0c92de",
600
+ "version_major": 2,
601
+ "version_minor": 0
602
+ },
603
+ "text/plain": [
604
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
605
+ ]
606
+ },
607
+ "metadata": {},
608
+ "output_type": "display_data"
609
+ },
610
+ {
611
+ "data": {
612
+ "application/vnd.jupyter.widget-view+json": {
613
+ "model_id": "c6586231c59049018016158fb6794f61",
614
+ "version_major": 2,
615
+ "version_minor": 0
616
+ },
617
+ "text/plain": [
618
+ "Map: 0%| | 0/5850 [00:00<?, ? examples/s]"
619
+ ]
620
+ },
621
+ "metadata": {},
622
+ "output_type": "display_data"
623
+ },
624
+ {
625
+ "data": {
626
+ "application/vnd.jupyter.widget-view+json": {
627
+ "model_id": "f07e6acfc0d74df9a33aafb88afa8821",
628
+ "version_major": 2,
629
+ "version_minor": 0
630
+ },
631
+ "text/plain": [
632
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
633
+ ]
634
+ },
635
+ "metadata": {},
636
+ "output_type": "display_data"
637
+ },
638
+ {
639
+ "data": {
640
+ "application/vnd.jupyter.widget-view+json": {
641
+ "model_id": "12fdc097fbf44edd95eccb5c6c953f3a",
642
+ "version_major": 2,
643
+ "version_minor": 0
644
+ },
645
+ "text/plain": [
646
+ "Map: 0%| | 0/5850 [00:00<?, ? examples/s]"
647
+ ]
648
+ },
649
+ "metadata": {},
650
+ "output_type": "display_data"
651
+ },
652
+ {
653
+ "data": {
654
+ "application/vnd.jupyter.widget-view+json": {
655
+ "model_id": "9a20d65772b245f5bff202c4f6ed0141",
656
+ "version_major": 2,
657
+ "version_minor": 0
658
+ },
659
+ "text/plain": [
660
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
661
+ ]
662
+ },
663
+ "metadata": {},
664
+ "output_type": "display_data"
665
+ },
666
+ {
667
+ "data": {
668
+ "application/vnd.jupyter.widget-view+json": {
669
+ "model_id": "2911bb4684894749b9188e1cd9e5d977",
670
+ "version_major": 2,
671
+ "version_minor": 0
672
+ },
673
+ "text/plain": [
674
+ "Map: 0%| | 0/5850 [00:00<?, ? examples/s]"
675
+ ]
676
+ },
677
+ "metadata": {},
678
+ "output_type": "display_data"
679
+ },
680
+ {
681
+ "data": {
682
+ "application/vnd.jupyter.widget-view+json": {
683
+ "model_id": "397fc1c8116f4280a49160432314e4d6",
684
+ "version_major": 2,
685
+ "version_minor": 0
686
+ },
687
+ "text/plain": [
688
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
689
+ ]
690
+ },
691
+ "metadata": {},
692
+ "output_type": "display_data"
693
+ },
694
+ {
695
+ "data": {
696
+ "application/vnd.jupyter.widget-view+json": {
697
+ "model_id": "aa14f6db540e4633a84ef3edfffec6a3",
698
+ "version_major": 2,
699
+ "version_minor": 0
700
+ },
701
+ "text/plain": [
702
+ "Map: 0%| | 0/5850 [00:00<?, ? examples/s]"
703
+ ]
704
+ },
705
+ "metadata": {},
706
+ "output_type": "display_data"
707
+ },
708
+ {
709
+ "data": {
710
+ "application/vnd.jupyter.widget-view+json": {
711
+ "model_id": "7c675a39c05a4288bce0248991e7a568",
712
+ "version_major": 2,
713
+ "version_minor": 0
714
+ },
715
+ "text/plain": [
716
+ "Creating parquet from Arrow format: 0%| | 0/59 [00:00<?, ?ba/s]"
717
+ ]
718
+ },
719
+ "metadata": {},
720
+ "output_type": "display_data"
721
+ },
722
+ {
723
+ "data": {
724
+ "application/vnd.jupyter.widget-view+json": {
725
+ "model_id": "ad22a4f84b56495e86319a5715064cb9",
726
+ "version_major": 2,
727
+ "version_minor": 0
728
+ },
729
+ "text/plain": [
730
+ "Deleting unused files from dataset repository: 0%| | 0/1 [00:00<?, ?it/s]"
731
+ ]
732
+ },
733
+ "metadata": {},
734
+ "output_type": "display_data"
735
+ },
736
+ {
737
+ "data": {
738
+ "application/vnd.jupyter.widget-view+json": {
739
+ "model_id": "ab8fcfdc61534e718e2672baf9b103b4",
740
+ "version_major": 2,
741
+ "version_minor": 0
742
+ },
743
+ "text/plain": [
744
+ "Downloading metadata: 0%| | 0.00/664 [00:00<?, ?B/s]"
745
+ ]
746
+ },
747
+ "metadata": {},
748
+ "output_type": "display_data"
749
+ }
750
+ ],
751
+ "source": [
752
+ "from datasets import DatasetDict\n",
753
+ "\n",
754
+ "dd = DatasetDict({\"test\":test_ds,\"train\":train_ds})\n",
755
+ "\n",
756
+ "dd.push_to_hub(\"glacierscopessegmentation/scopes\")\n",
757
+ "\n"
758
+ ]
759
  }
760
  ],
761
  "metadata": {
 
778
  "name": "python",
779
  "nbconvert_exporter": "python",
780
  "pygments_lexer": "ipython3",
781
+ "version": "3.11.5"
782
  },
783
  "widgets": {
784
  "application/vnd.jupyter.widget-state+json": {