# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import torch
import numpy as np
import cudf
from tqdm import tqdm

df = cudf.read_csv("../dataset/imet/train.csv")
df.shape

df.head()

N_CLASSES = 3474

target = torch.zeros(len(df), N_CLASSES)

target.shape

df.iloc[0, 1].split()

for i in tqdm(range(len(df))):
    for attribute_id in df.iloc[i, 1].split():
        target[i, int(attribute_id)] = 1

target.sum(0)

target.to('cpu').detach().numpy()

np.save("../dataset/imet/labels.npy", target.to('cpu').detach().numpy())



StarcoderdataJupyterScriptsDedupFiltered