# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
from scipy.sparse import coo_matrix, csr_matrix

src_dir = '/graph-learning/data-raw/papers100M-bin/'
output_dir = '/graph-learning/samgraph/papers100M/'

# +
file0 = np.load(src_dir + 'raw/data.npz')
file1 = np.load(src_dir + 'raw/node-label.npz')

features = file0['node_feat']
edge_index = file0['edge_index']
label = file1['node_label']

src = edge_index[0]
dst = edge_index[1]
data = np.zeros(src.shape)

train_idx = pd.read_csv(src_dir + 'split/time/train.csv.gz', compression='gzip', header = None).values.T[0]
valid_idx = pd.read_csv(src_dir + 'split/time/valid.csv.gz', compression='gzip', header = None).values.T[0]
test_idx = pd.read_csv(src_dir + 'split/time/test.csv.gz', compression='gzip', header = None).values.T[0]

# +
# convert to csc graph swap src and dst
coo = coo_matrix((data, (dst, src)), shape=(feature.shape[0], feature.shape[0]),dtype=np.uint32)
csr = coo.tocsr()

indptr = csr.indptr
indices = csr.indices

indptr.astype('uint32').tofile(output_dir + 'indptr.bin')
indices.astype('uint32').tofile(output_dir + 'indices.bin')
# -

src.sha

train_idx.astype('uint32').tofile(output_dir + 'train_set.bin')
valid_idx.astype('uint32').tofile(output_dir + 'valid_set.bin')
test_idx.astype('uint32').tofile(output_dir + 'test_set.bin')

features.astype('float32').tofile(output_dir + 'feat.bin')
label.astype('uint64').tofile(output_dir + 'label.bin')

StarcoderdataJupyterScriptsDedupFiltered