# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import ast
import copy

# ### Load CSV data

df = pd.read_csv('via_export_csv.csv',header=0)

# ### Check for labels other than specified ones

for i in range(len(df)):
    if ast.literal_eval(df.iloc[i, -1])['name'] != "class1":
        print(df.iloc[i,0],i)

len(df)

cols = ['filename',
        'file_size',
        'file_attributes',
        'region_count',
        'region_id',
        'region_shape_attributes',
        'region_attributes']

filenames = df['filename'].unique()

len(filenames)

# ## To convert 
# ### *img_name, x, y, width, height, class* 
# ### type CSV to df (OPTIONAL, use only if you have this kind of data)

# +
# Convert to DataFrame
# final_data = []
# for f in filenames:
#     same_files_len = len(df[df['file_name']==f])
    
#     file_name = [f for _ in range(same_files_len)]
#     file_size = [0 for _ in range(same_files_len)]
#     region_count = [same_files_len for _ in range(same_files_len)]
#     region_id = [i for i in range(same_files_len)]
#     file_attributes = ['"{}"' for _ in range(same_files_len)]
    
#     region_attributes = []
#     clses = df[df['file_name']==f]['class_name']
#     for x in range(len(clses)):
#         t = '{"name":"'+clses.iloc[x]+'"}'
#         region_attributes.append(t)
        
#     region_shape_attributes = []
#     attrs = df[df['file_name']==f][['x_max','x_min','y_max','y_min']]
#     for x in range(len(attrs)):
#         t = '{"name":"rect","x":'+str(attrs.iloc[x,1])+',"y":'+str(attrs.iloc[x,3])+',"width":'+str(int(attrs.iloc[x,0])-int(attrs.iloc[x,1]))+',"height":'+str(int(attrs.iloc[x,2])-int(attrs.iloc[x,3]))+'}'
#         region_shape_attributes.append(t)
    
#     for a,b,c,d,e,f,g in zip(file_name,file_size,file_attributes,region_count,region_id,region_shape_attributes,region_attributes):
#         final_data.append([a,b,c,d,e,f,g])

# cols = ['filename',
#         'file_size',
#         'file_attributes',
#         'region_count',
#         'region_id',
#         'region_shape_attributes',
#         'region_attributes']
# new_df = pd.DataFrame(final_data)
# new_df.columns = cols
# new_df.to_csv("final_data_via.csv",index=False)
# -

# ## Separate train val and test

# +
# Get train val test files
# Split by index
train_filenames = filenames[:418] 
val_filenames = filenames[418:434]
test_filenames = filenames[434:]

train_files = pd.DataFrame(columns=cols)
for r in train_filenames:
    train_files = train_files.append(new_df[new_df['filename']==r])

val_files = pd.DataFrame(columns=cols)
for r in val_filenames:
    val_files = val_files.append(new_df[new_df['filename']==r])
val_files = pd.DataFrame(val_files, columns=cols)
# -

# ## Create JSON for training set

train_json = {}
i=0
for f in train_filenames:
    same_files = train_files[train_files['filename']==f]
    t = {}
    t['filename'] = same_files.iloc[0,0]
    t['size'] = same_files.iloc[0,1]
    
    all_regions = []
    for x in range(len(same_files)):
        t1 = {}
        t1['shape_attributes'] = eval(same_files.iloc[x,5])
        t1['region_attributes'] = eval(same_files.iloc[x,-1])
        all_regions.append(t1)
        
    t['regions'] = all_regions
    
    train_json['x'+str(i)] = t
    i+=1

# ## Create JSON for validation set

val_json = {}
i=0
for f in val_filenames:
    same_files = val_files[val_files['filename']==f]
    t = {}
    t['filename'] = same_files.iloc[0,0]
    t['size'] = same_files.iloc[0,1]
    
    all_regions = []
    for x in range(len(same_files)):
        t1 = {}
        t1['shape_attributes'] = eval(same_files.iloc[x,5])
        t1['region_attributes'] = eval(same_files.iloc[x,-1])
        all_regions.append(t1)
        
    t['regions'] = all_regions
    
    val_json['x'+str(i)] = t
    i+=1

# ## Dump train val JSONs

# +
train_json = str(train_json).replace("'",'"')
val_json = str(val_json).replace("'",'"')

with open('train_json.json','w') as f:
    f.write(train_json)
    
with open('val_json.json','w') as f:
    f.write(val_json)
# -

# ## Dump test filenames

test_filenames

# Dump validation filenames
with open('validation.txt','w') as f:
    for i in val_filenames:
        f.write(i+'\n')

StarcoderdataJupyterScriptsDedupFiltered