# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd

dep_df= pd.read_csv("/Users/yangdan/Desktop/insight/deps_out.csv",names=['id','package1','version',])

dep_df

proj_df= pd.read_csv("/Users/yangdan/Desktop/insight/packages_out.csv")

proj_df[9000:]

repo_df= pd.read_csv("/Users/yangdan/Desktop/insight/repos_out.csv")

repo_df

# ## Repo

df= pd.read_csv("/Users/yangdan/Downloads/repositories.csv",skiprows= 1,usecols=[0,8,17,19],names=['id','url','issues','watchers'])

url_cleaned=df["url"].replace("\'",'').replace("\"",'')

df["url"]= url_cleaned

df.iloc[:10,:]

df.to_csv("/Users/yangdan/Desktop/insight/repository_cleaned.csv")

df

# ## Depdency

dep_df= pd.read_csv("/Users/yangdan/Downloads/dependencies1.csv",skiprows=1,usecols=[0,3,4,6,7,8,9,10,11],names=['id','package1','version','name','platform','kind','optional','requirement','package2'])

dep_df.columns=['id','package1','version','name','platform','kind','optional','requirement','package2']

dep_df=dep_df[~dep_df['optional'].isin([True])]

dep_df

dep_sub=dep_df[['id','name','kind','optional','requirement']]

dep_sub.iloc[:50000000,[0,1,2,4]]

dep_sub.iloc[:50000000,[0,1,2,4]].to_csv("/Users/yangdan/Desktop/insight/dependency_cleaned1.csv")

dep_sub.iloc[50000000:100000000,[0,1,2,4]].to_csv("/Users/yangdan/Desktop/insight/dependency_cleaned2.csv")

dep_sub.iloc[100000000:,[0,1,2,4]].to_csv("/Users/yangdan/Desktop/insight/dependency_cleaned3.csv")

# ## Relationship(for dependency)

dep_pltform=dep_df[['id','platform']]

dep_pltform





has_dep = dep_df[['id','package1','version']]

has_dep





belongs_to = dep_df[['id','package2']]

belongs_to = belongs_to.dropna()

belongs_to

# ## Has_manager(for package)

package_df= pd.read_csv("/Users/yangdan/Desktop/insight/project_cleaned.csv",usecols=[0,1],names=['id','platform'])

package_df

package_df.to_csv("/Users/yangdan/Desktop/insight/has_manager.csv")

StarcoderdataJupyterScriptsDedupFiltered