# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Programming Language Classification IBM Code Pattern - Build Data set

# Download pygithub, import libraries, and authenticate with your github credentials 

# !pip install pygithub

# +
import github
import base64
import numpy as np
from github import GithubException

#Create a GitHub personal access token at https://github.com/settings/tokens
login = 'YOURLOGIN'
token = '<PASSWORD>'

g = github.Github(login, token)
repos = []
org = g.get_organization('IBM')

#We'll look at code only with the following extensions. You can try this with other
#programming languages and for binary classification (2, not 8, classes).
targetlangs = ['.go','.java','.js','.m','.py','.sh','.swift','.xml']

#Get the list of repos at https://github.com/IBM.
for repo in g.get_organization('IBM').get_repos():
    repos.append(repo.name)
# -

# List repos

repos


# ## Download Directory
#
# This function recursively crawls through each file/directory starting at the given repo and path. 
# It is similar in functionality to the "find" linux command
# Calls to GitHub API are wrapped in Try/Except blocks to ignore errors, there will be plenty of code to grab for our dataset

def download_directory(repository, path):
    global dataset
    try:
        contents = repository.get_contents(path)
        for content in contents:
            if content.type == 'dir':
                download_directory(repository, content.path)
            else:
                if content.content:
                    if len(str(content.name).split(".")) == 2:
                        if any(substring == ("." + str(content.name).split(".")[1]) for substring in targetlangs):                           
                            try:
                                dataset.append([base64.b64decode(content.content),str(content.name).split(".")[1]])
                            except (GithubException, IOError) as exc:
                                print 'Error processing %s: %s', content.path, exc
    except (GithubException, IOError) as exc:
        print "error in dir " 



# loop through each repo and grab the contents that match one of our target languages
# they are being places in a multidimensional list with 
# dataset[i][0] being the code text and dataset[i][1] being the code's file extenstion
#
#
# there are two loops because I would ofter receive a 403 RateLimitExceededException from github
#
# if this happens just pick up in the next repo, again plenty of code

# +
dataset = []

for i in range(len(repos[1:])):
    if i % 2 == 0:
        print "getting repo " + str(i) + " of " + str(len(repos[1:]))
    repo = org.get_repo(str(repos[i]))
    download_directory(repo, "")
# -

for i in range(len(repos[44:])):
    if i % 2 == 0:
        print "getting repo " + str(i+44) + " of " + str(len(repos[1:]))
    repo = org.get_repo(str(repos[i]))
    download_directory(repo, "")

# ### Split the code into a training and a testing set.
# You can see the length of the dataset and its contents.
# Pickle the training and testing sets for the next notebook (mine are included in the github)

import random
data = dataset
random.shuffle(data)
train_data = data[:int((len(data)+1)*.80)] #Remaining 80% to training set
test_data = data[int(len(data)*.80+1):] #Splits 20% data to test set

len(dataset)

print dataset[0][1]

np.savez_compressed("githubtrainingdatacompressed.npz",np.array(train_data))
np.savez_compressed("githubtestdatacompressed.npz",np.array(test_data))

StarcoderdataJupyterScriptsDedupFiltered