import helpers
import json
import re

datfilepath = "../github-data/labRepos_CreationHistory.json"
allData = {}

# Check for and read existing data file
allData = helpers.read_existing(datfilepath)

# Read repo info data file (to use as repo list)
dataObj = helpers.read_json("../github-data/labReposInfo.json")

# Populate repo list
repolist = []
print("Getting internal repos ...")
repolist = sorted(dataObj["data"].keys())
print("Repo list complete. Found %d repos." % (len(repolist)))

# Read pretty GraphQL query
query_in = helpers.read_gql("../queries/repo-CreationDate.gql")

# Rest endpoint query
query_commits_in = "/repos/OWNNAME/REPONAME/commits?until=CREATETIME&per_page=100"
query_commits_in2 = "/repos/OWNNAME/REPONAME/commits?per_page=100"

# Retrieve authorization token
authhead = helpers.get_gitauth()

# Iterate through internal repos
print("Gathering data across multiple paginated queries...")
collective = {u'data': {}}
tab = "    "

for repo in repolist:

	# History doesn't change, only update new repos or those that had no previous commits
	if "data" in allData.keys() and repo in allData["data"].keys():
		if allData["data"][repo]["firstCommitAt"]:
			print(tab + "Already recorded data for '%s'" % (repo))
			continue

	pageNum = 1
	print("\n'%s'" % (repo))
	print(tab + "page %d" % (pageNum))

	repoSplit = repo.split("/")

	# Query 1
	print(tab + "Get creation date and default branch")
	print(tab + "Modifying query...")
	newquery = re.sub('OWNNAME', repoSplit[0], query_in)
	newquery = re.sub('REPONAME', repoSplit[1], newquery)
	gitquery = json.dumps({'query': newquery})
	print(tab + "Query ready!")

	# Actual query exchange
	outObj = helpers.query_github(authhead, gitquery)
	if outObj["errors"]:
		print(tab + "Could not complete '%s'" % (repo))
		collective["data"].pop(repo, None)
		continue

	# Update collective data
	collective["data"][repo] = outObj["data"]["repository"]

	# Query 2
	print(tab + "Get pre-GitHub commit timestamps")
	print(tab + "Modifying query...")
	gitquery = re.sub('OWNNAME', repoSplit[0], query_commits_in)
	gitquery = re.sub('REPONAME', repoSplit[1], gitquery)
	gitquery = re.sub('CREATETIME', collective["data"][repo]["createdAt"], gitquery)
	print(tab + "Query ready!")

	# Actual query exchange
	outObj = helpers.query_githubrest(authhead, gitquery)
	if outObj["errors"]:
		print(tab + "Could not get pre-GitHub commits for '%s'" % (repo))
		outObj["data"] = []

	# Update collective data
	collective["data"][repo]["commitTimestamps"] = []
	for commit in outObj["data"]:
		collective["data"][repo]["commitTimestamps"].append(commit["commit"]["committer"]["date"])

	# If no pre-GitHub commits, check the greater commit history
	if len(collective["data"][repo]["commitTimestamps"]) > 0 and collective["data"][repo]["commitTimestamps"][0]:
		collective["data"][repo]["initBeforeGitHubRepo"] = True
	else:
		print(tab + "No pre-GitHub commits found, getting full history")
		collective["data"][repo]["initBeforeGitHubRepo"] = False

		# Query 3
		print(tab + "Modifying query...")
		gitquery = re.sub('OWNNAME', repoSplit[0], query_commits_in2)
		gitquery = re.sub('REPONAME', repoSplit[1], gitquery)
		print(tab + "Query ready!")

		# Actual query exchange
		outObj = helpers.query_githubrest(authhead, gitquery)
		if outObj["errors"]:
			print(tab + "Could not complete '%s'" % (repo))
			collective["data"].pop(repo, None)
			continue

		# Update collective data
		for commit in outObj["data"]:
			collective["data"][repo]["commitTimestamps"].append(commit["commit"]["committer"]["date"])

	# Paginate if needed
	hasNext = ("next" in outObj)
	while hasNext:
		pageNum += 1
		print(tab + "page %d" % (pageNum))

		print(tab + "Modifying query...")
		newquery = gitquery + "&page=" + str(pageNum)
		print(tab + "Query ready!")

		# Actual query exchange
		outObj = helpers.query_githubrest(authhead, newquery)
		if outObj["errors"]:
			print(tab + "Could not complete '%s'" % (repo))
			collective["data"].pop(repo, None)
			continue

		# Update collective data
		for commit in outObj["data"]:
			collective["data"][repo]["commitTimestamps"].append(commit["commit"]["committer"]["date"])

		hasNext = ("next" in outObj)

	# Sort dates
	collective["data"][repo]["commitTimestamps"].sort()
	# Save earliest commit date
	firstdate = None
	if len(collective["data"][repo]["commitTimestamps"]) > 0:
		firstdate = collective["data"][repo]["commitTimestamps"][0]
	collective["data"][repo]["firstCommitAt"] = firstdate

	del collective["data"][repo]["commitTimestamps"]
	print("'%s' Done!" % (repo))

print("\nCollective data gathering complete!")

# Combine new data with existing data
if "data" not in allData.keys():
	allData["data"] = {}
for repo in collective["data"].keys():
	allData["data"][repo] = collective["data"][repo]
allDataString = json.dumps(allData, indent=4, sort_keys=True)

# Write output file
print("\nWriting file '%s'" % (datfilepath))
with open(datfilepath, "w") as fileout:
	fileout.write(allDataString)
print("Wrote file!")

print("\nDone!\n")

[((9, 10, 9, 44), 'helpers.read_existing', 'helpers.read_existing', ({(9, 32, 9, 43): 'datfilepath'}, {}), '(datfilepath)', False, 'import helpers\n'), ((12, 10, 12, 63), 'helpers.read_json', 'helpers.read_json', ({(12, 28, 12, 62): '"""../github-data/labReposInfo.json"""'}, {}), "('../github-data/labReposInfo.json')", False, 'import helpers\n'), ((21, 11, 21, 63), 'helpers.read_gql', 'helpers.read_gql', ({(21, 28, 21, 62): '"""../queries/repo-CreationDate.gql"""'}, {}), "('../queries/repo-CreationDate.gql')", False, 'import helpers\n'), ((28, 11, 28, 32), 'helpers.get_gitauth', 'helpers.get_gitauth', ({}, {}), '()', False, 'import helpers\n'), ((151, 16, 151, 61), 'json.dumps', 'json.dumps', (), '', False, 'import json\n'), ((52, 12, 52, 53), 're.sub', 're.sub', ({(52, 19, 52, 28): '"""OWNNAME"""', (52, 30, 52, 42): 'repoSplit[0]', (52, 44, 52, 52): 'query_in'}, {}), "('OWNNAME', repoSplit[0], query_in)", False, 'import re\n'), ((53, 12, 53, 54), 're.sub', 're.sub', ({(53, 19, 53, 29): '"""REPONAME"""', (53, 31, 53, 43): 'repoSplit[1]', (53, 45, 53, 53): 'newquery'}, {}), "('REPONAME', repoSplit[1], newquery)", False, 'import re\n'), ((54, 12, 54, 43), 'json.dumps', 'json.dumps', ({(54, 23, 54, 42): "{'query': newquery}"}, {}), "({'query': newquery})", False, 'import json\n'), ((58, 10, 58, 50), 'helpers.query_github', 'helpers.query_github', ({(58, 31, 58, 39): 'authhead', (58, 41, 58, 49): 'gitquery'}, {}), '(authhead, gitquery)', False, 'import helpers\n'), ((70, 12, 70, 61), 're.sub', 're.sub', ({(70, 19, 70, 28): '"""OWNNAME"""', (70, 30, 70, 42): 'repoSplit[0]', (70, 44, 70, 60): 'query_commits_in'}, {}), "('OWNNAME', repoSplit[0], query_commits_in)", False, 'import re\n'), ((71, 12, 71, 54), 're.sub', 're.sub', ({(71, 19, 71, 29): '"""REPONAME"""', (71, 31, 71, 43): 'repoSplit[1]', (71, 45, 71, 53): 'gitquery'}, {}), "('REPONAME', repoSplit[1], gitquery)", False, 'import re\n'), ((72, 12, 72, 81), 're.sub', 're.sub', ({(72, 19, 72, 31): '"""CREATETIME"""', (72, 33, 72, 70): "collective['data'][repo]['createdAt']", (72, 72, 72, 80): 'gitquery'}, {}), "('CREATETIME', collective['data'][repo]['createdAt'], gitquery)", False, 'import re\n'), ((76, 10, 76, 54), 'helpers.query_githubrest', 'helpers.query_githubrest', ({(76, 35, 76, 43): 'authhead', (76, 45, 76, 53): 'gitquery'}, {}), '(authhead, gitquery)', False, 'import helpers\n'), ((95, 13, 95, 63), 're.sub', 're.sub', ({(95, 20, 95, 29): '"""OWNNAME"""', (95, 31, 95, 43): 'repoSplit[0]', (95, 45, 95, 62): 'query_commits_in2'}, {}), "('OWNNAME', repoSplit[0], query_commits_in2)", False, 'import re\n'), ((96, 13, 96, 55), 're.sub', 're.sub', ({(96, 20, 96, 30): '"""REPONAME"""', (96, 32, 96, 44): 'repoSplit[1]', (96, 46, 96, 54): 'gitquery'}, {}), "('REPONAME', repoSplit[1], gitquery)", False, 'import re\n'), ((100, 11, 100, 55), 'helpers.query_githubrest', 'helpers.query_githubrest', ({(100, 36, 100, 44): 'authhead', (100, 46, 100, 54): 'gitquery'}, {}), '(authhead, gitquery)', False, 'import helpers\n'), ((121, 11, 121, 55), 'helpers.query_githubrest', 'helpers.query_githubrest', ({(121, 36, 121, 44): 'authhead', (121, 46, 121, 54): 'newquery'}, {}), '(authhead, newquery)', False, 'import helpers\n')]