#SPDX-License-Identifier: MIT
import logging, os, sys, time, requests, json
from datetime import datetime
from multiprocessing import Process, Queue
import pandas as pd
import sqlalchemy as s
from workers.worker_base import Worker

# NOTE: This worker primarily inserts rows into the REPO_INFO table, which serves the primary purposes of 
# 1. Displaying discrete metadata like "number of forks" and how they change over time 
# 2. Validating other workers, like those related to pull requests, issues, and commits. Our totals should be at or very near the totals in the repo_info table.

# This table also updates the REPO table in 2 cases: 
# 1. Recognizing when a repository is a forked repository by updating the "forked_from" field and 
# 2. Recognizing when a repository is archived, and recording the data we observed the change in status. 

class RepoInfoWorker(Worker):
    def __init__(self, config={}):

        worker_type = "repo_info_worker"
        
        # Define what this worker can be given and know how to interpret
        given = [['github_url']]
        models = ['repo_info']

        # Define the tables needed to insert, update, or delete on
        data_tables = ['repo_info', 'repo']
        operations_tables = ['worker_history', 'worker_job']

        # Run the general worker initialization
        super().__init__(worker_type, config, given, models, data_tables, operations_tables)

        # Define data collection info
        self.tool_source = 'Repo Info Worker'
        self.tool_version = '1.0.0'
        self.data_source = 'GitHub API'

    def repo_info_model(self, task, repo_id):

        github_url = task['given']['github_url']

        self.logger.info("Beginning filling the repo_info model for repo: " + github_url + "\n")

        owner, repo = self.get_owner_repo(github_url)

        url = 'https://api.github.com/graphql'

        query = """
            {
                repository(owner:"%s", name:"%s"){
                    updatedAt
                    hasIssuesEnabled
                    issues(states:OPEN) {
                        totalCount
                    }
                    hasWikiEnabled
                    forkCount
                    defaultBranchRef {
                        name
                    }
                    watchers {
                        totalCount
                    }
                    id
                    licenseInfo {
                        name
                        url
                    }
                    stargazers {
                        totalCount
                    }
                    codeOfConduct {
                        name
                        url
                    }
                    issue_count: issues {
                        totalCount
                    }
                    issues_closed: issues(states:CLOSED) {
                        totalCount
                    }
                    pr_count: pullRequests {
                        totalCount
                    }
                    pr_open: pullRequests(states: OPEN) {
                        totalCount
                    }
                    pr_closed: pullRequests(states: CLOSED) {
                        totalCount
                    }
                    pr_merged: pullRequests(states: MERGED) {
                        totalCount
                    }
                    ref(qualifiedName: "master") {
                        target {
                            ... on Commit {
                                history(first: 0){
                                    totalCount
                                }
                            }
                        }
                    }
                }
            }
        """ % (owner, repo)

        # Hit the graphql endpoint and retry 3 times in case of failure
        num_attempts = 0
        success = False
        data = None
        while num_attempts < 3:
            self.logger.info("Hitting endpoint: {} ...\n".format(url))
            r = requests.post(url, json={'query': query}, headers=self.headers)
            self.update_gh_rate_limit(r)

            try:
                data = r.json()
            except:
                data = json.loads(json.dumps(r.text))

            if 'errors' in data:
                self.logger.info("Error!: {}".format(data['errors']))
                if data['errors'][0]['message'] == 'API rate limit exceeded':
                    self.update_gh_rate_limit(r)
                    continue

            if 'data' in data:
                success = True
                data = data['data']['repository']
                break
            else:
                self.logger.info("Request returned a non-data dict: {}\n".format(data))
                if data['message'] == 'Not Found':
                    self.logger.info("Github repo was not found or does not exist for endpoint: {}\n".format(url))
                    break
                if data['message'] == 'You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.':
                    self.update_gh_rate_limit(r, temporarily_disable=True)
                    continue
                if data['message'] == 'Bad credentials':
                    self.update_gh_rate_limit(r, bad_credentials=True)
                    continue
            num_attempts += 1
        if not success:
            self.logger.error('Cannot hit endpoint after 3 attempts. \"Completing\" task.\n')
            self.register_task_completion(self.task, repo_id, 'repo_info')
            return

        # Just checking that the data is accessible (would not be if repo no longer exists)
        try:
            data['updatedAt']
        except Exception as e:
            self.logger.error('Cannot access repo_info data: {}\nError: {}. \"Completing\" task.'.format(data, e))
            self.register_task_completion(self.task, repo_id, 'repo_info')
            return

        # Get committers count info that requires seperate endpoint
        committers_count = self.query_committers_count(owner, repo)

        # Put all data together in format of the table
        self.logger.info(f'Inserting repo info for repo with id:{repo_id}, owner:{owner}, name:{repo}\n')
        rep_inf = {
            'repo_id': repo_id,
            'last_updated': data['updatedAt'] if 'updatedAt' in data else None,
            'issues_enabled': data['hasIssuesEnabled'] if 'hasIssuesEnabled' in data else None,
            'open_issues': data['issues']['totalCount'] if data['issues'] else None,
            'pull_requests_enabled': None,
            'wiki_enabled': data['hasWikiEnabled'] if 'hasWikiEnabled' in data else None,
            'pages_enabled': None,
            'fork_count': data['forkCount'] if 'forkCount' in data else None,
            'default_branch': data['defaultBranchRef']['name'] if data['defaultBranchRef'] else None,
            'watchers_count': data['watchers']['totalCount'] if data['watchers'] else None,
            'UUID': None,
            'license': data['licenseInfo']['name'] if data['licenseInfo'] else None,
            'stars_count': data['stargazers']['totalCount'] if data['stargazers'] else None,
            'committers_count': committers_count,
            'issue_contributors_count': None,
            'changelog_file': None,
            'contributing_file': None,
            'license_file': data['licenseInfo']['url'] if data['licenseInfo'] else None,
            'code_of_conduct_file': data['codeOfConduct']['url'] if data['codeOfConduct'] else None,
            'security_issue_file': None,
            'security_audit_file': None,
            'status': None,
            'keywords': None,
            'commit_count': data['ref']['target']['history']['totalCount'] if data['ref'] else None,
            'issues_count': data['issue_count']['totalCount'] if data['issue_count'] else None,
            'issues_closed': data['issues_closed']['totalCount'] if data['issues_closed'] else None,
            'pull_request_count': data['pr_count']['totalCount'] if data['pr_count'] else None,
            'pull_requests_open': data['pr_open']['totalCount'] if data['pr_open'] else None,
            'pull_requests_closed': data['pr_closed']['totalCount'] if data['pr_closed'] else None,
            'pull_requests_merged': data['pr_merged']['totalCount'] if data['pr_merged'] else None,
            'tool_source': self.tool_source,
            'tool_version': self.tool_version,
            'data_source': self.data_source
        }

        result = self.db.execute(self.repo_info_table.insert().values(rep_inf))
        self.logger.info(f"Primary Key inserted into repo_info table: {result.inserted_primary_key}\n")
        self.results_counter += 1

        # Note that the addition of information about where a repository may be forked from, and whether a repository is archived, updates the `repo` table, not the `repo_info` table.
        forked = self.is_forked(owner, repo)
        archived = self.is_archived(owner, repo)
        archived_date_collected = None
        if archived is not False:
            archived_date_collected = archived
            archived = 1
        else:
            archived = 0

        rep_additional_data = {
            'forked_from': forked,
            'repo_archived': archived,
            'repo_archived_date_collected': archived_date_collected
        }
        result = self.db.execute(self.repo_table.update().where(
            self.repo_table.c.repo_id==repo_id).values(rep_additional_data))

        self.logger.info(f"Inserted info for {owner}/{repo}\n")

        # Register this task as completed
        self.register_task_completion(self.task, repo_id, "repo_info")

    def query_committers_count(self, owner, repo):
        self.logger.info('Querying committers count\n')
        url = f'https://api.github.com/repos/{owner}/{repo}/contributors?per_page=100'
        committers = 0

        try:
            while True:
                r = requests.get(url, headers=self.headers)
                self.update_gh_rate_limit(r)
                committers += len(r.json())

                if 'next' not in r.links:
                    break
                else:
                    url = r.links['next']['url']
        except Exception:
            self.logger.exception('An error occured while querying contributor count\n')

        return committers

    def is_forked(self, owner, repo): #/repos/:owner/:repo parent
        self.logger.info('Querying parent info to verify if the repo is forked\n')
        url = f'https://api.github.com/repos/{owner}/{repo}'

        r = requests.get(url, headers=self.headers)
        self.update_gh_rate_limit(r)

        data = self.get_repo_data(url, r)

        if 'fork' in data:
            if 'parent' in data:
                return data['parent']['full_name']
            return 'Parent not available'

        return False

    def is_archived(self, owner, repo):
        self.logger.info('Querying committers count\n')
        url = f'https://api.github.com/repos/{owner}/{repo}'

        r = requests.get(url, headers=self.headers)
        self.update_gh_rate_limit(r)

        data = self.get_repo_data(url, r)

        if 'archived' in data:
            if data['archived']:
                if 'updated_at' in data:
                    return data['updated_at']
                return 'Date not available'
            return False

        return False

    def get_repo_data(self, url, response):
        success = False
        try:
            data = response.json()
        except:
            data = json.loads(json.dumps(response.text))

        if 'errors' in data:
            self.logger.info("Error!: {}".format(data['errors']))
            if data['errors'][0]['message'] == 'API rate limit exceeded':
                self.update_gh_rate_limit(response)

        if 'id' in data:
            success = True
        else:
            self.logger.info("Request returned a non-data dict: {}\n".format(data))
            if data['message'] == 'Not Found':
                self.logger.info("Github repo was not found or does not exist for endpoint: {}\n".format(url))
            if data['message'] == 'You have triggered an abuse detection mechanism. Please wait a few minutes before you try again.':
                self.update_gh_rate_limit(r, temporarily_disable=True)
            if data['message'] == 'Bad credentials':
                self.update_gh_rate_limit(r, bad_credentials=True)
        if not success:
            self.register_task_failure(self.task, repo_id, "Failed to hit endpoint: {}".format(url))

        return data

[(248, 'requests.get', 'requests.get', (['url'], {'headers': 'self.headers'}), False, 'import logging, os, sys, time, requests, json\n'), (264, 'requests.get', 'requests.get', (['url'], {'headers': 'self.headers'}), False, 'import logging, os, sys, time, requests, json\n'), (113, 'requests.post', 'requests.post', (['url'], {'json': "{'query': query}", 'headers': 'self.headers'}), False, 'import logging, os, sys, time, requests, json\n'), (231, 'requests.get', 'requests.get', (['url'], {'headers': 'self.headers'}), False, 'import logging, os, sys, time, requests, json\n'), (283, 'json.dumps', 'json.dumps', (['response.text'], {}), False, 'import logging, os, sys, time, requests, json\n'), (119, 'json.dumps', 'json.dumps', (['r.text'], {}), False, 'import logging, os, sys, time, requests, json\n')]