<reponame>ChiaraBi/asclepias-broker<gh_stars>0
# -*- coding: utf-8 -*-
#
# Copyright (C) 2018 CERN.
#
# <NAME> is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.

"""Versioning metadata harvester."""

from datetime import datetime
from typing import List

from ..events.api import EventAPI

import re
import time
import requests
from flask import current_app
from sqlalchemy.orm import relationship
from ..utils import chunks, GitHubAPIException, GithubUtility
from .base import MetadataHarvester


class GitHubClient:
    """GitHub client."""

    base_url = 'https://api.github.com/'
    _api_token = None
    
    # For some reason a different repo name when using id vs name in the Github API 

    def get_repo_metadata_from_id(self, id) -> dict:
        url = self.base_url + "repositories/" + id
        return self.query_api(url)
    
    def get_repo_metadata_from_name(self, user, repo) -> dict:
        url = self.base_url + "repos/" + user + "/" + repo
        return self.query_api(url)

    def get_repo_release_from_name(self, user, repo, tag) -> dict:
        repo_meta = self.get_repo_metadata_from_name(user, repo)
        releases_url = repo_meta['releases_url'].replace('{/id}','')
        return self.get_repo_release(releases_url, repo_meta, tag)

    def get_repo_release_from_id(self, id, tag) -> dict:
        repo_meta = self.get_repo_metadata_from_id(id)
        releases_url = repo_meta['releases_url'].replace('{/id}','')
        return self.get_repo_release(releases_url, repo_meta, tag)

    def get_repo_release(self, url, repo_meta, tag) -> dict:
        releases = self.query_api(url)
        for release in releases:
            if release['tag_name'] == tag:
                release['repo_id'] = repo_meta['id']
                release['repo_name'] = repo_meta['full_name']
                return release

    def get_api_token(self):
        """."""
        if self._api_token is None:
            self._api_token = current_app.config.get(
                'ASCLEPIAS_HARVESTER_GITHUB_API_TOKEN')
        elif callable(self._api_token):
            self._api_token = self._api_token()
        return self._api_token

    def query_api(self, url):
        try:
            headers = {'X-GitHub-Media-Type':'application/vnd.github.v3.raw+json'}
            token = self.get_api_token()
            if token is not None:
                headers['Authorization'] = 'token ' + token
            res = requests.get(url, headers=headers)
            if res.ok:
                return res.json()
            else:
                res.raise_for_status()
        except Exception as exc:
            raise GitHubAPIException(exc)

class GitHubHarvester(MetadataHarvester):
    """Metadata harvester for Github"""

    def __init__(self, *, provider_name: str = None):
        """."""
        self.provider_name = provider_name or "Github harvester"

    def can_harvest(self, identifier: str, scheme: str,
                    providers: List[str] = None) -> bool:
        """."""
        is_provider = False
        if providers:
            is_provider = self.provider_name in providers

        return (self._is_valid_github_url(scheme, identifier) or\
            self._is_github_repo_id(scheme, identifier) or\
            self._is_github_repo_id(scheme, identifier) or\
            self._is_github_release_id(scheme, identifier) )\
            and not is_provider

    def harvest(self, identifier: str, scheme: str,
                providers: List[str] = None):
        """."""
        try:
            #Added a sleeper timer to slow down the harvesters since during large ingestions they cause HTTP 403 errors when too many queries are sent
            time.sleep(1)
            providers = set(providers) if providers else set()
            providers.add(self.provider_name)
            payloads = []
            if self._is_valid_github_url(scheme, identifier):
                parsed_info = GithubUtility.parse_url_info(identifier)
            elif self._is_github_repo_id(scheme, identifier):
                parsed_info = dict(id=identifier, identifier=identifier, scheme='github')
            elif self._is_github_release_id(scheme, identifier):
                id, _, release_id = identifier.split('/')
                parsed_info = dict(id=id, sub_type='release_id', release_id=release_id, identifier=identifier, scheme='github')
           
            child = None
            if 'sub_type' in parsed_info.keys():
                for payload in add_version_identifiers(parsed_info, providers):
                    payloads.append(payload)
                child = {
                    "ID": parsed_info['identifier'],
                    "IDScheme": parsed_info['scheme']
                }
            
            for payload in add_parent_identifiers(parsed_info, providers, child):
                payloads.append(payload)

            for event_chunk in chunks(payloads, 100):
                try:
                    EventAPI.handle_event(list(event_chunk), no_index=True, eager=True)
                except ValueError:
                    current_app.logger.exception(
                        'Error while processing github harvesting event.')
        except Exception as exc:
            raise GitHubAPIException(exc)

    def _is_github_repo_id(self,  scheme: str, identifier: str) -> bool:
        if scheme.lower() == 'github' and re.match('^\d+$',identifier):
            return True
        return False

    def _is_github_release_id(self,  scheme: str, identifier: str) -> bool:
        if scheme.lower() == 'github' and re.match('^\d+/releases/\d+$',identifier):
            return True
        return False
        
    def _is_valid_github_url(self,  scheme: str, identifier: str) -> bool:
        if scheme.lower() == 'url' and 'github.com' in identifier.lower():
            try:
                GithubUtility.parse_url_info(identifier)
            except:
                return False
            return True
        else:
            return False
    
def add_parent_identifiers(parsed_info, providers, child = None) -> List[dict]:
    client = GitHubClient()
    add_old_name = False
    if 'user' in parsed_info.keys():
        parent_meta = client.get_repo_metadata_from_name(user=parsed_info['user'], repo=parsed_info['repo'])
        if parent_meta['full_name'] != parsed_info['user'] + '/' + parsed_info['repo']:
            add_old_name = True
    else:
        parent_meta = client.get_repo_metadata_from_id(id=parsed_info['id'])
    id = parent_meta['id']
    full_name = parent_meta['full_name']

    id_identifier = {
            "ID": str(id),
            "IDScheme": 'github'
    }
    name_identifier = {
            "ID": 'https://github.com/' + full_name,
            "IDScheme": 'url'
    }
    payloads = []
    payloads.append(create_relationship_event(src=id_identifier, target=name_identifier, relationship='IsIdenticalTo', providers=providers))
    if child != None:
        payloads.append(create_relationship_event(src=id_identifier, target=child, relationship='HasVersion', providers=providers))
    if add_old_name:
        old_name_identifier = {
            "ID": 'github.com/' + parsed_info['user'] + '/' + parsed_info['repo'],
            "IDScheme": 'url'
        }
        payloads.append(create_relationship_event(src=name_identifier, target=old_name_identifier, relationship='IsIdenticalTo', providers=providers))

    return payloads

def add_version_identifiers(parsed_info, providers)  -> List[dict]:
    client = GitHubClient()
    add_old_name = False
    if 'user' in parsed_info.keys():
        version_meta = client.get_repo_release_from_name(user=parsed_info['user'], repo=parsed_info['repo'], tag=parsed_info['tag'])
        if version_meta is not None and version_meta['html_url'] != parsed_info['identifier']:
            add_old_name = True
    else:
        version_meta = client.get_repo_release_from_id(id=parsed_info['id'], tag = parsed_info['release_id'])

    payloads = []

    if version_meta is not None:
        id_identifier = {
                "ID": str(version_meta['repo_id']) + '/releases/' + str(version_meta['id']),
                "IDScheme": 'github'
        }
        name_identifier = {
                "ID": version_meta['html_url'],
                "IDScheme": 'url'
        }
    
        payloads.append(create_relationship_event(src=id_identifier, target=name_identifier, relationship='IsIdenticalTo', providers=providers))
        if add_old_name:
            old_name_identifier = {
                "ID": parsed_info['identifier'],
                "IDScheme": 'url'
            }
            payloads.append(create_relationship_event(src=name_identifier, target=old_name_identifier, relationship='IsIdenticalTo', providers=providers))

    return payloads

def create_relationship_event(src, target, relationship, providers) -> dict:
    link_publication_date = datetime.now().isoformat()
    link_providers = providers or ['unknown']
    link_providers = [{'Name': provider} for provider in link_providers]
    payload = {
        'RelationshipType': {
            'Name': 'IsRelatedTo',
            'SubTypeSchema': 'DataCite',
            'SubType': relationship
        },
        'Target': {
            'Identifier': target,
            'Type': {'Name': 'unknown'}
        },
        'LinkProvider': link_providers,
        'Source': {
            'Identifier': src,
            'Type': {'Name': 'unknown'}
        },
        'LinkPublicationDate': link_publication_date,
    }

    return payload