scraper/code_gov/models.py
def from_github3(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from github3 Repository object
        """
        if not isinstance(repository, github3.repos.repo._Repository):
            raise TypeError('Repository must be a github3 Repository object')

        logger.info('Processing: %s', repository.full_name)

        project = klass()

        logger.debug('GitHub3: repository=%s', repository)

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.git_url
        project['description'] = repository.description

        try:
            repo_license = repository.license()
        except github3.exceptions.NotFoundError:
            logger.debug('no license found for repo=%s', repository)
            repo_license = None

        if repo_license:
            license = repo_license.license
            if license:
                logger.debug('license spdx=%s; url=%s', license.spdx_id, license.url)
                if license.url is None:
                    project['permissions']['licenses'] = [{"name": license.spdx_id}]
                else:
                    project['permissions']['licenses'] = [{"URL": license.url, "name": license.spdx_id}]
            else:
                project['permissions']['licenses'] = None

        public_server = repository.html_url.startswith('https://github.com')
        if not repository.private and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['github']
        old_accept = repository.session.headers['Accept']
        repository.session.headers['Accept'] = 'application/vnd.github.mercy-preview+json'
        topics = repository._get(repository.url + '/topics').json()
        project['tags'].extend(topics.get('names', []))
        repository.session.headers['Accept'] = old_accept

        # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0
        owner_url = repository.owner.url
        owner_api_response = repository._get(owner_url)
        organization = repository._json(owner_api_response, 200)
        project['contact']['email'] = organization['email']
        project['contact']['URL'] = organization['html_url']

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = organization['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.html_url

        project['downloadURL'] = repository.downloads_url

        project['languages'] = [l for l, _ in repository.languages()]

        # project['partners'] = []

        # project['relatedCode'] = []

        # project['reusedCode'] = []

        # date: [object] A date object describing the release.
        #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
        #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
        try:
            created_at = repository.created_at.date()
        except AttributeError:
            created_at = date_parse(repository.created_at).date()
        try:
            updated_at = repository.updated_at.date()
        except AttributeError:
            updated_at = date_parse(repository.updated_at).date()

        project['date'] = {
            'created': created_at.isoformat(),
            'lastModified': updated_at.isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
def from_github3(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from github3 Repository object
        """
        if not isinstance(repository, github3.repos.repo._Repository):
            raise TypeError('Repository must be a github3 Repository object')

        logger.info('Processing: %s', repository.full_name)

        project = klass()

        logger.debug('GitHub3: repository=%s', repository)

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.git_url
        project['description'] = repository.description

        try:
            repo_license = repository.license()
        except github3.exceptions.NotFoundError:
            logger.debug('no license found for repo=%s', repository)
            repo_license = None

        if repo_license:
            license = repo_license.license
            if license:
                logger.debug('license spdx=%s; url=%s', license.spdx_id, license.url)
                if license.url is None:
                    project['permissions']['licenses'] = [{"name": license.spdx_id}]
                else:
                    project['permissions']['licenses'] = [{"URL": license.url, "name": license.spdx_id}]
            else:
                project['permissions']['licenses'] = None

        public_server = repository.html_url.startswith('https://github.com')
        if not repository.private and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['github']
        old_accept = repository.session.headers['Accept']
        repository.session.headers['Accept'] = 'application/vnd.github.mercy-preview+json'
        topics = repository._get(repository.url + '/topics').json()
        project['tags'].extend(topics.get('names', []))
        repository.session.headers['Accept'] = old_accept

        # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0
        owner_url = repository.owner.url
        owner_api_response = repository._get(owner_url)
        organization = repository._json(owner_api_response, 200)
        project['contact']['email'] = organization['email']
        project['contact']['URL'] = organization['html_url']

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = organization['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.html_url

        project['downloadURL'] = repository.downloads_url

        project['languages'] = [l for l, _ in repository.languages()]

        # project['partners'] = []

        # project['relatedCode'] = []

        # project['reusedCode'] = []

        # date: [object] A date object describing the release.
        #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
        #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
        #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
        try:
            created_at = repository.created_at.date()
        except AttributeError:
            created_at = date_parse(repository.created_at).date()
        try:
            updated_at = repository.updated_at.date()
        except AttributeError:
            updated_at = date_parse(repository.updated_at).date()

        project['date'] = {
            'created': created_at.isoformat(),
            'lastModified': updated_at.isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
def from_github3 ( klass , repository , labor_hours = True ) : if not isinstance ( repository , github3 . repos . repo . _Repository ) : raise TypeError ( 'Repository must be a github3 Repository object' ) logger . info ( 'Processing: %s' , repository . full_name ) project = klass ( ) logger . debug ( 'GitHub3: repository=%s' , repository ) # -- REQUIRED FIELDS -- project [ 'name' ] = repository . name project [ 'repositoryURL' ] = repository . git_url project [ 'description' ] = repository . description try : repo_license = repository . license ( ) except github3 . exceptions . NotFoundError : logger . debug ( 'no license found for repo=%s' , repository ) repo_license = None if repo_license : license = repo_license . license if license : logger . debug ( 'license spdx=%s; url=%s' , license . spdx_id , license . url ) if license . url is None : project [ 'permissions' ] [ 'licenses' ] = [ { "name" : license . spdx_id } ] else : project [ 'permissions' ] [ 'licenses' ] = [ { "URL" : license . url , "name" : license . spdx_id } ] else : project [ 'permissions' ] [ 'licenses' ] = None public_server = repository . html_url . startswith ( 'https://github.com' ) if not repository . private and public_server : project [ 'permissions' ] [ 'usageType' ] = 'openSource' elif date_parse ( repository . created_at ) < POLICY_START_DATE : project [ 'permissions' ] [ 'usageType' ] = 'exemptByPolicyDate' if labor_hours : project [ 'laborHours' ] = labor_hours_from_url ( project [ 'repositoryURL' ] ) else : project [ 'laborHours' ] = 0 project [ 'tags' ] = [ 'github' ] old_accept = repository . session . headers [ 'Accept' ] repository . session . headers [ 'Accept' ] = 'application/vnd.github.mercy-preview+json' topics = repository . _get ( repository . url + '/topics' ) . json ( ) project [ 'tags' ] . extend ( topics . get ( 'names' , [ ] ) ) repository . session . headers [ 'Accept' ] = old_accept # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0 owner_url = repository . owner . url owner_api_response = repository . _get ( owner_url ) organization = repository . _json ( owner_api_response , 200 ) project [ 'contact' ] [ 'email' ] = organization [ 'email' ] project [ 'contact' ] [ 'URL' ] = organization [ 'html_url' ] # -- OPTIONAL FIELDS -- # project['version'] = '' project [ 'organization' ] = organization [ 'name' ] # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project [ 'status' ] = 'Development' project [ 'vcs' ] = 'git' project [ 'homepageURL' ] = repository . html_url project [ 'downloadURL' ] = repository . downloads_url project [ 'languages' ] = [ l for l , _ in repository . languages ( ) ] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] # date: [object] A date object describing the release. #   created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format. #   lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format. #   metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format. try : created_at = repository . created_at . date ( ) except AttributeError : created_at = date_parse ( repository . created_at ) . date ( ) try : updated_at = repository . updated_at . date ( ) except AttributeError : updated_at = date_parse ( repository . updated_at ) . date ( ) project [ 'date' ] = { 'created' : created_at . isoformat ( ) , 'lastModified' : updated_at . isoformat ( ) , 'metadataLastUpdated' : '' , } _prune_dict_null_str ( project ) return project
Create CodeGovProject object from github3 Repository object
Create CodeGovProject object from github3 Repository object