scraper/github/queryManager.py
def queryGitHub(self, gitquery, gitvars={}, verbosity=0, paginate=False, cursorVar=None, keysToList=[], rest=False, requestCount=0, pageNum=0):
        """Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.

        """
        requestCount += 1
        if pageNum < 0:  # no negative page numbers
            pageNum = 0
        pageNum += 1

        if paginate:
            _vPrint((verbosity >= 0), "Page %d" % (pageNum))
        _vPrint((verbosity >= 0), "Sending %s query..." % ("REST" if rest else "GraphQL"))
        response = self._submitQuery(gitquery, gitvars=gitvars, verbose=(verbosity > 0), rest=rest)
        _vPrint((verbosity >= 0), "Checking response...")
        _vPrint((verbosity >= 0), response["headDict"]["http"])
        statusNum = response["statusNum"]

        # Decrement page count before error checks to properly reflect any repeated queries
        pageNum -= 1

        # Make sure the query limit didn't run out
        try:
            apiStatus = {
                "limit": int(response["headDict"]["X-RateLimit-Limit"]),
                "remaining": int(response["headDict"]["X-RateLimit-Remaining"]),
                "reset": int(response["headDict"]["X-RateLimit-Reset"])
            }
            _vPrint((verbosity >= 0), "API Status %s" % (json.dumps(apiStatus)))
            if not apiStatus["remaining"] > 0:
                _vPrint((verbosity >= 0), "API usage limit reached during query.")
                self._awaitReset(apiStatus["reset"])
                _vPrint((verbosity >= 0), "Repeating query...")
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=(requestCount - 1), pageNum=pageNum)
        except KeyError:
            # Handles error cases that don't return X-RateLimit data
            _vPrint((verbosity >= 0), "Failed to check API Status.")

        # Check for accepted but not yet processed, usually due to un-cached data
        if statusNum == 202:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Query accepted but not yet processed. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for server error responses
        if statusNum == 502 or statusNum == 503:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Server error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for other error responses
        if statusNum >= 400 or statusNum == 204:
            raise RuntimeError("Request got an Error response.\n%s\n%s" % (response["headDict"]["http"], response["result"]))

        _vPrint((verbosity >= 0), "Data received!")
        outObj = json.loads(response["result"])

        # Check for GraphQL API errors (e.g. repo not found)
        if not rest and "errors" in outObj:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            elif len(outObj["errors"]) == 1 and len(outObj["errors"][0]) == 1:
                # Poorly defined error type, usually intermittent, try again.
                _vPrint((verbosity >= 0), "GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))
                self._countdown(self.__retryDelay, printString="Unknown API error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
            else:
                raise RuntimeError("GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))

        # Re-increment page count before the next page query
        pageNum += 1

        # Pagination
        if paginate:
            if rest and response["linkDict"]:
                if "next" in response["linkDict"]:
                    nextObj = self.queryGitHub(response["linkDict"]["next"], gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    outObj.extend(nextObj)
            elif not rest:
                if not cursorVar:
                    raise ValueError("Must specify argument 'cursorVar' to use GraphQL auto-pagination.")
                if not len(keysToList) > 0:
                    raise ValueError("Must specify argument 'keysToList' as a non-empty list to use GraphQL auto-pagination.")
                aPage = outObj
                for key in keysToList[0:-1]:
                    aPage = aPage[key]
                gitvars[cursorVar] = aPage["pageInfo"]["endCursor"]
                if aPage["pageInfo"]["hasNextPage"]:
                    nextObj = self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    newPage = nextObj
                    for key in keysToList[0:-1]:
                        newPage = newPage[key]
                    aPage[keysToList[-1]].extend(newPage[keysToList[-1]])
                aPage.pop("pageInfo", None)

        return outObj
def queryGitHub(self, gitquery, gitvars={}, verbosity=0, paginate=False, cursorVar=None, keysToList=[], rest=False, requestCount=0, pageNum=0):
        """Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.

        """
        requestCount += 1
        if pageNum < 0:  # no negative page numbers
            pageNum = 0
        pageNum += 1

        if paginate:
            _vPrint((verbosity >= 0), "Page %d" % (pageNum))
        _vPrint((verbosity >= 0), "Sending %s query..." % ("REST" if rest else "GraphQL"))
        response = self._submitQuery(gitquery, gitvars=gitvars, verbose=(verbosity > 0), rest=rest)
        _vPrint((verbosity >= 0), "Checking response...")
        _vPrint((verbosity >= 0), response["headDict"]["http"])
        statusNum = response["statusNum"]

        # Decrement page count before error checks to properly reflect any repeated queries
        pageNum -= 1

        # Make sure the query limit didn't run out
        try:
            apiStatus = {
                "limit": int(response["headDict"]["X-RateLimit-Limit"]),
                "remaining": int(response["headDict"]["X-RateLimit-Remaining"]),
                "reset": int(response["headDict"]["X-RateLimit-Reset"])
            }
            _vPrint((verbosity >= 0), "API Status %s" % (json.dumps(apiStatus)))
            if not apiStatus["remaining"] > 0:
                _vPrint((verbosity >= 0), "API usage limit reached during query.")
                self._awaitReset(apiStatus["reset"])
                _vPrint((verbosity >= 0), "Repeating query...")
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=(requestCount - 1), pageNum=pageNum)
        except KeyError:
            # Handles error cases that don't return X-RateLimit data
            _vPrint((verbosity >= 0), "Failed to check API Status.")

        # Check for accepted but not yet processed, usually due to un-cached data
        if statusNum == 202:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Query accepted but not yet processed. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for server error responses
        if statusNum == 502 or statusNum == 503:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            else:
                self._countdown(self.__retryDelay, printString="Server error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
        # Check for other error responses
        if statusNum >= 400 or statusNum == 204:
            raise RuntimeError("Request got an Error response.\n%s\n%s" % (response["headDict"]["http"], response["result"]))

        _vPrint((verbosity >= 0), "Data received!")
        outObj = json.loads(response["result"])

        # Check for GraphQL API errors (e.g. repo not found)
        if not rest and "errors" in outObj:
            if requestCount >= self.maxRetry:
                raise RuntimeError("Query attempted but failed %d times.\n%s\n%s" % (self.maxRetry, response["headDict"]["http"], response["result"]))
            elif len(outObj["errors"]) == 1 and len(outObj["errors"][0]) == 1:
                # Poorly defined error type, usually intermittent, try again.
                _vPrint((verbosity >= 0), "GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))
                self._countdown(self.__retryDelay, printString="Unknown API error. Trying again in %*dsec...", verbose=(verbosity >= 0))
                return self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=requestCount, pageNum=pageNum)
            else:
                raise RuntimeError("GraphQL API error.\n%s" % (json.dumps(outObj["errors"])))

        # Re-increment page count before the next page query
        pageNum += 1

        # Pagination
        if paginate:
            if rest and response["linkDict"]:
                if "next" in response["linkDict"]:
                    nextObj = self.queryGitHub(response["linkDict"]["next"], gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    outObj.extend(nextObj)
            elif not rest:
                if not cursorVar:
                    raise ValueError("Must specify argument 'cursorVar' to use GraphQL auto-pagination.")
                if not len(keysToList) > 0:
                    raise ValueError("Must specify argument 'keysToList' as a non-empty list to use GraphQL auto-pagination.")
                aPage = outObj
                for key in keysToList[0:-1]:
                    aPage = aPage[key]
                gitvars[cursorVar] = aPage["pageInfo"]["endCursor"]
                if aPage["pageInfo"]["hasNextPage"]:
                    nextObj = self.queryGitHub(gitquery, gitvars=gitvars, verbosity=verbosity, paginate=paginate, cursorVar=cursorVar, keysToList=keysToList, rest=rest, requestCount=0, pageNum=pageNum)
                    newPage = nextObj
                    for key in keysToList[0:-1]:
                        newPage = newPage[key]
                    aPage[keysToList[-1]].extend(newPage[keysToList[-1]])
                aPage.pop("pageInfo", None)

        return outObj
def queryGitHub ( self , gitquery , gitvars = { } , verbosity = 0 , paginate = False , cursorVar = None , keysToList = [ ] , rest = False , requestCount = 0 , pageNum = 0 ) : requestCount += 1 if pageNum < 0 : # no negative page numbers pageNum = 0 pageNum += 1 if paginate : _vPrint ( ( verbosity >= 0 ) , "Page %d" % ( pageNum ) ) _vPrint ( ( verbosity >= 0 ) , "Sending %s query..." % ( "REST" if rest else "GraphQL" ) ) response = self . _submitQuery ( gitquery , gitvars = gitvars , verbose = ( verbosity > 0 ) , rest = rest ) _vPrint ( ( verbosity >= 0 ) , "Checking response..." ) _vPrint ( ( verbosity >= 0 ) , response [ "headDict" ] [ "http" ] ) statusNum = response [ "statusNum" ] # Decrement page count before error checks to properly reflect any repeated queries pageNum -= 1 # Make sure the query limit didn't run out try : apiStatus = { "limit" : int ( response [ "headDict" ] [ "X-RateLimit-Limit" ] ) , "remaining" : int ( response [ "headDict" ] [ "X-RateLimit-Remaining" ] ) , "reset" : int ( response [ "headDict" ] [ "X-RateLimit-Reset" ] ) } _vPrint ( ( verbosity >= 0 ) , "API Status %s" % ( json . dumps ( apiStatus ) ) ) if not apiStatus [ "remaining" ] > 0 : _vPrint ( ( verbosity >= 0 ) , "API usage limit reached during query." ) self . _awaitReset ( apiStatus [ "reset" ] ) _vPrint ( ( verbosity >= 0 ) , "Repeating query..." ) return self . queryGitHub ( gitquery , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = ( requestCount - 1 ) , pageNum = pageNum ) except KeyError : # Handles error cases that don't return X-RateLimit data _vPrint ( ( verbosity >= 0 ) , "Failed to check API Status." ) # Check for accepted but not yet processed, usually due to un-cached data if statusNum == 202 : if requestCount >= self . maxRetry : raise RuntimeError ( "Query attempted but failed %d times.\n%s\n%s" % ( self . maxRetry , response [ "headDict" ] [ "http" ] , response [ "result" ] ) ) else : self . _countdown ( self . __retryDelay , printString = "Query accepted but not yet processed. Trying again in %*dsec..." , verbose = ( verbosity >= 0 ) ) return self . queryGitHub ( gitquery , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = requestCount , pageNum = pageNum ) # Check for server error responses if statusNum == 502 or statusNum == 503 : if requestCount >= self . maxRetry : raise RuntimeError ( "Query attempted but failed %d times.\n%s\n%s" % ( self . maxRetry , response [ "headDict" ] [ "http" ] , response [ "result" ] ) ) else : self . _countdown ( self . __retryDelay , printString = "Server error. Trying again in %*dsec..." , verbose = ( verbosity >= 0 ) ) return self . queryGitHub ( gitquery , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = requestCount , pageNum = pageNum ) # Check for other error responses if statusNum >= 400 or statusNum == 204 : raise RuntimeError ( "Request got an Error response.\n%s\n%s" % ( response [ "headDict" ] [ "http" ] , response [ "result" ] ) ) _vPrint ( ( verbosity >= 0 ) , "Data received!" ) outObj = json . loads ( response [ "result" ] ) # Check for GraphQL API errors (e.g. repo not found) if not rest and "errors" in outObj : if requestCount >= self . maxRetry : raise RuntimeError ( "Query attempted but failed %d times.\n%s\n%s" % ( self . maxRetry , response [ "headDict" ] [ "http" ] , response [ "result" ] ) ) elif len ( outObj [ "errors" ] ) == 1 and len ( outObj [ "errors" ] [ 0 ] ) == 1 : # Poorly defined error type, usually intermittent, try again. _vPrint ( ( verbosity >= 0 ) , "GraphQL API error.\n%s" % ( json . dumps ( outObj [ "errors" ] ) ) ) self . _countdown ( self . __retryDelay , printString = "Unknown API error. Trying again in %*dsec..." , verbose = ( verbosity >= 0 ) ) return self . queryGitHub ( gitquery , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = requestCount , pageNum = pageNum ) else : raise RuntimeError ( "GraphQL API error.\n%s" % ( json . dumps ( outObj [ "errors" ] ) ) ) # Re-increment page count before the next page query pageNum += 1 # Pagination if paginate : if rest and response [ "linkDict" ] : if "next" in response [ "linkDict" ] : nextObj = self . queryGitHub ( response [ "linkDict" ] [ "next" ] , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = 0 , pageNum = pageNum ) outObj . extend ( nextObj ) elif not rest : if not cursorVar : raise ValueError ( "Must specify argument 'cursorVar' to use GraphQL auto-pagination." ) if not len ( keysToList ) > 0 : raise ValueError ( "Must specify argument 'keysToList' as a non-empty list to use GraphQL auto-pagination." ) aPage = outObj for key in keysToList [ 0 : - 1 ] : aPage = aPage [ key ] gitvars [ cursorVar ] = aPage [ "pageInfo" ] [ "endCursor" ] if aPage [ "pageInfo" ] [ "hasNextPage" ] : nextObj = self . queryGitHub ( gitquery , gitvars = gitvars , verbosity = verbosity , paginate = paginate , cursorVar = cursorVar , keysToList = keysToList , rest = rest , requestCount = 0 , pageNum = pageNum ) newPage = nextObj for key in keysToList [ 0 : - 1 ] : newPage = newPage [ key ] aPage [ keysToList [ - 1 ] ] . extend ( newPage [ keysToList [ - 1 ] ] ) aPage . pop ( "pageInfo" , None ) return outObj
Submit a GitHub query.

        Args:
            gitquery (str): The query or endpoint itself.
                Examples:
                       query: 'query { viewer { login } }'
                    endpoint: '/user'
            gitvars (Optional[Dict]): All query variables.
                Defaults to empty.
                GraphQL Only.
            verbosity (Optional[int]): Changes output verbosity levels.
                If < 0, all extra printouts are suppressed.
                If == 0, normal print statements are displayed.
                If > 0, additional status print statements are displayed.
                Defaults to 0.
            paginate (Optional[bool]): Pagination will be completed
                automatically if True. Defaults to False.
            cursorVar (Optional[str]): Key in 'gitvars' that represents the
                pagination cursor. Defaults to None.
                GraphQL Only.
            keysToList (Optional[List[str]]): Ordered list of keys needed to
                retrieve the list in the query results to be extended by
                pagination. Defaults to empty.
                Example:
                    ['data', 'viewer', 'repositories', 'nodes']
                GraphQL Only.
            rest (Optional[bool]): If True, uses the REST API instead
                of GraphQL. Defaults to False.
            requestCount (Optional[int]): Counter for repeated requests.
            pageNum (Optional[int]): Counter for pagination.
                For user readable log messages only, does not affect data.

        Returns:
            Dict: A JSON style dictionary.
Submit a GitHub query .