def _external_request(self, method, url, *args, **kwargs):
        """
        Wrapper for requests.get with useragent automatically set.
        And also all requests are reponses are cached.
        """
        self.last_url = url
        if url in self.responses.keys() and method == 'get':
            return self.responses[url] # return from cache if its there

        headers = kwargs.pop('headers', None)
        custom = {'User-Agent': useragent}
        if headers:
            headers.update(custom)
            kwargs['headers'] = headers
        else:
            kwargs['headers'] = custom

        if self.timeout:
            # add timeout parameter to requests.get if one was passed in on construction...
            kwargs['timeout'] = self.timeout

        start = datetime.datetime.now()
        response = getattr(requests, method)(url, verify=self.ssl_verify, *args, **kwargs)
        self.total_external_fetch_duration += datetime.datetime.now() - start

        if self.verbose:
            print("Got Response: %s (took %s)" % (url, (datetime.datetime.now() - start)))

        self.last_raw_response = response

        self.check_error(response)

        if method == 'get':
            self.responses[url] = response # cache for later

        return response
Wrapper for requests.get with useragent automatically set.
        And also all requests are reponses are cached.