def read (self):
        """Read the robots.txt URL and feeds it to the parser."""
        self._reset()
        kwargs = dict(
            headers = {
                'User-Agent': configuration.UserAgent,
                'Accept-Encoding': ACCEPT_ENCODING,
            }
        )
        if self.auth:
            kwargs["auth"] = self.auth
        if self.proxies:
            kwargs["proxies"] = self.proxies
        try:
            response = self.session.get(self.url, **kwargs)
            response.raise_for_status()
            content_type = response.headers.get('content-type')
            if content_type and content_type.lower().startswith('text/plain'):
                self.parse(response.iter_lines())
            else:
                log.debug(LOG_CHECK, "%r allow all (no text content)", self.url)
                self.allow_all = True
        except requests.HTTPError as x:
            if x.response.status_code in (401, 403):
                self.disallow_all = True
                log.debug(LOG_CHECK, "%r disallow all (code %d)", self.url, x.response.status_code)
            else:
                self.allow_all = True
                log.debug(LOG_CHECK, "%r allow all (HTTP error)", self.url)
        except requests.exceptions.Timeout:
            raise
        except requests.exceptions.RequestException:
            # no network or other failure
            self.allow_all = True
            log.debug(LOG_CHECK, "%r allow all (request error)", self.url)
Read the robots.txt URL and feeds it to the parser.