'Reads the robots.txt URL and feeds it to the parser.'

def read(self):

    opener = URLopener()
    f = opener.open(self.url)
    lines = [line.strip() for line in f]
    f.close()
    self.errcode = opener.errcode
    if (self.errcode in (401, 403)):
        self.disallow_all = True
    elif ((self.errcode >= 400) and (self.errcode < 500)):
        self.allow_all = True
    elif ((self.errcode == 200) and lines):
        self.parse(lines)
