'parse the input lines from a robots.txt file.
We allow that a user-agent: line is not preceded by
one or more blank lines.'

def parse(self, lines):

    state = 0
    linenumber = 0
    entry = Entry()
    for line in lines:
        linenumber += 1
        if (not line):
            if (state == 1):
                entry = Entry()
                state = 0
            elif (state == 2):
                self._add_entry(entry)
                entry = Entry()
                state = 0
        i = line.find('#')
        if (i >= 0):
            line = line[:i]
        line = line.strip()
        if (not line):
            continue
        line = line.split(':', 1)
        if (len(line) == 2):
            line[0] = line[0].strip().lower()
            line[1] = urllib.unquote(line[1].strip())
            if (line[0] == 'user-agent'):
                if (state == 2):
                    self._add_entry(entry)
                    entry = Entry()
                entry.useragents.append(line[1])
                state = 1
            elif (line[0] == 'disallow'):
                if (state != 0):
                    entry.rulelines.append(RuleLine(line[1], False))
                    state = 2
            elif (line[0] == 'allow'):
                if (state != 0):
                    entry.rulelines.append(RuleLine(line[1], True))
                    state = 2
    if (state == 2):
        self._add_entry(entry)
