def parse(self, s):
        """Parses the passed string as a set of robots.txt rules."""
        self._sitemaps = [ ]
        self.__rulesets = [ ]
        
        if (PY_MAJOR_VERSION > 2) and (isinstance(s, bytes) or isinstance(s, bytearray)) or \
           (PY_MAJOR_VERSION == 2) and (not isinstance(s, unicode)):            
            s = s.decode("iso-8859-1")
    
        # Normalize newlines.
        s = _end_of_line_regex.sub("\n", s)
        
        lines = s.split("\n")
        
        previous_line_was_a_user_agent = False
        current_ruleset = None
        
        for line in lines:
            line = line.strip()
            
            if line and line[0] == '#':
                # "Lines containing only a comment are discarded completely, 
                # and therefore do not indicate a record boundary." (MK1994)
                pass
            else:
                # Remove comments
                i = line.find("#")
                if i != -1: line = line[:i]
        
                line = line.strip()
                
                if not line:
                    # An empty line indicates the end of a ruleset.
                    if current_ruleset and current_ruleset.is_not_empty():
                        self.__rulesets.append(current_ruleset)
                    
                    current_ruleset = None
                    previous_line_was_a_user_agent = False
                else:
                    # Each non-empty line falls into one of six categories:
                    # 1) User-agent: blah blah blah
                    # 2) Disallow: blah blah blah
                    # 3) Allow: blah blah blah
                    # 4) Crawl-delay: blah blah blah
                    # 5) Sitemap: blah blah blah
                    # 6) Everything else
                    # 1 - 5 are interesting and I find them with the regex 
                    # below. Category 6 I discard as directed by the MK1994 
                    # ("Unrecognised headers are ignored.")
                    # Note that 4 & 5 are specific to GYM2008 syntax, but 
                    # respecting them here is not a problem. They're just 
                    # additional information the the caller is free to ignore.
                    matches = _directive_regex.findall(line)
                    
                    # Categories 1 - 5 produce two matches, #6 produces none.
                    if matches:
                        field, data = matches[0]
                        field = field.lower()
                        data = _scrub_data(data)

                        # Matching "useragent" is a deviation from the 
                        # MK1994/96 which permits only "user-agent".
                        if field in ("useragent", "user-agent"):
                            if previous_line_was_a_user_agent:
                                # Add this UA to the current ruleset 
                                if current_ruleset and data:
                                    current_ruleset.add_robot_name(data)
                            else:
                                # Save the current ruleset and start a new one.
                                if current_ruleset and current_ruleset.is_not_empty():
                                    self.__rulesets.append(current_ruleset)
                                #else:
                                    # (is_not_empty() == False) ==> malformed 
                                    # robots.txt listed a UA line but provided
                                    # no name or didn't provide any rules 
                                    # for a named UA.
                                current_ruleset = _Ruleset()
                                if data: 
                                    current_ruleset.add_robot_name(data)
                            
                            previous_line_was_a_user_agent = True
                        elif field == "allow":
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                current_ruleset.add_allow_rule(data)
                        elif field == "sitemap":
                            previous_line_was_a_user_agent = False
                            self._sitemaps.append(data)
                        elif field == "crawl-delay":
                            # Only Yahoo documents the syntax for Crawl-delay.
                            # ref: http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-03.html
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                try:
                                    current_ruleset.crawl_delay = float(data)
                                except ValueError:
                                    # Invalid crawl-delay -- ignore.
                                    pass
                        else:
                            # This is a disallow line
                            previous_line_was_a_user_agent = False
                            if current_ruleset:
                                current_ruleset.add_disallow_rule(data)

        if current_ruleset and current_ruleset.is_not_empty():
            self.__rulesets.append(current_ruleset)
            
        # Now that I have all the rulesets, I want to order them in a way 
        # that makes comparisons easier later. Specifically, any ruleset that 
        # contains the default user agent '*' should go at the end of the list
        # so that I only apply the default as a last resort. According to 
        # MK1994/96, there should only be one ruleset that specifies * as the 
        # user-agent, but you know how these things go.
        not_defaults = [r for r in self.__rulesets if not r.is_default()]
        defaults = [r for r in self.__rulesets if r.is_default()]

        self.__rulesets = not_defaults + defaults
Parses the passed string as a set of robots.txt rules.