def tokenize(self, data):
        """
        Tokenizes the given string. A token is a 4-tuple of the form:

            (token_type, tag_name, tag_options, token_text)

            token_type
                One of: TOKEN_TAG_START, TOKEN_TAG_END, TOKEN_NEWLINE, TOKEN_DATA
            tag_name
                The name of the tag if token_type=TOKEN_TAG_*, otherwise None
            tag_options
                A dictionary of options specified for TOKEN_TAG_START, otherwise None
            token_text
                The original token text
        """
        data = data.replace('\r\n', '\n').replace('\r', '\n')
        pos = start = end = 0
        ld = len(data)
        tokens = []
        while pos < ld:
            start = data.find(self.tag_opener, pos)
            if start >= pos:
                # Check to see if there was data between this start and the last end.
                if start > pos:
                    tl = self._newline_tokenize(data[pos:start])
                    tokens.extend(tl)
                    pos = start

                # Find the extent of this tag, if it's ever closed.
                end, found_close = self._tag_extent(data, start)
                if found_close:
                    tag = data[start:end]
                    valid, tag_name, closer, opts = self._parse_tag(tag)
                    # Make sure this is a well-formed, recognized tag, otherwise it's just data.
                    if valid and tag_name in self.recognized_tags:
                        if closer:
                            tokens.append((self.TOKEN_TAG_END, tag_name, None, tag))
                        else:
                            tokens.append((self.TOKEN_TAG_START, tag_name, opts, tag))
                    elif valid and self.drop_unrecognized and tag_name not in self.recognized_tags:
                        # If we found a valid (but unrecognized) tag and self.drop_unrecognized is True, just drop it.
                        pass
                    else:
                        tokens.extend(self._newline_tokenize(tag))
                else:
                    # We didn't find a closing tag, tack it on as text.
                    tokens.extend(self._newline_tokenize(data[start:end]))
                pos = end
            else:
                # No more tags left to parse.
                break
        if pos < ld:
            tl = self._newline_tokenize(data[pos:])
            tokens.extend(tl)
        return tokens
Tokenizes the given string. A token is a 4-tuple of the form:

            (token_type, tag_name, tag_options, token_text)

            token_type
                One of: TOKEN_TAG_START, TOKEN_TAG_END, TOKEN_NEWLINE, TOKEN_DATA
            tag_name
                The name of the tag if token_type=TOKEN_TAG_*, otherwise None
            tag_options
                A dictionary of options specified for TOKEN_TAG_START, otherwise None
            token_text
                The original token text