def tokenize (self, value):
        """Take a string and break it into tokens. Return the tokens as a list of
        strings.

        """

        # This code uses a state machine:
        class STATE:
            NORMAL = 0
            GROUP_PUNCTUATION = 1
            PROCESS_HTML_TAG = 2
            PROCESS_HTML_ENTITY = 3
            GROUP_LINEBREAKS = 4

        state_names = {
            STATE.NORMAL: "normal",
            STATE.GROUP_PUNCTUATION: "punctuation",
            STATE.PROCESS_HTML_TAG: "html",
            STATE.PROCESS_HTML_ENTITY: "html_entity",
            STATE.GROUP_LINEBREAKS: "break"
        }

        # "state" and "token" have array values to allow their
        # contents to be modified within finishToken().
        state = [STATE.NORMAL]
        token = [""] # The current token being assembled.
        tokens = [] # The tokens extracted from the input.
        index = -1

        def clearToken():
            """Clear the current token and return to normal state."""
            token[0] = ""
            state[0] = STATE.NORMAL

        def emitToken():
            """Emit the current token, if any, and return to normal state."""
            if len(token[0]) > 0:
                # add character end and start
                char_start, char_end = index, index + len(token[0])
                if self.create_structured_tokens:
                    new_token = {'value': token[0], 'type': state_names[state[0]], 'char_start': char_start, 'char_end': char_end}
                    tokens.append(new_token)
                else:
                    tokens.append(token[0])
            clearToken()

        def fixBrokenHtmlEntity():
            # This is not a valid HTML entity.
            # TODO: embedded "#" characters should be treated better
            # here.
            if not self.recognizePunctuation:
                # If we aren't treating punctuation specially, then just treat
                # the broken HTML entity as an ordinary token.
                #
                # TODO: This is not quite correct.  "x& " should
                # be treated as a single token, althouth "s & "
                # should result in two tokens.
                state[0] = STATE.NORMAL
                return
            if self.groupPunctuation:
                # If all the saved tokens are punctuation characters, then
                # enter STATE.GROUP_PUNCTUATION insted of STATE.NORMAL.
                sawOnlyPunctuation = True
                for c in token[0]:
                    if c not in CrfTokenizer.punctuationSet:
                        sawOnlyPunctuation = False
                        break
                if sawOnlyPunctuation:
                    state[0] = STATE.GROUP_PUNCTUATION
                    return

            # Emit the ampersand that began the prospective entity and use the
            # rest as a new current token.
            saveToken = token[0]
            token[0] = saveToken[0:1]
            emitToken()
            if len(saveToken) > 1:
                token[0] = saveToken[1:]
            # The caller should continue processing with the current
            # character.

        # Process each character in the input string:
        for c in value:
            index += 1
            if state[0] == STATE.PROCESS_HTML_TAG:
                if c in CrfTokenizer.whitespaceSet:
                    continue # Suppress for safety. CRF++ doesn't like spaces in tokens, for example.
                token[0] += c
                if c == CrfTokenizer.END_HTML_TAG_CHAR:
                    if self.skipHtmlTags:
                        clearToken()
                    else:
                        emitToken()
                continue

            if state[0] == STATE.PROCESS_HTML_ENTITY:
                # Parse an HTML entity name. TODO: embedded "#"
                # characters imply more extensive parsing rules should
                # be performed here.
                if c == CrfTokenizer.END_HTML_ENTITY_CHAR:
                    if len(token[0]) == 1:
                        # This is the special case of "&;", which is not a
                        # valid HTML entity.  If self.groupPunctuation is
                        # True, return to normal parsing state in case more
                        # punctuation follows.  Otherwise, emit "&" and ";" as
                        # separate tokens.
                        if not self.recognizePunctuation:
                            # TODO: This is not quite correct.  "x&;" should
                            # be treated as a single token, althouth "s &;"
                            # should result in two tokens.
                            token[0] = token[0] + c
                            state[0] = STATE.NORMAL
                        elif self.groupPunctuation:
                            token[0] = token[0] + c
                            state[0] = STATE.GROUP_PUNCTUATION
                        else:
                            emitToken() # Emit the "&" as a seperate token.
                            token[0] = token[0] + c
                            emitToken() # Emit the ";' as a seperate token.
                        continue
                    token[0] = token[0] + c
                    if self.skipHtmlEntities:
                        clearToken()
                    else:
                        emitToken()
                    continue
                elif c in CrfTokenizer.htmlEntityNameCharacterSet:
                    token[0] = token[0] + c
                    continue
                else:
                    # This is not a valid HTML entity.
                    fixBrokenHtmlEntity()
                    # intentional fall-through

            if state[0] == STATE.GROUP_LINEBREAKS:
                # we will look for \n\r and ignore spaces
                if c in CrfTokenizer.linebreaking_character_set:
                    token[0] += c
                    continue
                elif c in CrfTokenizer.whitespaceSet:
                    continue
                else:
                    emitToken()
                    state[0] = STATE.NORMAL

            if c in CrfTokenizer.whitespaceSet:
                # White space terminates the current token, then is dropped.
                emitToken()
                # Check to see whether we should look for line breaks
                if c in CrfTokenizer.linebreaking_start_character_set and self.recognize_linebreaks:
                    state[0] = STATE.GROUP_LINEBREAKS
                    token[0] = c

            elif c == CrfTokenizer.START_HTML_TAG_CHAR and self.recognizeHtmlTags:
                emitToken()
                state[0] = STATE.PROCESS_HTML_TAG
                token[0] = c

            elif c == CrfTokenizer.START_HTML_ENTITY_CHAR and self.recognizeHtmlEntities:
                emitToken()
                state[0] = STATE.PROCESS_HTML_ENTITY
                token[0] = c

            elif c in CrfTokenizer.punctuationSet and self.recognizePunctuation:
                if self.groupPunctuation:
                    # Finish any current token.  Concatenate
                    # contiguous punctuation into a single token:
                    if state[0] != STATE.GROUP_PUNCTUATION:
                        emitToken()
                        state[0] = STATE.GROUP_PUNCTUATION
                    token[0] = token[0] + c
                else:
                    # Finish any current token and form a token from
                    # the punctuation character:
                    emitToken()
                    token[0] = c
                    emitToken()

            else:
                # Everything else goes here. Presumably, that includes
                # Unicode characters that aren't ASCII
                # strings. Further work is needed.
                if state[0] != STATE.NORMAL:
                    emitToken()
                token[0] = token[0] + c

        # Finish any final token and return the array of tokens:
        if state[0] == STATE.PROCESS_HTML_ENTITY:
            fixBrokenHtmlEntity()
        emitToken()

        # Was a token prefix requested? If so, we'll apply it now.  If the
        # normal case is not to apply a token prefix, this might be a little
        # more efficient than applying the prefix in emitToken().
        if self.tokenPrefix is not None and len(self.tokenPrefix) > 0:
            tokens = map(lambda x: self.tokenPrefix + x, tokens)

        return tokens
Take a string and break it into tokens. Return the tokens as a list of
        strings.