def tokens(self, sentence_dom):
<EOL>self.sent_pos = <NUM_LIT:0><EOL>mention_id = <NUM_LIT:0><EOL>while len(sentence_dom.childNodes) > <NUM_LIT:0>:<EOL><INDENT>node = sentence_dom.childNodes.pop(<NUM_LIT:0>)<EOL>if node.nodeType == node.TEXT_NODE:<EOL><INDENT>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>assert node.nodeName == '<STR_LIT>', node.nodeName<EOL>chain_id = node.attributes.get('<STR_LIT>').value<EOL>entity_type = node.attributes.get('<STR_LIT>').value<EOL>for node in node.childNodes:<EOL><INDENT>assert node.nodeType == node.TEXT_NODE, node.nodeType<EOL>for line in node.data.splitlines(True):<EOL><INDENT>self._input_string = line<EOL>for start, end in self.word_tokenizer.span_tokenize(line):<EOL><INDENT>tok = self._make_token(start, end)<EOL>if tok:<EOL><INDENT>if entity_type in _PRONOUNS:<EOL><INDENT>tok.mention_type = MentionType.PRO<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL>attr = Attribute(<EOL>attribute_type=AttributeType.PER_GENDER,<EOL>value=str(_PRONOUNS[entity_type])<EOL>)<EOL>self.attributes.append(attr)<EOL><DEDENT>else:<EOL><INDENT>tok.mention_type = MentionType.NAME<EOL>tok.entity_type = _ENTITY_TYPES[entity_type]<EOL><DEDENT>tok.equiv_id = int(chain_id)<EOL>tok.mention_id = mention_id<EOL>yield tok<EOL><DEDENT><DEDENT>if line.endswith('<STR_LIT:\n>'):<EOL><INDENT>self.line_idx += <NUM_LIT:1><EOL><DEDENT>self.byte_idx += len(line.encode('<STR_LIT:utf-8>'))<EOL><DEDENT><DEDENT>mention_id += <NUM_LIT:1><EOL><DEDENT><DEDENT>
Tokenize all the words and preserve NER labels from ENAMEX tags