def run_ner(self, doc):
        entities = []
        # Apply the ner algorithm which takes a list of sentences and returns
        # a list of sentences, each being a list of NER-tokens, each of which is
        # a pairs (tokenstring, class)
        ner_sentences = self.ner(doc.get_sentences())
        # Flatten the nested list above into just a list of kinds
        ner_kinds = (k for s in ner_sentences for (_, k) in s)

        # We build a large iterator z that goes over tuples like the following:
        #  (offset, (token, kind))
        # offset just goes incrementally from 0

        z = itertools.chain(
            enumerate(zip(doc.tokens, ner_kinds)),
            # Add a sentinel last token to simplify last iteration of loop below
            [(len(doc.tokens), (None, 'INVALID'))]
        )

        # Traverse z, looking for changes in the kind field. If there is a
        # change of kind, we have a new set of contiguous tokens; if the kind
        # of those isn't "O" (which means "other"), record the occurrence
        #
        # offset keeps the start of the current token run; last_kind keeps the kind.
        last_kind = 'O'
        offset = 0
        for i, (token, kind) in z:
            if kind != last_kind:
                if last_kind != 'O':
                    # Found a new entity in offset:i
                    name = ' '.join(doc.tokens[offset:i])
                    entities.append(
                        self.build_occurrence(name, last_kind.lower(), name, offset, i)
                    )
                # Restart offset counter at each change of entity type
                offset = i
            last_kind = kind

        # Just a sanity check: verify that all NER tokens were consumed
        try:
            next(ner_kinds)
            assert False, "ner_kinds should have been completely consumed"
        except __HOLE__:
            # Actually the stop iteration is the expected result here
            pass

        return entities
dataset/ETHPy150Open machinalis/iepy/iepy/preprocess/ner/stanford.py/NERRunner.run_ner