'Helper used to implement the view methods -- returns a list of
tokens, (segmented) words, chunks, or sentences. The tokens
and chunks may optionally be tagged (with POS and sense
information).
:param fileid: The name of the underlying file.
:param unit: One of `\'token\'`, `\'word\'`, or `\'chunk\'`.
:param bracket_sent: If true, include sentence bracketing.
:param pos_tag: Whether to include part-of-speech tags.
:param sem_tag: Whether to include semantic tags, namely WordNet lemma
and OOV named entity status.'

def _words(self, fileid, unit, bracket_sent, pos_tag, sem_tag):

    assert (unit in (u'token', u'word', u'chunk'))
    result = []
    xmldoc = ElementTree.parse(fileid).getroot()
    for xmlsent in xmldoc.findall(u'.//s'):
        sent = []
        for xmlword in _all_xmlwords_in(xmlsent):
            itm = SemcorCorpusReader._word(xmlword, unit, pos_tag, sem_tag, self._wordnet)
            if (unit == u'word'):
                sent.extend(itm)
            else:
                sent.append(itm)
        if bracket_sent:
            result.append(SemcorSentence(xmlsent.attrib[u'snum'], sent))
        else:
            result.extend(sent)
    assert (None not in result)
    return result
