'Returns the arguments for Sentence.append() from a tagged token representation.
The order in which token tags appear can be specified.
The default order is (separated by slashes):
- word,
- part-of-speech,
- (IOB-)chunk,
- (IOB-)preposition,
- chunk(-relation)(-role),
- anchor,
- lemma.
Examples:
The/DT/B-NP/O/NP-SBJ-1/O/the
cats/NNS/I-NP/O/NP-SBJ-1/O/cat
clawed/VBD/B-VP/O/VP-1/A1/claw
at/IN/B-PP/B-PNP/PP/P1/at
the/DT/B-NP/I-PNP/NP/P1/the
sofa/NN/I-NP/I-PNP/NP/P1/sofa
././O/O/O/O/.
Returns a (word, lemma, type, chunk, role, relation, preposition, anchor, iob, custom)-tuple,
which can be passed to Sentence.append(): Sentence.append(*Sentence.parse_token("cats/NNS/NP"))
The custom value is a dictionary of (tag, value)-items of unrecognized tags in the token.'

def parse_token(self, token, tags=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):

    p = {WORD: '', POS: None, IOB: None, CHUNK: None, PNP: None, REL: None, ROLE: None, ANCHOR: None, LEMMA: None}
    custom = {}
    for (k, v) in izip(tags, token.split('/')):
        if (SLASH0 in v):
            v = v.replace(SLASH, '/')
        if (k == 'pos'):
            k = POS
        if (k not in p):
            custom[k] = None
        if ((v != OUTSIDE) or (k == WORD) or (k == LEMMA)):
            (p if (k not in custom) else custom)[k] = v
    ch = p[CHUNK]
    if ((ch is not None) and ch.startswith(('B-', 'I-'))):
        (p[IOB], p[CHUNK]) = (ch[:1], ch[2:])
    if (p[REL] is not None):
        (ch, p[REL], p[ROLE]) = self._parse_relation(p[REL])
        if ((ch == 'PP') and self._previous and (self._previous[REL] == p[REL]) and (self._previous[ROLE] == p[ROLE])):
            ch = 'NP'
        if ((p[CHUNK] is None) and (ch != OUTSIDE)):
            p[CHUNK] = ch
    self._previous = p
    return (p[WORD], p[LEMMA], p[POS], p[CHUNK], p[ROLE], p[REL], p[PNP], p[ANCHOR], p[IOB], custom)
