'Test that tokens are split with types as expected.'

def testTokenTypes(self):

    for (source, expected) in (('a    &&    b    ||    c', [('a', 'a'), ('&&', 'c'), ('b', 'a'), ('||', 'c'), ('c', 'a')]),):
        s = shlex.shlex(source, punctuation_chars=True)
        observed = []
        while True:
            t = s.get_token()
            if (t == s.eof):
                break
            if (t[0] in s.punctuation_chars):
                tt = 'c'
            else:
                tt = 'a'
            observed.append((t, tt))
        self.assertEqual(observed, expected)
