'Check that the tokens in the tree have the correct penalties.
Args:
tree: the pytree.
list_of_expected: list of (name, penalty) pairs. Non-semantic tokens are
filtered out from the expected values.'

def _CheckPenalties(self, tree, list_of_expected):

    def FlattenRec(tree):
        if (pytree_utils.NodeName(tree) in pytree_utils.NONSEMANTIC_TOKENS):
            return []
        if isinstance(tree, pytree.Leaf):
            return [(tree.value, pytree_utils.GetNodeAnnotation(tree, pytree_utils.Annotation.SPLIT_PENALTY))]
        nodes = []
        for node in tree.children:
            nodes += FlattenRec(node)
        return nodes
    self.assertEqual(list_of_expected, FlattenRec(tree))
