'Calculate the split penalty and total length for the tokens.'

def CalculateFormattingInformation(self):

    self.first.spaces_required_before = 1
    self.first.total_length = len(self.first.value)
    prev_token = self.first
    prev_length = self.first.total_length
    for token in self._tokens[1:]:
        if ((token.spaces_required_before == 0) and _SpaceRequiredBetween(prev_token, token)):
            token.spaces_required_before = 1
        tok_len = (len(token.value) if (not token.is_pseudo_paren) else 0)
        token.total_length = ((prev_length + tok_len) + token.spaces_required_before)
        token.split_penalty += _SplitPenalty(prev_token, token)
        token.must_break_before = _MustBreakBefore(prev_token, token)
        token.can_break_before = (token.must_break_before or _CanBreakBefore(prev_token, token))
        prev_length = token.total_length
        prev_token = token
