'Helper method to return all tokens in an extent.
This functionality is needed multiple places in this module. We define
it here because it seems like a logical place.'

@staticmethod
def get_tokens(tu, extent):

    tokens_memory = POINTER(Token)()
    tokens_count = c_uint()
    conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), byref(tokens_count))
    count = int(tokens_count.value)
    if (count < 1):
        return
    tokens_array = cast(tokens_memory, POINTER((Token * count))).contents
    token_group = TokenGroup(tu, tokens_memory, tokens_count)
    for i in range(0, count):
        token = Token()
        token.int_data = tokens_array[i].int_data
        token.ptr_data = tokens_array[i].ptr_data
        token._tu = tu
        token._group = token_group
        (yield token)
