def search(index, query, snippet_folder=SNIPPET_FOLDER, term_freq=term_freq):
    """
    The highest level of search function
    """
    fallback_log = []
    code_list = []
    tokens = tokenize(query)
    tokens, abbr_log = abbr_expand(tokens)
    tokens, correct_log = correct(tokens, term_freq)
    tokens = lemmatize(tokens)
    tokens = filterout(tokens)
    while len(tokens) > 0: # Fallback mechanism
        code_list = fetch(index, tokens)
        if len(code_list) > 0:
            break
        tokens.sort(key=lambda tk:len(index.get(tk, [])))
        remove = tokens.pop()
        fallback_log.append(remove)
    snippets = get_snippets(code_list, snippet_folder)
    hints, hint_scores = get_hints(code_list, current_tokens=tokens)
    response = list(zip(code_list, snippets))
    response.sort(key=result_sort_key, reverse=True)

    # Count search_frequency
    if len(response) <= MAX_RESULT: # the respone can be shown in one page
        search_freq.update(code_list)
        with open(SEARCH_FREQ_JSON, 'w') as f:
            json.dump(search_freq, f, indent=2)

    return response, tokens, hints, hint_scores, \
           abbr_log, correct_log, fallback_log
The highest level of search function