third_party/zxcvbn-cpp/data-scripts/build_frequency_lists.py
def filter_frequency_lists(freq_lists):
    '''
    filters frequency data according to:
        - filter out short tokens if they are too rare.
        - filter out tokens if they already appear in another dict
          at lower rank.
        - cut off final freq_list at limits set in DICTIONARIES, if any.
    '''
    filtered_token_and_rank = {} # maps {name: [(token, rank), ...]}
    token_count = {}             # maps freq list name: current token count.
    for name in freq_lists:
        filtered_token_and_rank[name] = []
        token_count[name] = 0
    minimum_rank = {} # maps token -> lowest token rank across all freq lists
    minimum_name = {} # maps token -> freq list name with lowest token rank
    for name, token_to_rank in sorted(freq_lists.items()):
        for token, rank in token_to_rank.items():
            if token not in minimum_rank:
                assert token not in minimum_name
                minimum_rank[token] = rank
                minimum_name[token] = name
            else:
                assert token in minimum_name
                assert minimum_name[token] != name, 'same token occurs multiple times in %s' % name
                min_rank = minimum_rank[token]
                if rank < min_rank:
                    minimum_rank[token] = rank
                    minimum_name[token] = name
    for name, token_to_rank in sorted(freq_lists.items()):
        for token, rank in token_to_rank.items():
            if minimum_name[token] != name:
                continue
            if is_rare_and_short(token, rank) or has_comma_or_double_quote(token, rank, name):
                continue
            filtered_token_and_rank[name].append((token, rank))
            token_count[name] += 1
    result = {}
    for name, token_rank_pairs in filtered_token_and_rank.items():
        token_rank_pairs.sort(key=itemgetter(1))
        cutoff_limit = DICTIONARIES[name]
        if cutoff_limit and len(token_rank_pairs) > cutoff_limit:
            token_rank_pairs = token_rank_pairs[:cutoff_limit]
        result[name] = [pair[0] for pair in token_rank_pairs] # discard rank post-sort
    return result

    filters frequency data according to:
        - filter out short tokens if they are too rare.
        - filter out tokens if they already appear in another dict
          at lower rank.
        - cut off final freq_list at limits set in DICTIONARIES, if any.
    
filters frequency data according to:
filter out short tokens if they are too rare.
filter out tokens if they already appear in another dict
at lower rank.
cut off final freq_list at limits set in DICTIONARIES, if any.
filters frequency data according to : filter out short tokens if they are too rare . filter out tokens if they already appear in another dict at lower rank . cut off final freq_list at limits set in DICTIONARIES if any .
def filter_frequency_lists(freq_lists):
    filtered_token_and_rank = {} 
    token_count = {}             
    for name in freq_lists:
        filtered_token_and_rank[name] = []
        token_count[name] = 0
    minimum_rank = {} 
    minimum_name = {} 
    for name, token_to_rank in sorted(freq_lists.items()):
        for token, rank in token_to_rank.items():
            if token not in minimum_rank:
                assert token not in minimum_name
                minimum_rank[token] = rank
                minimum_name[token] = name
            else:
                assert token in minimum_name
                assert minimum_name[token] != name, 'same token occurs multiple times in %s' % name
                min_rank = minimum_rank[token]
                if rank < min_rank:
                    minimum_rank[token] = rank
                    minimum_name[token] = name
    for name, token_to_rank in sorted(freq_lists.items()):
        for token, rank in token_to_rank.items():
            if minimum_name[token] != name:
                continue
            if is_rare_and_short(token, rank) or has_comma_or_double_quote(token, rank, name):
                continue
            filtered_token_and_rank[name].append((token, rank))
            token_count[name] += 1
    result = {}
    for name, token_rank_pairs in filtered_token_and_rank.items():
        token_rank_pairs.sort(key=itemgetter(1))
        cutoff_limit = DICTIONARIES[name]
        if cutoff_limit and len(token_rank_pairs) > cutoff_limit:
            token_rank_pairs = token_rank_pairs[:cutoff_limit]
        result[name] = [pair[0] for pair in token_rank_pairs] 
    return result
def filter_frequency_lists ( freq_lists ) : filtered_token_and_rank = { } token_count = { } for name in freq_lists : filtered_token_and_rank [ name ] = [ ] token_count [ name ] = 0 minimum_rank = { } minimum_name = { } for name , token_to_rank in sorted ( freq_lists . items ( ) ) : for token , rank in token_to_rank . items ( ) : if token not in minimum_rank : assert token not in minimum_name minimum_rank [ token ] = rank minimum_name [ token ] = name else : assert token in minimum_name assert minimum_name [ token ] != name , 'same token occurs multiple times in %s' % name min_rank = minimum_rank [ token ] if rank < min_rank : minimum_rank [ token ] = rank minimum_name [ token ] = name for name , token_to_rank in sorted ( freq_lists . items ( ) ) : for token , rank in token_to_rank . items ( ) : if minimum_name [ token ] != name : continue if is_rare_and_short ( token , rank ) or has_comma_or_double_quote ( token , rank , name ) : continue filtered_token_and_rank [ name ] . append ( ( token , rank ) ) token_count [ name ] += 1 result = { } for name , token_rank_pairs in filtered_token_and_rank . items ( ) : token_rank_pairs . sort ( key = itemgetter ( 1 ) ) cutoff_limit = DICTIONARIES [ name ] if cutoff_limit and len ( token_rank_pairs ) > cutoff_limit : token_rank_pairs = token_rank_pairs [ : cutoff_limit ] result [ name ] = [ pair [ 0 ] for pair in token_rank_pairs ] return result
filters frequency data according to:
 filter out short tokens if they are too rare.
filters frequency data according to : filter out short tokens if they are too rare .
'''
    filters frequency data according to:
        - filter out short tokens if they are too rare.
        - filter out tokens if they already appear in another dict
          at lower rank.
        - cut off final freq_list at limits set in DICTIONARIES, if any.
    ''' # maps {name: [(token, rank), ...]} # maps freq list name: current token count. # maps token -> lowest token rank across all freq lists # maps token -> freq list name with lowest token rank # discard rank post-sort
freq_lists