def normalize_key_phrases (path, ranks, stopwords=None, spacy_nlp=None, skip_ner=True):
global STOPWORDS, SPACY_NLP<EOL>if (type(stopwords) is list) or (type(stopwords) is set):<EOL><INDENT>stopwords = set(stopwords)<EOL><DEDENT>else:<EOL><INDENT>if not STOPWORDS:<EOL><INDENT>STOPWORDS = load_stopwords(stopwords)<EOL><DEDENT>stopwords = STOPWORDS<EOL><DEDENT>if not spacy_nlp:<EOL><INDENT>if not SPACY_NLP:<EOL><INDENT>SPACY_NLP = spacy.load("<STR_LIT>")<EOL><DEDENT>spacy_nlp = SPACY_NLP<EOL><DEDENT>single_lex = {}<EOL>phrase_lex = {}<EOL>if isinstance(path, str):<EOL><INDENT>path = json_iter(path)<EOL><DEDENT>for meta in path:<EOL><INDENT>sent = [w for w in map(WordNode._make, meta["<STR_LIT>"])]<EOL>for rl in collect_keyword(sent, ranks, stopwords):<EOL><INDENT>id = str(rl.ids)<EOL>if id not in single_lex:<EOL><INDENT>single_lex[id] = rl<EOL><DEDENT>else:<EOL><INDENT>prev_lex = single_lex[id]<EOL>single_lex[id] = rl._replace(count = prev_lex.count + <NUM_LIT:1>)<EOL><DEDENT><DEDENT>if not skip_ner:<EOL><INDENT>for rl in collect_entities(sent, ranks, stopwords, spacy_nlp):<EOL><INDENT>id = str(rl.ids)<EOL>if id not in phrase_lex:<EOL><INDENT>phrase_lex[id] = rl<EOL><DEDENT>else:<EOL><INDENT>prev_lex = phrase_lex[id]<EOL>phrase_lex[id] = rl._replace(count = prev_lex.count + <NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT>for rl in collect_phrases(sent, ranks, spacy_nlp):<EOL><INDENT>id = str(rl.ids)<EOL>if id not in phrase_lex:<EOL><INDENT>phrase_lex[id] = rl<EOL><DEDENT>else:<EOL><INDENT>prev_lex = phrase_lex[id]<EOL>phrase_lex[id] = rl._replace(count = prev_lex.count + <NUM_LIT:1>)<EOL><DEDENT><DEDENT><DEDENT>rank_list = [rl.rank for rl in single_lex.values()]<EOL>if len(rank_list) < <NUM_LIT:1>:<EOL><INDENT>max_single_rank = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>max_single_rank = max(rank_list)<EOL><DEDENT>repeated_roots = {}<EOL>for rl in sorted(phrase_lex.values(), key=lambda rl: len(rl), reverse=True):<EOL><INDENT>rank_list = []<EOL>for i in iter(range(<NUM_LIT:0>, len(rl.ids))):<EOL><INDENT>id = rl.ids[i]<EOL>if not id in repeated_roots:<EOL><INDENT>repeated_roots[id] = <NUM_LIT:1.0><EOL>rank_list.append(rl.rank[i])<EOL><DEDENT>else:<EOL><INDENT>repeated_roots[id] += <NUM_LIT:1.0><EOL>rank_list.append(rl.rank[i] / repeated_roots[id])<EOL><DEDENT><DEDENT>phrase_rank = calc_rms(rank_list)<EOL>single_lex[str(rl.ids)] = rl._replace(rank = phrase_rank)<EOL><DEDENT>sum_ranks = sum([rl.rank for rl in single_lex.values()])<EOL>for rl in sorted(single_lex.values(), key=lambda rl: rl.rank, reverse=True):<EOL><INDENT>if sum_ranks > <NUM_LIT:0.0>:<EOL><INDENT>rl = rl._replace(rank=rl.rank / sum_ranks)<EOL><DEDENT>elif rl.rank == <NUM_LIT:0.0>:<EOL><INDENT>rl = rl._replace(rank=<NUM_LIT:0.1>)<EOL><DEDENT>rl = rl._replace(text=re.sub(r"<STR_LIT>", r"<STR_LIT>", rl.text))<EOL>yield rl<EOL><DEDENT>
collect keyphrases, named entities, etc., while removing stop words