def remove_near_duplicate_relation(triples, threshold=<NUM_LIT>):
logging.debug("<STR_LIT>")<EOL>_assert_threshold(threshold)<EOL>duplicate_rel_counter = defaultdict(list)<EOL>relations = set()<EOL>for t in triples:<EOL><INDENT>duplicate_rel_counter[t.relation].append(f"<STR_LIT>")<EOL>relations.add(t.relation)<EOL><DEDENT>relations = list(relations)<EOL>num_triples = len(triples)<EOL>removal_relation_set = set()<EOL>for rel, values in duplicate_rel_counter.items():<EOL><INDENT>duplicate_rel_counter[rel] = Superminhash(values)<EOL><DEDENT>for i in relations:<EOL><INDENT>for j in relations:<EOL><INDENT>if i == j or i in removal_relation_set or j in removal_relation_set: continue<EOL>close_relations = [i]<EOL>if _set_close_to(duplicate_rel_counter[i], duplicate_rel_counter[j], threshold):<EOL><INDENT>close_relations.append(j)<EOL><DEDENT><DEDENT>if len(close_relations) > <NUM_LIT:1>:<EOL><INDENT>close_relations.pop(np.random.randint(len(close_relations)))<EOL>removal_relation_set |= set(close_relations)<EOL><DEDENT><DEDENT>logging.info("<STR_LIT>".format(len(removal_relation_set), str(removal_relation_set)))<EOL>return list(filterfalse(lambda x: x.relation in removal_relation_set, triples))<EOL>
If entity pairs in a relation is as close as another relations, only keep one relation of such set.