ups

app.py

@@ -1,5 +1,26 @@
 import streamlit as st
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+from japanese.embedding import encode_sentences, get_cadidate_embeddings
+from japanese.tokenizer import extract_keyphrase_candidates
+from japanese.ranker import DirectedCentralityRnak
+
+from transformers import AutoTokenizer
+from transformers import AutoModel
+
+
+if __name__ == '__main__':
+    # load model
+    model = AutoModel.from_pretrained('cl-tohoku/bert-base-japanese')
+    tokenizer = AutoTokenizer.from_pretrained('cl-tohoku/bert-base-japanese')
+
+    text = "紀元前509年、第7代の王タルクィニウス・スペルブスを追放し共和制を敷いたローマだが、問題は山積していた。まず、王に代わった執政官（コンスル）が元老院の意向で決められるようになったこと、またその被選挙権が40歳以上に限定されていたことから、若い市民を中心としてタルクィニウスを王位に復する王政復古の企みが起こった。これは失敗して、初代執政官ルキウス・ユニウス・ブルトゥスは、彼自身の息子ティトゥスを含む陰謀への参加者を処刑した。ラテン同盟諸都市やエトルリア諸都市との同盟は、これらの都市とローマ王との同盟という形であったため、王の追放で当然に同盟は解消され、対立関係となった。"
+    tokens, keyphrases = extract_keyphrase_candidates(text, tokenizer)
+
+    document_embs = encode_sentences([tokens], tokenizer, model)
+    document_feats = get_cadidate_embeddings([keyphrases], document_embs, [tokens])
+    ranker = DirectedCentralityRnak(document_feats, beta=0.1, lambda1=1, lambda2=0.9, alpha=1.2, processors=8)
+    phrases = ranker.extract_summary()
+    x = st.slider('Select a value')
+    st.write(x, 'squared is', x * x)
+    phrases
 

japanese/embedding.py

@@ -0,0 +1,100 @@
+import os
+import re
+import string
+import tojson
+import pickle
+
+import torch
+import numpy as np
+
+from transformers import BertTokenizer, AutoTokenizer, BertModel, AutoModel
+
+
+def encode_sentence(tokenizer, model, tokens):
+    is_split = []
+    input_tokens = ['[CLS]']
+    for token in tokens:
+        tmp = tokenizer.tokenize(token)
+        
+        if len(input_tokens) + len(tmp) >= 511:
+            break
+        else:
+            input_tokens.extend(tmp)
+            is_split.append(len(tmp))
+    input_tokens += ["[SEP]"]
+    input_ids = tokenizer.convert_tokens_to_ids(input_tokens)
+    
+    input_ids = torch.LongTensor([input_ids])
+    outputs = model(input_ids, output_hidden_states=True).last_hidden_state.detach().numpy()
+    bertcls  = outputs[0, 0, :]
+    o1 = outputs[0, :, :]
+    cls_token = o1[0]
+    
+    tokens_emb = []
+    i = 1
+    for j in is_split:
+        if j == 1:
+            tokens_emb.append(o1[i])
+            i += 1
+        else:
+            tokens_emb.append(sum(o1[i:i+j]) / j)
+            # tokens_emb.append(np.max(np.array(o1[i: i+j]), axis=0))
+            i += j
+        # if i >= len(is_split):
+        #     break
+    assert len(tokens_emb) == len(is_split)
+    return tokens_emb, bertcls, cls_token
+
+def flat_list(l):
+    return [x for ll in l for x in ll]
+
+def encode_sentences(token_list, tokenizer, model):
+    tokenizer.do_word_tokenize = False
+
+    document_embeddings = []
+    cnt = 0
+    for tokens in token_list:
+        tokens_emb, bertcls, cls_token = encode_sentence(tokenizer, model, tokens)
+
+        document_embeddings.append({
+            'document_id': cnt,
+            'doc_cls': cls_token,
+            'doc_bertcls': bertcls,
+            "tokens": tokens_emb
+        })
+        cnt += 1
+
+    return document_embeddings
+
+
+def get_cadidate_embeddings(token_list, document_embeddings, tokens):
+    document_feats = []
+    cnt = 0
+    for candidate_phrase, document_emb, each_tokens in zip(token_list, document_embeddings, tokens):
+        sentence_emb = document_emb['tokens']
+        
+        tmp_embeddings = []
+        tmp_candidate_phrase = []
+        
+        for tmp, (i, j) in candidate_phrase:
+            if j<=i:
+                continue
+            if j >= len(sentence_emb):
+                break
+            # tmp_embeddings.append(sum(sentence_emb[i:j]) / (j-i))
+            tmp_embeddings.append(np.max(np.array(sentence_emb[i:j]), axis=0))
+            tmp_candidate_phrase.append(tmp)
+
+        candidate_phrases_embeddings = tmp_embeddings
+        candidate_phrases = tmp_candidate_phrase
+
+        document_feats.append({
+            'document_id': cnt,
+            'tokens': each_tokens,
+            'candidate_phrases': candidate_phrases,
+            'candidate_phrases_embeddings': candidate_phrases_embeddings,
+            # 'sentence_embeddings': document_emb['doc_bertcls'],
+            'sentence_embeddings': document_emb['doc_cls'],
+        })
+        cnt += 1
+    return document_feats

japanese/ranker.py

@@ -0,0 +1,147 @@
+import numpy
+import pickle
+
+import numpy as np
+import os
+import re
+import string
+import tojson
+from multiprocessing import Pool
+
+import sys
+import tojson
+import re
+import string
+
+
+import numpy as np
+from numpy.linalg import norm
+
+class DirectedCentralityRnak(object):
+    def __init__(self, 
+                document_feats,
+                extract_num=20,
+                beta=0.2, 
+                lambda1=1, 
+                lambda2=0.8,
+                alpha=1,
+                processors=8):
+        self.extract_num = extract_num
+        self.processors = processors
+        self.beta = beta
+        self.lambda1 = lambda1
+        self.lambda2 = lambda2
+        self.alpha = alpha
+
+        self.candidate_phrases = [x['candidate_phrases'] for x in document_feats]
+        self.doc_embeddings = [x['sentence_embeddings'] for x in document_feats]
+        self.tokens_embeddings = [x['candidate_phrases_embeddings'] for x in document_feats]
+    
+    def flat_list(self, l):
+        return [x for ll in l for x in ll]
+    
+    def extract_summary(self,):
+        paired_scores = self.rank()
+        
+
+        rank_list_phrases = []
+        for candidate, paired_score in zip(self.candidate_phrases, paired_scores):
+            candidates = []
+            for i in range(len(candidate)):
+                phrase = candidate[i]
+                candidates.append([phrase, paired_score[i][0], paired_score[i][1]])
+            rank_list_phrases.append(candidates)
+
+
+        predicted_candidation = []
+        for i in range(len(rank_list_phrases)):
+            final_score = []
+            position_weight = 1 / (np.array(list(range(1, len(rank_list_phrases[i]) + 1))))
+            position_weight = np.exp(position_weight) / np.sum(np.exp(position_weight))
+            cnt = 0
+            for candidate, index, score in rank_list_phrases[i]:
+                final_score.append([candidate, score * position_weight[cnt]])
+                cnt += 1
+            final_score.sort(key = lambda x: x[1], reverse = True)
+            candidates = [x[0].strip() for x in final_score]
+            predicted_candidation.append(candidates)
+        return predicted_candidation
+
+
+    def pairdown(self, scores, pair_indice, length):
+        out_matrix = np.ones((length, length))
+        for pair in pair_indice:
+            out_matrix[pair[0][0]][pair[0][1]] = scores[pair[1]]
+            out_matrix[pair[0][1]][pair[0][0]] = scores[pair[1]]
+            
+        return out_matrix
+
+    def get_similarity_matrix(self, sentence_embeddings):
+        pairs = []
+        scores = []
+        cnt = 0
+        for i in range(len(sentence_embeddings)-1):
+            for j in range(i, len(sentence_embeddings)):
+                if type(sentence_embeddings[i]) == float or type(sentence_embeddings[j]) == float:
+                    scores.append(0)
+                else:
+                    scores.append(np.dot(sentence_embeddings[i], sentence_embeddings[j])) 
+
+                pairs.append(([i, j], cnt))
+                cnt += 1
+        return self.pairdown(scores, pairs, len(sentence_embeddings))
+
+    def compute_scores(self, similarity_matrix, edge_threshold=0):
+
+        forward_scores = [1e-10 for i in range(len(similarity_matrix))]
+        backward_scores = [1e-10 for i in range(len(similarity_matrix))]
+        edges = []
+        n = len(similarity_matrix)
+        alpha = self.alpha
+        for i in range(len(similarity_matrix)):
+            for j in range(i+1, len(similarity_matrix[i])):
+                edge_score = similarity_matrix[i][j]
+                # boundary_position_function
+                db_i = min(i, alpha * (n-i))
+                db_j = min(j, alpha * (n-j))
+                if edge_score > edge_threshold:
+                    if db_i < db_j:
+                        forward_scores[i] += edge_score
+                        backward_scores[j] += edge_score
+                        edges.append((i,j,edge_score))
+                    else:
+                        forward_scores[j] += edge_score
+                        backward_scores[i] += edge_score
+                        edges.append((j,i,edge_score))
+
+        return np.asarray(forward_scores), np.asarray(backward_scores), edges
+    
+    def _rank_part(self, similarity_matrix, doc_vector, candidate_phrases_embeddings):
+        min_score = np.min(similarity_matrix)
+        max_score = np.max(similarity_matrix)
+        threshold = min_score + self.beta * (max_score - min_score)
+        new_matrix = similarity_matrix - threshold
+        dist = []
+        for emb in candidate_phrases_embeddings:
+            if type(doc_vector) == float or type(emb) == float:
+                dist.append(0)
+            else:
+                dist.append(1/np.sum(np.abs(emb - doc_vector)))
+
+        forward_score, backward_score, _ = self.compute_scores(new_matrix)
+
+        paired_scores = []
+        for node in range(len(forward_score)):
+            paired_scores.append([node,  (self.lambda1 * forward_score[node] + self.lambda2 * backward_score[node]) * (dist[node])])
+
+        return paired_scores
+
+    def rank(self,):
+        
+        similarity_matrix = []
+        extracted_list = []
+        for embedded in self.tokens_embeddings:
+            similarity_matrix.append(self.get_similarity_matrix(embedded))
+        for matrix, doc_vector, candidate_phrases_embeddings in zip(similarity_matrix, self.doc_embeddings, self.tokens_embeddings):
+            extracted_list.append(self._rank_part(matrix, doc_vector, candidate_phrases_embeddings))
+        return extracted_list

japanese/tokenizer.py

@@ -0,0 +1,61 @@
+import MeCab
+import os
+
+
+def extract_keyphrase_candidates(text, tokenizer):
+    tagger = MeCab.Tagger()
+    tagger.parse("")
+
+    t = [to.split('\t') for to in tagger.parse(text).split('\n') if to]
+    t = [(to[0], to[1].split(',')[0]) for to in t if len(to) > 1]
+
+    keyphrase_candidates = []
+    phrase = []
+
+    tokens = []
+    idx = len(t) - 1
+    start_pos = -1
+    end_pos = -1
+    cnt = 0
+    phrase_set = set()
+
+    while idx >= 0:
+        while idx >= 0 and t[idx][1] != '名詞':
+            tokens.append(t[idx][0])
+            idx -= 1
+        
+        if idx >= 0 and t[idx][1] == '名詞':
+            tokens.append(t[idx][0])
+            end_pos = len(tokens)
+            phrase.append(t[idx][0])
+            idx -= 1
+
+        while idx >= 0 and t[idx][1] == '名詞':
+            tokens.append(t[idx][0])
+            phrase.append(t[idx][0])
+            idx -= 1
+
+        while idx >= 0 and t[idx][1] == '形容詞':
+            tokens.append(t[idx][0])
+            phrase.append(t[idx][0])
+            idx -= 1
+
+        if len(phrase) > 1:
+            start_pos = len(tokens)
+            keyphrase_candidates.append(('_'.join(phrase[::-1]), (len(t) - start_pos, len(t) - end_pos)))
+
+        phrase = []
+        start_pos = -1
+        end_pos = -1
+
+    while idx >= 0:
+        tokens.extend(tokenizer.tokenize(t[idx][0])[::-1])
+        idx -= 1
+
+
+    outputs = []
+    for keyphrase in keyphrase_candidates[::-1]:
+        if keyphrase[0] not in phrase_set:
+            outputs.append(keyphrase)
+            phrase_set.add(keyphrase[0])
+    return tokens[::-1], outputs