|
|
|
from doctest import OutputChecker |
|
import sys |
|
import torch |
|
import re |
|
import os |
|
import gradio as gr |
|
import requests |
|
|
|
|
|
|
|
|
|
from sentence_transformers import SentenceTransformer, util |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
model_sts = gr.Interface.load('huggingface/sentence-transformers/stsb-distilbert-base') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from transformers import GPT2Tokenizer, GPT2LMHeadModel |
|
import numpy as np |
|
import re |
|
|
|
def Sort_Tuple(tup): |
|
|
|
|
|
tup.sort(key = lambda x: x[1]) |
|
return tup[::-1] |
|
|
|
|
|
def softmax(x): |
|
exps = np.exp(x) |
|
return np.divide(exps, np.sum(exps)) |
|
|
|
|
|
|
|
|
|
|
|
model = gr.Interface.load('huggingface/distilgpt2', output_hidden_states = True, output_attentions = True) |
|
|
|
|
|
tokenizer = gr.Interface.load('huggingface/distilgpt2') |
|
|
|
|
|
|
|
|
|
|
|
def cloze_prob(text): |
|
|
|
whole_text_encoding = tokenizer.encode(text) |
|
|
|
text_list = text.split() |
|
stem = ' '.join(text_list[:-1]) |
|
stem_encoding = tokenizer.encode(stem) |
|
|
|
|
|
cw_encoding = whole_text_encoding[len(stem_encoding):] |
|
|
|
|
|
tokens_tensor = torch.tensor([whole_text_encoding]) |
|
|
|
with torch.no_grad(): |
|
outputs = model(tokens_tensor) |
|
predictions = outputs[0] |
|
|
|
logprobs = [] |
|
|
|
start = -1-len(cw_encoding) |
|
for j in range(start,-1,1): |
|
raw_output = [] |
|
for i in predictions[-1][j]: |
|
raw_output.append(i.item()) |
|
|
|
logprobs.append(np.log(softmax(raw_output))) |
|
|
|
|
|
|
|
|
|
|
|
conditional_probs = [] |
|
for cw,prob in zip(cw_encoding,logprobs): |
|
conditional_probs.append(prob[cw]) |
|
|
|
|
|
|
|
return np.exp(np.sum(conditional_probs)) |
|
|
|
|
|
|
|
|
|
|
|
def cos_sim(a, b): |
|
return np.inner(a, b) / (np.linalg.norm(a) * (np.linalg.norm(b))) |
|
|
|
|
|
|
|
def Visual_re_ranker(caption, visual_context_label, visual_context_prob): |
|
caption = caption |
|
visual_context_label= visual_context_label |
|
visual_context_prob = visual_context_prob |
|
caption_emb = model_sts.encode(caption, convert_to_tensor=True) |
|
visual_context_label_emb = model_sts.encode(visual_context_label, convert_to_tensor=True) |
|
|
|
|
|
sim = cosine_scores = util.pytorch_cos_sim(caption_emb, visual_context_label_emb) |
|
sim = sim.cpu().numpy() |
|
sim = str(sim)[1:-1] |
|
sim = str(sim)[1:-1] |
|
|
|
LM = cloze_prob(caption) |
|
|
|
score = pow(float(LM),pow((1-float(sim))/(1+ float(sim)),1-float(visual_context_prob))) |
|
|
|
|
|
|
|
return {"init hypothesis": float(LM)/1, "Visual Belief Revision": float(score)/1 } |
|
|
|
|
|
|
|
|
|
demo = gr.Interface( |
|
fn=Visual_re_ranker, |
|
description="Demo for Belief Revision based Caption Re-ranker with Visual Semantic Information", |
|
|
|
outputs=[gr.Textbox(value="Language Model Score") , gr.Textbox(value="Semantic Similarity Score"), gr.Textbox(value="Belief revision score via visual context")], |
|
|
|
) |
|
demo.launch() |
|
|