Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import openai
|
3 |
+
import torch
|
4 |
+
import tensorflow as tf
|
5 |
+
from transformers import AutoTokenizer, TFAutoModelForQuestionAnswering
|
6 |
+
import gradio as gr
|
7 |
+
import re
|
8 |
+
|
9 |
+
# Set your OpenAI API key here temporarily for testing
|
10 |
+
openai.api_key = os.getenv("OPENAI_API_KEY")
|
11 |
+
|
12 |
+
# Check if GPU is available and use it if possible
|
13 |
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
14 |
+
|
15 |
+
# Load the English models and tokenizers
|
16 |
+
qa_model_name_v1 = 'salsarra/ConfliBERT-QA'
|
17 |
+
qa_model_v1 = TFAutoModelForQuestionAnswering.from_pretrained(qa_model_name_v1)
|
18 |
+
qa_tokenizer_v1 = AutoTokenizer.from_pretrained(qa_model_name_v1)
|
19 |
+
|
20 |
+
bert_model_name_v1 = 'salsarra/BERT-base-cased-SQuAD-v1'
|
21 |
+
bert_qa_model_v1 = TFAutoModelForQuestionAnswering.from_pretrained(bert_model_name_v1)
|
22 |
+
bert_qa_tokenizer_v1 = AutoTokenizer.from_pretrained(bert_model_name_v1)
|
23 |
+
|
24 |
+
# Load Spanish models and tokenizers
|
25 |
+
confli_model_spanish_name = 'salsarra/ConfliBERT-Spanish-Beto-Cased-NewsQA'
|
26 |
+
confli_model_spanish = TFAutoModelForQuestionAnswering.from_pretrained(confli_model_spanish_name)
|
27 |
+
confli_tokenizer_spanish = AutoTokenizer.from_pretrained(confli_model_spanish_name)
|
28 |
+
|
29 |
+
beto_model_spanish_name = 'salsarra/Beto-Spanish-Cased-NewsQA'
|
30 |
+
beto_model_spanish = TFAutoModelForQuestionAnswering.from_pretrained(beto_model_spanish_name)
|
31 |
+
beto_tokenizer_spanish = AutoTokenizer.from_pretrained(beto_model_spanish_name)
|
32 |
+
|
33 |
+
# Load the additional Spanish models
|
34 |
+
confli_sqac_model_spanish = 'salsarra/ConfliBERT-Spanish-Beto-Cased-SQAC'
|
35 |
+
confli_sqac_model_spanish_qa = TFAutoModelForQuestionAnswering.from_pretrained(confli_sqac_model_spanish)
|
36 |
+
confli_sqac_tokenizer_spanish = AutoTokenizer.from_pretrained(confli_sqac_model_spanish)
|
37 |
+
|
38 |
+
beto_sqac_model_spanish = 'salsarra/Beto-Spanish-Cased-SQAC'
|
39 |
+
beto_sqac_model_spanish_qa = TFAutoModelForQuestionAnswering.from_pretrained(beto_sqac_model_spanish)
|
40 |
+
beto_sqac_tokenizer_spanish = AutoTokenizer.from_pretrained(beto_sqac_model_spanish)
|
41 |
+
|
42 |
+
# Load specified ConfliBERT Arabic models
|
43 |
+
confli_model_arabic_1_name = 'salsarra/ConfliBERT-Arabic-Arabertv2-QA-MLQA'
|
44 |
+
confli_model_arabic_1 = TFAutoModelForQuestionAnswering.from_pretrained(confli_model_arabic_1_name)
|
45 |
+
confli_tokenizer_arabic_1 = AutoTokenizer.from_pretrained(confli_model_arabic_1_name)
|
46 |
+
|
47 |
+
confli_model_arabic_2_name = 'salsarra/ConfliBERT-Arabic-Arabertv2-QA-XQUAD'
|
48 |
+
confli_model_arabic_2 = TFAutoModelForQuestionAnswering.from_pretrained(confli_model_arabic_2_name)
|
49 |
+
confli_tokenizer_arabic_2 = AutoTokenizer.from_pretrained(confli_model_arabic_2_name)
|
50 |
+
|
51 |
+
confli_model_arabic_3_name = 'salsarra/ConfliBERT-Arabic-Arabertv2-QA-ARCD'
|
52 |
+
confli_model_arabic_3 = TFAutoModelForQuestionAnswering.from_pretrained(confli_model_arabic_3_name)
|
53 |
+
confli_tokenizer_arabic_3 = AutoTokenizer.from_pretrained(confli_model_arabic_3_name)
|
54 |
+
|
55 |
+
# Load specified BERT Arabic models (AraBERTv2)
|
56 |
+
bert_model_arabic_1_name = 'salsarra/Bert-Base-Arabertv2-QA-MLQA'
|
57 |
+
bert_qa_model_arabic_1 = TFAutoModelForQuestionAnswering.from_pretrained(bert_model_arabic_1_name)
|
58 |
+
bert_qa_tokenizer_arabic_1 = AutoTokenizer.from_pretrained(bert_model_arabic_1_name)
|
59 |
+
|
60 |
+
bert_model_arabic_2_name = 'salsarra/Bert-Base-Arabertv2-QA-XQUAD'
|
61 |
+
bert_qa_model_arabic_2 = TFAutoModelForQuestionAnswering.from_pretrained(bert_model_arabic_2_name)
|
62 |
+
bert_qa_tokenizer_arabic_2 = AutoTokenizer.from_pretrained(bert_model_arabic_2_name)
|
63 |
+
|
64 |
+
bert_model_arabic_3_name = 'salsarra/Bert-Base-Arabertv2-QA-ARCD'
|
65 |
+
bert_qa_model_arabic_3 = TFAutoModelForQuestionAnswering.from_pretrained(bert_model_arabic_3_name)
|
66 |
+
bert_qa_tokenizer_arabic_3 = AutoTokenizer.from_pretrained(bert_model_arabic_3_name)
|
67 |
+
|
68 |
+
|
69 |
+
|
70 |
+
# Define error handling to separate input size errors from other issues
|
71 |
+
def handle_error_message(e, default_limit=512):
|
72 |
+
error_message = str(e)
|
73 |
+
pattern = re.compile(r"The size of tensor a \\((\\d+)\\) must match the size of tensor b \\((\\d+)\\)")
|
74 |
+
match = pattern.search(error_message)
|
75 |
+
if match:
|
76 |
+
number_1, number_2 = match.groups()
|
77 |
+
return f"<span style='color: red; font-weight: bold;'>Error: Text Input is over limit where inserted text size {number_1} is larger than model limits of {number_2}</span>"
|
78 |
+
|
79 |
+
pattern_qa = re.compile(r"indices\\[0,(\\d+)\\] = \\d+ is not in \\[0, (\\d+)\\)")
|
80 |
+
match_qa = pattern_qa.search(error_message)
|
81 |
+
if match_qa:
|
82 |
+
number_1, number_2 = match_qa.groups()
|
83 |
+
return f"<span style='color: red; font-weight: bold;'>Error: Text Input is over limit where inserted text size {number_1} is larger than model limits of {number_2}</span>"
|
84 |
+
|
85 |
+
return f"<span style='color: red; font-weight: bold;'>Error: {error_message}</span>"
|
86 |
+
|
87 |
+
# Define question_answering_v1 for ConfliBERT English with truncation=True
|
88 |
+
def question_answering_v1(context, question):
|
89 |
+
try:
|
90 |
+
inputs = qa_tokenizer_v1(question, context, return_tensors='tf', truncation=True)
|
91 |
+
outputs = qa_model_v1(inputs)
|
92 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
93 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
94 |
+
answer = qa_tokenizer_v1.convert_tokens_to_string(
|
95 |
+
qa_tokenizer_v1.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
96 |
+
)
|
97 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
98 |
+
except Exception as e:
|
99 |
+
return handle_error_message(e)
|
100 |
+
|
101 |
+
# Define bert_question_answering_v1 for BERT English with truncation=True
|
102 |
+
def bert_question_answering_v1(context, question):
|
103 |
+
try:
|
104 |
+
inputs = bert_qa_tokenizer_v1(question, context, return_tensors='tf', truncation=True)
|
105 |
+
outputs = bert_qa_model_v1(inputs)
|
106 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
107 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
108 |
+
answer = bert_qa_tokenizer_v1.convert_tokens_to_string(
|
109 |
+
bert_qa_tokenizer_v1.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
110 |
+
)
|
111 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
112 |
+
except Exception as e:
|
113 |
+
return handle_error_message(e)
|
114 |
+
|
115 |
+
# Define question_answering_spanish for ConfliBERT-Spanish-Beto-Cased-NewsQA
|
116 |
+
def question_answering_spanish(context, question):
|
117 |
+
try:
|
118 |
+
inputs = confli_tokenizer_spanish.encode_plus(question, context, return_tensors='tf', truncation=True)
|
119 |
+
outputs = confli_model_spanish(inputs)
|
120 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
121 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
122 |
+
answer = confli_tokenizer_spanish.convert_tokens_to_string(
|
123 |
+
confli_tokenizer_spanish.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
124 |
+
)
|
125 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
126 |
+
except Exception as e:
|
127 |
+
return handle_error_message(e)
|
128 |
+
|
129 |
+
# Define beto_question_answering_spanish for Beto-Spanish-Cased-NewsQA
|
130 |
+
def beto_question_answering_spanish(context, question):
|
131 |
+
try:
|
132 |
+
inputs = beto_tokenizer_spanish.encode_plus(question, context, return_tensors='tf', truncation=True)
|
133 |
+
outputs = beto_model_spanish(inputs)
|
134 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
135 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
136 |
+
answer = beto_tokenizer_spanish.convert_tokens_to_string(
|
137 |
+
beto_tokenizer_spanish.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
138 |
+
)
|
139 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
140 |
+
except Exception as e:
|
141 |
+
return handle_error_message(e)
|
142 |
+
|
143 |
+
# Define confli_sqac_question_answering_spanish for ConfliBERT-Spanish-Beto-Cased-SQAC
|
144 |
+
def confli_sqac_question_answering_spanish(context, question):
|
145 |
+
inputs = confli_sqac_tokenizer_spanish.encode_plus(question, context, return_tensors="tf", truncation=True)
|
146 |
+
outputs = confli_sqac_model_spanish_qa(inputs)
|
147 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
148 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
149 |
+
answer = confli_sqac_tokenizer_spanish.convert_tokens_to_string(
|
150 |
+
confli_sqac_tokenizer_spanish.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
151 |
+
)
|
152 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
153 |
+
|
154 |
+
# Define beto_sqac_question_answering_spanish for Beto-Spanish-Cased-SQAC
|
155 |
+
def beto_sqac_question_answering_spanish(context, question):
|
156 |
+
inputs = beto_sqac_tokenizer_spanish.encode_plus(question, context, return_tensors="tf", truncation=True)
|
157 |
+
outputs = beto_sqac_model_spanish_qa(inputs)
|
158 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
159 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
160 |
+
answer = beto_sqac_tokenizer_spanish.convert_tokens_to_string(
|
161 |
+
beto_sqac_tokenizer_spanish.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
162 |
+
)
|
163 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
164 |
+
|
165 |
+
# ConfliBERT Arabic Model 1
|
166 |
+
def question_answering_confli_arabic_1(context, question):
|
167 |
+
try:
|
168 |
+
inputs = confli_tokenizer_arabic_1(question, context, return_tensors='tf', truncation=True)
|
169 |
+
outputs = confli_model_arabic_1(inputs)
|
170 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
171 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
172 |
+
answer = confli_tokenizer_arabic_1.convert_tokens_to_string(
|
173 |
+
confli_tokenizer_arabic_1.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
174 |
+
)
|
175 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
176 |
+
except Exception as e:
|
177 |
+
return handle_error_message(e)
|
178 |
+
|
179 |
+
# Add functions for other ConfliBERT and BERT models similarly
|
180 |
+
|
181 |
+
def question_answering_confli_arabic_2(context, question):
|
182 |
+
inputs = confli_tokenizer_arabic_2(question, context, return_tensors='tf', truncation=True)
|
183 |
+
outputs = confli_model_arabic_2(inputs)
|
184 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
185 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
186 |
+
answer = confli_tokenizer_arabic_2.convert_tokens_to_string(
|
187 |
+
confli_tokenizer_arabic_2.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
188 |
+
)
|
189 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
190 |
+
|
191 |
+
def question_answering_confli_arabic_3(context, question):
|
192 |
+
inputs = confli_tokenizer_arabic_3(question, context, return_tensors='tf', truncation=True)
|
193 |
+
outputs = confli_model_arabic_3(inputs)
|
194 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
195 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
196 |
+
answer = confli_tokenizer_arabic_3.convert_tokens_to_string(
|
197 |
+
confli_tokenizer_arabic_3.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
198 |
+
)
|
199 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
200 |
+
|
201 |
+
# Similarly, for BERT models
|
202 |
+
def question_answering_bert_arabic_1(context, question):
|
203 |
+
inputs = bert_qa_tokenizer_arabic_1(question, context, return_tensors='tf', truncation=True)
|
204 |
+
outputs = bert_qa_model_arabic_1(inputs)
|
205 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
206 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
207 |
+
answer = bert_qa_tokenizer_arabic_1.convert_tokens_to_string(
|
208 |
+
bert_qa_tokenizer_arabic_1.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
209 |
+
)
|
210 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
211 |
+
|
212 |
+
# BERT Arabic Model 2 (XQUAD)
|
213 |
+
def question_answering_bert_arabic_2(context, question):
|
214 |
+
try:
|
215 |
+
inputs = bert_qa_tokenizer_arabic_2(question, context, return_tensors='tf', truncation=True)
|
216 |
+
outputs = bert_qa_model_arabic_2(inputs)
|
217 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
218 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
219 |
+
answer = bert_qa_tokenizer_arabic_2.convert_tokens_to_string(
|
220 |
+
bert_qa_tokenizer_arabic_2.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
221 |
+
)
|
222 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
223 |
+
except Exception as e:
|
224 |
+
return handle_error_message(e)
|
225 |
+
|
226 |
+
# BERT Arabic Model 3 (ARCD)
|
227 |
+
def question_answering_bert_arabic_3(context, question):
|
228 |
+
try:
|
229 |
+
inputs = bert_qa_tokenizer_arabic_3(question, context, return_tensors='tf', truncation=True)
|
230 |
+
outputs = bert_qa_model_arabic_3(inputs)
|
231 |
+
answer_start = tf.argmax(outputs.start_logits, axis=1).numpy()[0]
|
232 |
+
answer_end = tf.argmax(outputs.end_logits, axis=1).numpy()[0] + 1
|
233 |
+
answer = bert_qa_tokenizer_arabic_3.convert_tokens_to_string(
|
234 |
+
bert_qa_tokenizer_arabic_3.convert_ids_to_tokens(inputs['input_ids'].numpy()[0][answer_start:answer_end])
|
235 |
+
)
|
236 |
+
return f"<span style='font-weight: bold;'>{answer}</span>"
|
237 |
+
except Exception as e:
|
238 |
+
return handle_error_message(e)
|
239 |
+
|
240 |
+
|
241 |
+
|
242 |
+
# Define a function to get ChatGPT's answer in English using the latest OpenAI API
|
243 |
+
def chatgpt_question_answering(context, question):
|
244 |
+
messages = [
|
245 |
+
{"role": "system", "content": "You are a helpful assistant. Only answer based on the provided context. Do not use any external knowledge."},
|
246 |
+
{"role": "user", "content": f"Context: {context}\nQuestion: {question}\nAnswer:"}
|
247 |
+
]
|
248 |
+
response = openai.ChatCompletion.create(
|
249 |
+
model="gpt-3.5-turbo",
|
250 |
+
messages=messages,
|
251 |
+
max_tokens=500
|
252 |
+
)
|
253 |
+
return response['choices'][0]['message']['content'].strip()
|
254 |
+
|
255 |
+
# Define a function to get ChatGPT's answer in Spanish using the latest OpenAI API
|
256 |
+
def chatgpt_question_answering_spanish(context, question):
|
257 |
+
messages = [
|
258 |
+
{"role": "system", "content": "You are a helpful assistant that responds in Spanish. Only answer based on the provided context. Do not use any external knowledge."},
|
259 |
+
{"role": "user", "content": f"Contexto: {context}\nPregunta: {question}\nRespuesta:"}
|
260 |
+
]
|
261 |
+
response = openai.ChatCompletion.create(
|
262 |
+
model="gpt-3.5-turbo",
|
263 |
+
messages=messages,
|
264 |
+
max_tokens=500
|
265 |
+
)
|
266 |
+
return response['choices'][0]['message']['content'].strip()
|
267 |
+
|
268 |
+
# Define a function to get ChatGPT's answer in Arabic using the latest OpenAI API
|
269 |
+
def chatgpt_question_answering_arabic(context, question):
|
270 |
+
messages = [
|
271 |
+
{"role": "system", "content": "أنت مساعد ذكي ومفيد. أجب فقط بناءً على النص المُعطى في السياق. لا تستخدم أي معرفة خارجية."},
|
272 |
+
{"role": "user", "content": f"السياق: {context}\nالسؤال: {question}\nالإجابة:"}
|
273 |
+
]
|
274 |
+
response = openai.ChatCompletion.create(
|
275 |
+
model="gpt-3.5-turbo",
|
276 |
+
messages=messages,
|
277 |
+
max_tokens=500
|
278 |
+
)
|
279 |
+
return response['choices'][0]['message']['content'].strip()
|
280 |
+
|
281 |
+
|
282 |
+
# Main comparison function with language selection
|
283 |
+
def compare_question_answering(language, context, question):
|
284 |
+
if language == "English":
|
285 |
+
confli_answer_v1 = question_answering_v1(context, question)
|
286 |
+
bert_answer_v1 = bert_question_answering_v1(context, question)
|
287 |
+
chatgpt_answer = chatgpt_question_answering(context, question)
|
288 |
+
return f"""
|
289 |
+
<div>
|
290 |
+
<h2 style='color: #2e8b57; font-weight: bold;'>Answers:</h2>
|
291 |
+
</div><br>
|
292 |
+
<div>
|
293 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT English:</strong><br><span style='font-weight: bold;'>{confli_answer_v1}</span></div><br>
|
294 |
+
<div>
|
295 |
+
<strong style='color: orange; font-weight: bold;'>BERT:</strong><br><span style='font-weight: bold;'>{bert_answer_v1}</span>
|
296 |
+
</div><br>
|
297 |
+
<div>
|
298 |
+
<strong style='color: #74AA9C; font-weight: bold;'>ChatGPT:</strong><br><span style='font-weight: bold;'>{chatgpt_answer}</span>
|
299 |
+
</div><br>
|
300 |
+
<div>
|
301 |
+
<strong>Model Information:</strong><br>
|
302 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-QA' target='_blank'>ConfliBERT English (Cont-Cased-SQuAD-v1)</a><br>
|
303 |
+
<a href='https://huggingface.co/salsarra/BERT-base-cased-SQuAD-v1' target='_blank'>BERT (Base-Cased-SQuAD-v1)</a><br>
|
304 |
+
<a href='https://platform.openai.com/docs/models/gpt-3-5' target='_blank'>ChatGPT (GPT-3.5 Turbo)</a><br></p>
|
305 |
+
</div>
|
306 |
+
"""
|
307 |
+
elif language == "Spanish":
|
308 |
+
confli_answer_spanish = question_answering_spanish(context, question)
|
309 |
+
beto_answer_spanish = beto_question_answering_spanish(context, question)
|
310 |
+
confli_sqac_answer_spanish = confli_sqac_question_answering_spanish(context, question)
|
311 |
+
beto_sqac_answer_spanish = beto_sqac_question_answering_spanish(context, question)
|
312 |
+
chatgpt_answer_spanish = chatgpt_question_answering_spanish(context, question)
|
313 |
+
return f"""
|
314 |
+
<div>
|
315 |
+
<h2 style='color: #2e8b57; font-weight: bold;'>Answers:</h2>
|
316 |
+
</div><br>
|
317 |
+
<div>
|
318 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT Spanish:</strong><br><span style='font-weight: bold;'>{confli_answer_spanish}</span></div><br>
|
319 |
+
<div>
|
320 |
+
<strong style='color: orange; font-weight: bold;'>BERT Spanish (BETO):</strong><br><span style='font-weight: bold;'>{beto_answer_spanish}</span>
|
321 |
+
</div><br>
|
322 |
+
<div>
|
323 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT Spanish:</strong><br><span style='font-weight: bold;'>{confli_sqac_answer_spanish}</span>
|
324 |
+
</div><br>
|
325 |
+
<div>
|
326 |
+
<strong style='color: orange; font-weight: bold;'>BERT Spanish (BETO):</strong><br><span style='font-weight: bold;'>{beto_sqac_answer_spanish}</span>
|
327 |
+
</div><br>
|
328 |
+
<div>
|
329 |
+
<strong style='color: #74AA9C; font-weight: bold;'>ChatGPT:</strong><br><span style='font-weight: bold;'>{chatgpt_answer_spanish}</span>
|
330 |
+
</div><br>
|
331 |
+
<div>
|
332 |
+
<strong>Model Information:</strong><br>
|
333 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-Spanish-Beto-Cased-NewsQA' target='_blank'>ConfliBERT Spanish (Beto-Cased-NewsQA)</a><br>
|
334 |
+
<a href='https://huggingface.co/salsarra/Beto-Spanish-Cased-NewsQA' target='_blank'>BERT Spanish (BETO) (Beto-Spanish-Cased-NewsQA)</a><br>
|
335 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-Spanish-Beto-Cased-SQAC' target='_blank'>ConfliBERT Spanish (Beto-Cased-SQAC)</a><br>
|
336 |
+
<a href='https://huggingface.co/salsarra/Beto-Spanish-Cased-SQAC' target='_blank'>BERT Spanish (BETO) (Beto-Cased-SQAC)</a><br>
|
337 |
+
<a href='https://platform.openai.com/docs/models/gpt-3-5' target='_blank'>ChatGPT (GPT-3.5 Turbo)</a><br></p>
|
338 |
+
</div>
|
339 |
+
"""
|
340 |
+
elif language == "Arabic":
|
341 |
+
confli_answer_arabic_1 = question_answering_confli_arabic_1(context, question)
|
342 |
+
bert_answer_arabic_1 = question_answering_bert_arabic_1(context, question)
|
343 |
+
confli_answer_arabic_2 = question_answering_confli_arabic_2(context, question)
|
344 |
+
bert_answer_arabic_2 = question_answering_bert_arabic_2(context, question)
|
345 |
+
confli_answer_arabic_3 = question_answering_confli_arabic_3(context, question)
|
346 |
+
bert_answer_arabic_3 = question_answering_bert_arabic_3(context, question)
|
347 |
+
chatgpt_answer_arabic = chatgpt_question_answering_arabic(context, question)
|
348 |
+
|
349 |
+
return f"""
|
350 |
+
<div dir="rtl" style="text-align: right;">
|
351 |
+
<h2 style='color: #2e8b57; font-weight: bold;'>الإجابات:</h2>
|
352 |
+
</div><br>
|
353 |
+
<div dir="rtl" style="text-align: right;">
|
354 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT Arabic (MLQA):</strong><br>
|
355 |
+
{confli_answer_arabic_1}
|
356 |
+
</div><br>
|
357 |
+
<div dir="rtl" style="text-align: right;">
|
358 |
+
<strong style='color: orange; font-weight: bold;'>BERT Arabic (MLQA):</strong><br>
|
359 |
+
{bert_answer_arabic_1}
|
360 |
+
</div><br>
|
361 |
+
<div dir="rtl" style="text-align: right;">
|
362 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT Arabic (XQUAD):</strong><br>
|
363 |
+
{confli_answer_arabic_2}
|
364 |
+
</div><br>
|
365 |
+
<div dir="rtl" style="text-align: right;">
|
366 |
+
<strong style='color: orange; font-weight: bold;'>BERT Arabic (XQUAD):</strong><br>
|
367 |
+
{bert_answer_arabic_2}
|
368 |
+
</div><br>
|
369 |
+
<div dir="rtl" style="text-align: right;">
|
370 |
+
<strong style='color: green; font-weight: bold;'>ConfliBERT Arabic (ARCD):</strong><br>
|
371 |
+
{confli_answer_arabic_3}
|
372 |
+
</div><br>
|
373 |
+
<div dir="rtl" style="text-align: right;">
|
374 |
+
<strong style='color: orange; font-weight: bold;'>BERT Arabic (ARCD):</strong><br>
|
375 |
+
{bert_answer_arabic_3}
|
376 |
+
</div><br>
|
377 |
+
<div dir="rtl" style="text-align: right;">
|
378 |
+
<strong style='color: #74AA9C; font-weight: bold;'>ChatGPT:</strong><br>
|
379 |
+
{chatgpt_answer_arabic}
|
380 |
+
</div><br>
|
381 |
+
<div dir="rtl" style="text-align: right;">
|
382 |
+
<strong>Model Information:</strong><br>
|
383 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-Arabic-Arabertv2-QA-MLQA' target='_blank'>ConfliBERT Arabic (MLQA)</a><br>
|
384 |
+
<a href='https://huggingface.co/salsarra/Bert-Base-Arabertv2-QA-MLQA' target='_blank'>BERT Arabic (MLQA)</a><br>
|
385 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-Arabic-Arabertv2-QA-XQUAD' target='_blank'>ConfliBERT Arabic (XQUAD)</a><br>
|
386 |
+
<a href='https://huggingface.co/salsarra/Bert-Base-Arabertv2-QA-XQUAD' target='_blank'>BERT Arabic (XQUAD)</a><br>
|
387 |
+
<a href='https://huggingface.co/salsarra/ConfliBERT-Arabic-Arabertv2-QA-ARCD' target='_blank'>ConfliBERT Arabic (ARCD)</a><br>
|
388 |
+
<a href='https://huggingface.co/salsarra/Bert-Base-Arabertv2-QA-ARCD' target='_blank'>BERT Arabic (ARCD)</a><br>
|
389 |
+
<a href='https://platform.openai.com/docs/models/gpt-3-5' target='_blank'>ChatGPT (GPT-3.5 Turbo)</a><br>
|
390 |
+
</div>
|
391 |
+
"""
|
392 |
+
|
393 |
+
|
394 |
+
|
395 |
+
# Gradio interface setup
|
396 |
+
with gr.Blocks(css="""
|
397 |
+
body {
|
398 |
+
background-color: #f0f8ff;
|
399 |
+
font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
|
400 |
+
}
|
401 |
+
h1, h1 a {
|
402 |
+
color: #2e8b57;
|
403 |
+
text-align: center;
|
404 |
+
font-size: 2em;
|
405 |
+
text-decoration: none;
|
406 |
+
}
|
407 |
+
h1 a:hover {
|
408 |
+
color: #ff8c00;
|
409 |
+
}
|
410 |
+
h2 {
|
411 |
+
color: #ff8c00;
|
412 |
+
text-align: center;
|
413 |
+
font-size: 1.5em;
|
414 |
+
}
|
415 |
+
""") as demo:
|
416 |
+
|
417 |
+
gr.Markdown("# [ConfliBERT-QA](https://eventdata.utdallas.edu/conflibert/)", elem_id="title")
|
418 |
+
gr.Markdown("Compare answers between ConfliBERT, BERT, and ChatGPT for English, and ConfliBERT, BETO, ConfliBERT-SQAC, Beto-SQAC, and ChatGPT for Spanish.")
|
419 |
+
|
420 |
+
language = gr.Dropdown(choices=["English", "Spanish", "Arabic"], label="Select Language")
|
421 |
+
context = gr.Textbox(lines=5, placeholder="Enter the context here...", label="Context")
|
422 |
+
question = gr.Textbox(lines=2, placeholder="Enter your question here...", label="Question")
|
423 |
+
output = gr.HTML(label="Output")
|
424 |
+
|
425 |
+
with gr.Row():
|
426 |
+
clear_btn = gr.Button("Clear")
|
427 |
+
submit_btn = gr.Button("Submit")
|
428 |
+
|
429 |
+
submit_btn.click(fn=compare_question_answering, inputs=[language, context, question], outputs=output)
|
430 |
+
clear_btn.click(fn=lambda: ("", "", "", ""), inputs=[], outputs=[language, context, question, output])
|
431 |
+
|
432 |
+
gr.Markdown("""
|
433 |
+
<div style="text-align: center; margin-top: 20px;">
|
434 |
+
Built by: <a href="https://www.linkedin.com/in/sultan-alsarra-phd-56977a63/" target="_blank">Sultan Alsarra</a>
|
435 |
+
</div>
|
436 |
+
""")
|
437 |
+
|
438 |
+
demo.launch(share=True)
|