merged two blocks

.gitignore

@@ -1,2 +1 @@
-__pycache__/
-copy.bat
+__pycache__/

inference.py

@@ -1,6 +1,6 @@
 import torch
-import requests
 from transformers import BertForSequenceClassification, AutoTokenizer
+from transformers import PegasusForConditionalGeneration, PegasusTokenizer
 
 # path_emo = 'Djacon/rubert-tiny2-russian-emotion-detection'
 path_emo = './models/emotion_detection/'
@@ -29,21 +29,16 @@ def predict_emotions(text: str) -> str:
                                                     key=lambda x: -x[1]))
 
 
-def predict_summarization(text: str) -> str:
-    url = "https://www.editpad.org/tool/tool/summarizingTool"
-    headers = {"Content-Type": "application/x-www-form-urlencoded"}
-
-    data = {"text": text, "modd": 1, "captha": 0, 'percnt': 10,
-            "max-length": len(text) / 10}
+path_sum = './models/summarizer/'
+model_sum = PegasusForConditionalGeneration.from_pretrained(path_sum)
+tokenizer_sum = PegasusTokenizer.from_pretrained(path_sum)
 
-    response = requests.post(url, headers=headers, data=data)
 
-    if response.status_code != 200:
-        return 'err'
-
-    response_json = response.json()
-    print(len(response_json.get('content')), len(text))
-    return response_json.get("content", 'err')
+def predict_summarization(text: str) -> str:
+    batch = tokenizer_sum([text], truncation=True, padding="longest",
+                          return_tensors="pt")
+    translated = model_sum.generate(**batch)
+    return tokenizer_sum.batch_decode(translated, skip_special_tokens=True)[0]
 
 
 def test():

models/summarizer/config.json

@@ -0,0 +1,57 @@
+{
+  "_name_or_path": "./",
+  "activation_dropout": 0.1,
+  "activation_function": "relu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "PegasusForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 16,
+  "decoder_start_token_id": 0,
+  "do_blenderbot_90_layernorm": false,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 16,
+  "eos_token_id": 1,
+  "extra_pos_embeddings": 0,
+  "force_bos_token_to_be_generated": false,
+  "forced_eos_token_id": 1,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "length_penalty": 0.6,
+  "max_length": 64,
+  "max_position_embeddings": 512,
+  "model_type": "pegasus",
+  "normalize_before": true,
+  "normalize_embedding": false,
+  "num_beams": 8,
+  "num_hidden_layers": 16,
+  "pad_token_id": 0,
+  "scale_embedding": true,
+  "static_position_embeddings": true,
+  "transformers_version": "4.11.0.dev0",
+  "use_cache": true,
+  "vocab_size": 96103
+}

models/summarizer/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "forced_eos_token_id": 1,
+  "length_penalty": 0.6,
+  "max_length": 64,
+  "num_beams": 8,
+  "pad_token_id": 0,
+  "transformers_version": "4.27.0.dev0"
+}

models/summarizer/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19169fbf6d5bf3b3c713cb933e40a5fa22ffdb6e0d7628309e2deabc5978e59
+size 2275329241

models/summarizer/special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

models/summarizer/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0015189ef36359283fec8b93cf6d9ce51bca37eb1101defc68a53b394913b96c
+size 1912529

models/summarizer/tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 512, "special_tokens_map_file": null, "full_tokenizer_file": null}

static/emotion_detection.html

@@ -226,9 +226,16 @@
 
                     <div class="mt-6">
                         <div class="w-full p-4 bg-white rounded-lg xl:p-6">
-                            <h1 class="text-lg font-medium text-gray-700 capitalize sm:text-xl md:text-2xl">Upload text</h1>
-
-                            <form id='sum-form' class="mt-6 space-y-5" onsubmit="return false;">
+                            <ul class="flex text-sm text-center text-gray-500 divide-x divide-gray-200">
+                                <li class="w-full">
+                                    <button id="show-original" class="bg-gray-100 inline-block w-full p-2 text-gray-900 focus:ring-4 focus:ring-blue-300 focus:outline-none">Original</button>
+                                </li>
+                                <li class="w-full">
+                                    <button id="show-summary" class="inline-block w-full p-2 hover:text-gray-700 hover:bg-gray-50 focus:ring-4 focus:ring-blue-300 focus:outline-none">Summary</button>
+                                </li>
+                            </ul>
+
+                            <div id='sum-original' class="mt-6 space-y-5" onsubmit="return false;">
                                 <div id="sum-text-div">
                                     <textarea id="sum-text-input" required class="block w-full px-4 py-2.5 mt-2 text-gray-600 placeholder-gray-400 bg-white border border-gray-200 rounded-md focus:border-indigo-400 focus:outline-none focus:ring focus:ring-indigo-300 focus:ring-opacity-40"
                                     style="min-height: 250px;" placeholder="Enter or upload your text and press &quot;Detect&quot;"></textarea>
@@ -253,13 +260,9 @@
                                         Detect
                                     </button>
                                 </div>
-                            </form>
-                        </div>
-
-                        <div class="w-full p-4 mt-8 bg-white rounded-lg xl:p-6">
-                            <h1 class="text-lg font-medium text-gray-700 capitalize sm:text-xl md:text-2xl">Result</h1>
+                            </div>
 
-                            <div class="md:flex gap-8">
+                            <div id="sum-summary" class='hidden mt-6 space-y-5'>
                                 <div class="hidden w-full md:w-1/2 grid-cols-2 mt-6">
                                     <label for="description"
                                         class="block text-sm text-gray-700 capitalize">Extracted Text</label>
@@ -268,8 +271,6 @@
                                 </div>
 
                                 <div class="w-full md:w-1/2 grid-cols-2 mt-6">
-                                    <label for="description"
-                                        class="block text-sm text-gray-700 capitalize">Emotion Prediction</label>
                                     <textarea id="summarized-text"
                                         class="minh-text block w-full px-4 py-2.5 mt-2 text-gray-600 placeholder-gray-400 bg-white border border-gray-200 rounded-md focus:border-indigo-400 focus:outline-none focus:ring focus:ring-indigo-300 focus:ring-opacity-40"
                                         style="min-height: 250px;"></textarea>

static/js/detection.js

@@ -1,7 +1,6 @@
 // Form Divs
-const form = document.getElementById('sum-form');
-const sumText  = document.getElementById('sum-text-div');
-const sumFile  = document.getElementById('sum-file-div')
+const sumText = document.getElementById('sum-text-div');
+const sumFile = document.getElementById('sum-file-div')
 
 // Form Data
 const sumTextInput  = document.getElementById('sum-text-input');
@@ -17,6 +16,12 @@ const summaryText = document.getElementById('summarized-text');
 // Word Counter
 const wordsCount = document.getElementById('word-counter');
 
+// Tabs
+const original = document.getElementById('sum-original');
+const summary  = document.getElementById('sum-summary');
+const showOriginal = document.getElementById('show-original');
+const showSummary  = document.getElementById('show-summary');
+
 const MAX_SIZE = 20000;
 
 
@@ -87,6 +92,8 @@ async function summarize(event) {
 
     sumError.classList.add('hidden');
 
+    _show_summary();
+
     // Here we can finally summarize data
     summaryText.value = 'Please wait...';
     extractText.value = sumTextInput.value.trim().slice(0, MAX_SIZE);
@@ -106,6 +113,22 @@ function _update_counter() {
     wordsCount.innerHTML = `Words: ${text.split(/\s+/).length} | Chars: ${text.length}`
 }
 
+function _show_summary() {
+    showOriginal.classList.remove('bg-gray-100');
+    showSummary.classList.add('bg-gray-100');
+
+    summary.classList.remove('hidden');
+    original.classList.add('hidden');
+}
+
+function _show_original() {
+    showOriginal.classList.add('bg-gray-100');
+    showSummary.classList.remove('bg-gray-100');
+
+    original.classList.remove('hidden');
+    summary.classList.add('hidden');
+}
+
 document.addEventListener('DOMContentLoaded', function () {
     var submitButton = document.getElementById('submit');
     submitButton.addEventListener('click', summarize);
@@ -133,4 +156,7 @@ document.addEventListener('DOMContentLoaded', function () {
     });
 
     sumTextInput.addEventListener('input', _update_counter);
+
+    showSummary.addEventListener('click', _show_summary);
+    showOriginal.addEventListener('click', _show_original);
 });

static/js/summarizer.js

@@ -1,5 +1,4 @@
 // Form Divs
-const form = document.getElementById('sum-form');
 const sumText  = document.getElementById('sum-text-div');
 const sumFile  = document.getElementById('sum-file-div')
 const sumVideo = document.getElementById('sum-video-div');
@@ -20,6 +19,12 @@ const summaryText = document.getElementById('summarized-text');
 // Word Counter
 const wordsCount = document.getElementById('word-counter');
 
+// Tabs
+const original = document.getElementById('sum-original');
+const summary  = document.getElementById('sum-summary');
+const showOriginal = document.getElementById('show-original');
+const showSummary  = document.getElementById('show-summary');
+
 const MAX_SIZE = 20000;
 
 
@@ -34,10 +39,11 @@ function _summarize() {
         if (xhr.readyState === 4 && xhr.status === 200) {
             result = xhr.responseText.split('\\n').join('\n');
             summaryText.value = result.slice(1, -1);
+            _show_summary();
         }
     };
 
-    xhr.send(data)
+    xhr.send(data);
     return;
 }
 
@@ -102,6 +108,8 @@ async function summarize(event) {
 
     sumError.classList.add('hidden');
 
+    _show_summary();
+
     // Here we can finally summarize data
     summaryText.value = 'Please wait...';
     switch (selectOption.value) {
@@ -150,6 +158,23 @@ function _update_counter() {
 }
 
 
+function _show_summary() {
+    showOriginal.classList.remove('bg-gray-100');
+    showSummary.classList.add('bg-gray-100');
+
+    summary.classList.remove('hidden');
+    original.classList.add('hidden');
+}
+
+function _show_original() {
+    showOriginal.classList.add('bg-gray-100');
+    showSummary.classList.remove('bg-gray-100');
+
+    original.classList.remove('hidden');
+    summary.classList.add('hidden');
+}
+
+
 document.addEventListener('DOMContentLoaded', function () {
     selectOption.addEventListener('change', _update_option);
 
@@ -182,4 +207,7 @@ document.addEventListener('DOMContentLoaded', function () {
     });
 
     sumTextInput.addEventListener('input', _update_counter);
+
+    showSummary.addEventListener('click', _show_summary);
+    showOriginal.addEventListener('click', _show_original);
 });

static/text_summarizer.html

@@ -225,10 +225,17 @@
                     </div>
 
                     <div class="mt-6">
-                        <div class="w-full p-4 bg-white rounded-lg xl:p-6">
-                            <h1 class="text-lg font-medium text-gray-700 capitalize sm:text-xl md:text-2xl">Upload text</h1>
-
-                            <form id='sum-form' class="mt-6 space-y-5" onsubmit="return false;">
+                        <div class="w-full p-4 bg-white xl:p-6">
+                            <ul class="flex text-sm text-center text-gray-500 divide-x divide-gray-200">
+                                <li class="w-full">
+                                    <button id="show-original" class="bg-gray-100 inline-block w-full p-2 text-gray-900 focus:ring-4 focus:ring-blue-300 focus:outline-none">Original</button>
+                                </li>
+                                <li class="w-full">
+                                    <button id="show-summary" class="inline-block w-full p-2 hover:text-gray-700 hover:bg-gray-50 focus:ring-4 focus:ring-blue-300 focus:outline-none">Summary</button>
+                                </li>
+                            </ul>
+                            
+                            <div id='sum-original' class="mt-6 space-y-5">
                                 <div>
                                     <label for="sum-type" class="block text-sm text-gray-700 capitalize">Select Type</label>
                                     <select id="sum-type"
@@ -270,13 +277,9 @@
                                         Summarize
                                     </button>
                                 </div>
-                            </form>
-                        </div>
-
-                        <div class="w-full p-4 mt-8 bg-white rounded-lg xl:p-6">
-                            <h1 class="text-lg font-medium text-gray-700 capitalize sm:text-xl md:text-2xl">Result</h1>
+                            </div>
 
-                            <div class="md:flex gap-8">
+                            <div id="sum-summary" class='hidden mt-6 space-y-5'>
                                 <div class="w-full md:w-1/2 grid-cols-2 mt-6 hidden">
                                     <label for="description"
                                         class="block text-sm text-gray-700 capitalize">Extracted Text</label>
@@ -285,8 +288,6 @@
                                 </div>
 
                                 <div class="w-full md:w-1/2 grid-cols-2 mt-6">
-                                    <label for="description"
-                                        class="block text-sm text-gray-700 capitalize">Summarized Text</label>
                                     <textarea id="summarized-text"
                                         class="minh-text block w-full px-4 py-2.5 mt-2 text-gray-600 placeholder-gray-400 bg-white border border-gray-200 rounded-md focus:border-indigo-400 focus:outline-none focus:ring focus:ring-indigo-300 focus:ring-opacity-40"
                                         style="min-height: 250px;"></textarea>