Add File support (.pdf | .txt)

inference.py

@@ -6,8 +6,8 @@ path = './model/'
 model = BertForSequenceClassification.from_pretrained(path)
 tokenizer = AutoTokenizer.from_pretrained(path)
 
-LABELS = ['радость', 'интерес', 'удивление', 'печаль', 'гнев', 'отвращение',
-          'страх', 'вина', 'нейтрально']
+LABELS = ['Joy', 'Interest', 'Surprise', 'Sadness', 'Anger', 'Disgust', 'Fear',
+          'Guilt', 'Neutral']
 
 
 # Probabilistic prediction of emotion in a text

static/emotion_detection.html

@@ -223,7 +223,7 @@
                                     <select id="sum-type"
                                         class="block w-full px-4 py-2.5 mt-2 text-gray-600 placeholder-gray-400 bg-white border border-gray-200 rounded-md focus:border-indigo-400 focus:outline-none focus:ring focus:ring-indigo-300 focus:ring-opacity-40">
                                         <option value="sum-text">Identify Text</option>
-                                        <option value="sum-file">Identify PDF</option>
+                                        <option value="sum-file">Identify File</option>
                                     </select>
                                 </div>
 
@@ -244,8 +244,8 @@
                                                 d="M16.88 9.1A4 4 0 0 1 16 17H5a5 5 0 0 1-1-9.9V7a3 3 0 0 1 4.52-2.59A4.98 4.98 0 0 1 17 8c0 .38-.04.74-.12 1.1zM11 11h3l-4-4-4 4h3v3h2v-3z">
                                             </path>
                                         </svg>
-                                        <span class="mt-4">Select PDF File</span>
-                                        <input id="sum-file-input" type="file" class="hidden">
+                                        <span class="mt-4">Select File</span>
+                                        <input id="sum-file-input" type="file" accept=".pdf, .txt" class="hidden">
                                     </label>
                                 </div>
 
@@ -298,6 +298,7 @@
         </div>
     </div>
 
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/2.0.943/pdf.min.js"></script>
     <script src="js/detection.js"></script>
 </body>
 </html>

static/js/detection.js

@@ -7,6 +7,7 @@ const sumVideo = document.getElementById("sum-video-div");
 // Form Data
 const selectOption  = document.getElementById("sum-type");
 const sumTextInput  = document.getElementById("sum-text-input");
+const sumFileInput  = document.getElementById("sum-file-input");
 const sumVideoInput = document.getElementById("sum-video-input");
 
 // Error Output Section
@@ -16,8 +17,9 @@ const sumError = document.getElementById("sum-err");
 const extractText = document.getElementById("extracted-text");
 const summaryText = document.getElementById("summarized-text");
 
+const MAX_SIZE = 5000;
+
 
-// In progress...
 function _summarize(text) {
     var xhr = new XMLHttpRequest();
     xhr.open("POST", "/predict_emotion", true);
@@ -37,9 +39,40 @@ function _summarize(text) {
     return;
 }
 
-// In progress...
-function _extractPDF(file) {
-    return file;
+function _extractFile() {
+    const file = sumFileInput.files[0];
+    if (file.type === 'text/plain') {
+        const reader = new FileReader();
+        reader.onload = function() {
+            extractText.value = reader.result.slice(0, MAX_SIZE);
+        };
+        reader.readAsText(file, 'CP1251');
+        return;
+    } else if (file.type === 'application/pdf') {
+        extractText.value = '';
+        const reader = new FileReader();
+        reader.onload = function (e) {
+            const pdfData = e.target.result;
+            pdfjsLib.getDocument(pdfData).promise.then(function (pdfDocument) {
+                for (let pageNum = 1; pageNum <= pdfDocument.numPages; pageNum++) {
+                    pdfDocument.getPage(pageNum).then(function (pdfPage) {
+                      pdfPage.getTextContent().then(function (textContent) {
+                        let size = extractText.value.length;
+                        let pageText = [];
+                        for (const textItem of textContent.items) {
+                            pageText.push(textItem.str);
+                            size += textItem.str.length;
+                            if (size > MAX_SIZE) break;
+                        }
+                        extractText.value += pageText.join(' ');
+                      });
+                    });
+                  }
+            });
+        };
+        reader.readAsDataURL(file);
+    }
+    return;
 }
 
 // In progress...
@@ -67,29 +100,37 @@ function summarize(event) {
             }
             break;
         case 'sum-file':
-            sumError.innerText = 'This option is not supported, yet';
-            sumError.classList.remove('hidden');
-            return;
+            const allowedTypes = ['application/pdf', 'text/plain'];
+            const file = sumFileInput.files[0];
+
+            if (!file) {
+                sumError.innerText = 'There is no File';
+                sumError.classList.remove('hidden');
+                return;
+            }
+
+            if (!allowedTypes.includes(file.type)) {
+                sumError.innerText = 'Not supported type (Only `.pdf` or `.txt`)';
+                sumError.classList.remove('hidden');
+                return;
+            }
     }
 
     sumError.classList.add('hidden');
 
     // Here we can finally summarize data
-    let extractedText = '';
     switch (selectOption.value) {
         case 'sum-text':
-            extractedText = sumTextInput.value;
+            extractText.value = sumTextInput.value.slice(0, MAX_SIZE);
             break;
         case 'sum-file':
-            extractedText = _extractPDF(sumFileInput);
+            _extractFile();
             break;
         case 'sum-video':
-            extractedText = _getCaptions(sumVideoInput.value);
+            _getCaptions(sumVideoInput.value);
             break;
     }
-
-    extractText.value = extractedText;
-    _summarize(extractedText);
+    _summarize(extractText.value);
 }
 
 

static/js/summarizer.js

@@ -7,6 +7,7 @@ const sumVideo = document.getElementById("sum-video-div");
 // Form Data
 const selectOption  = document.getElementById("sum-type");
 const sumTextInput  = document.getElementById("sum-text-input");
+const sumFileInput  = document.getElementById("sum-file-input");
 const sumVideoInput = document.getElementById("sum-video-input");
 
 // Error Output Section
@@ -16,6 +17,8 @@ const sumError = document.getElementById("sum-err");
 const extractText = document.getElementById("extracted-text");
 const summaryText = document.getElementById("summarized-text");
 
+const MAX_SIZE = 5000;
+
 
 // In progress...
 function _summarize(text) {
@@ -35,9 +38,40 @@ function _summarize(text) {
     return;
 }
 
-// In progress...
-function _extractPDF(file) {
-    return file;
+function _extractFile() {
+    const file = sumFileInput.files[0];
+    if (file.type === 'text/plain') {
+        const reader = new FileReader();
+        reader.onload = function() {
+            extractText.value = reader.result.slice(0, MAX_SIZE);
+        };
+        reader.readAsText(file, 'CP1251');
+        return;
+    } else if (file.type === 'application/pdf') {
+        extractText.value = '';
+        const reader = new FileReader();
+        reader.onload = function (e) {
+            const pdfData = e.target.result;
+            pdfjsLib.getDocument(pdfData).promise.then(function (pdfDocument) {
+                for (let pageNum = 1; pageNum <= pdfDocument.numPages; pageNum++) {
+                    pdfDocument.getPage(pageNum).then(function (pdfPage) {
+                      pdfPage.getTextContent().then(function (textContent) {
+                        let size = extractText.value.length;
+                        let pageText = [];
+                        for (const textItem of textContent.items) {
+                            pageText.push(textItem.str);
+                            size += textItem.str.length;
+                            if (size > MAX_SIZE) break;
+                        }
+                        extractText.value += pageText.join(' ');
+                      });
+                    });
+                  }
+            });
+        };
+        reader.readAsDataURL(file);
+    }
+    return;
 }
 
 // In progress...
@@ -65,29 +99,37 @@ function summarize(event) {
             }
             break;
         case 'sum-file':
-            sumError.innerText = 'This option is not supported, yet';
-            sumError.classList.remove('hidden');
-            return;
+            const allowedTypes = ['application/pdf', 'text/plain'];
+            const file = sumFileInput.files[0];
+
+            if (!file) {
+                sumError.innerText = 'There is no File';
+                sumError.classList.remove('hidden');
+                return;
+            }
+
+            if (!allowedTypes.includes(file.type)) {
+                sumError.innerText = 'Not supported type (Only `.pdf` or `.txt`)';
+                sumError.classList.remove('hidden');
+                return;
+            }
     }
 
     sumError.classList.add('hidden');
 
     // Here we can finally summarize data
-    let extractedText = '';
     switch (selectOption.value) {
         case 'sum-text':
-            extractedText = sumTextInput.value;
+            extractText.value = sumTextInput.value.slice(0, MAX_SIZE);
             break;
         case 'sum-file':
-            extractedText = _extractPDF(sumFileInput);
+            _extractFile();
             break;
         case 'sum-video':
-            extractedText = _getCaptions(sumVideoInput.value);
+            _getCaptions(sumVideoInput.value);
             break;
     }
-
-    extractText.value = extractedText;
-    _summarize(extractedText);
+    _summarize(extractText.value);
 }
 
 

static/text_summarizer.html

@@ -223,7 +223,7 @@
                                     <select id="sum-type"
                                         class="block w-full px-4 py-2.5 mt-2 text-gray-600 placeholder-gray-400 bg-white border border-gray-200 rounded-md focus:border-indigo-400 focus:outline-none focus:ring focus:ring-indigo-300 focus:ring-opacity-40">
                                         <option value="sum-text">Summarize Text</option>
-                                        <option value="sum-file">Summarize PDF</option>
+                                        <option value="sum-file">Summarize File</option>
                                         <!-- In progress... -->
                                         <!-- <option value="sum-video">Summarize Youtube Video</option> -->
                                     </select>
@@ -247,7 +247,7 @@
                                             </path>
                                         </svg>
                                         <span class="mt-4">Select PDF File</span>
-                                        <input id="sum-file-input" type="file" class="hidden">
+                                        <input id="sum-file-input" type="file" accept=".pdf, .txt" class="hidden">
                                     </label>
                                 </div>
 
@@ -300,6 +300,7 @@
         </div>
     </div>
 
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/2.0.943/pdf.min.js"></script>
     <script src="js/summarizer.js"></script>
 </body>
 </html>