Update epi_pipeline.py

epi_pipeline.py

@@ -224,9 +224,11 @@ def streamlit_getAbs(searchterm_list:Union[List[str],List[int],str], maxResults:
 import os
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras.preprocessing.text import tokenizer_from_json
 import tensorflow as tf
 import numpy as np
 import spacy
+import json
 
 class Classify_Pipeline:
     def __init__(self,model:str='LSTM_RNN_Model'):
@@ -235,9 +237,12 @@ class Classify_Pipeline:
         self.nlpSci = spacy.load("en_ner_bc5cdr_md")
         self.nlpSci2 = spacy.load('en_ner_bionlp13cg_md')
         # load the tokenizer
-        with open(model+'/tokenizer.pickle', 'rb') as handle:
-            import pickle
-            self.classify_tokenizer = pickle.load(handle)
+        with open('tokenizer.json') as f:
+            self.classify_tokenizer = tokenizer_from_json(json.load(f))
+        #OLD Code - used pickle which is unsafe
+        #with open(model+'/tokenizer.pickle', 'rb') as handle:
+        #    import pickle
+        #    self.classify_tokenizer = pickle.load(handle)
         # Defaults to load my_model_orphanet_final, the most up-to-date version of the classification model,
         # but can also be run on any other tf.keras model
     
@@ -450,7 +455,6 @@ import re
 from transformers import BertConfig, AutoModelForTokenClassification, BertTokenizer, Trainer
 from unidecode import unidecode
 from collections import OrderedDict
-import json
 import pandas as pd
 from more_itertools import pairwise