Add time tracking for each process

app.py

@@ -142,6 +142,7 @@ if uploaded_files:
                         # st.write(pdf.name)
                         L = []
                         # Entity Extraction
+                        start_time_ext = time.time()
                         st.write("☑ Extracting Entities ...")
                         bytes_data = uploaded_file.read()
                         journal = Journal(uploaded_file.name, bytes_data)
@@ -181,12 +182,14 @@ if uploaded_files:
                             chunkdf.append(df)
 
                         concat = pd.concat(chunkdf, axis=0).reset_index().drop('index', axis=1).fillna('')
-                        st.write("☑ Entities Extraction Done ..")
+                        st.write("☑ Entities Extraction Done ..", round((time.time() - start_time_ext) / 60, 2), "minutes")
                         time.sleep(0.1)
+                        start_time_summ = time.time()
                         st.write("☑ Generating Summary ...")
                         summary = get_summ(pdf.name)
-                        st.write("☑ Generating Summary Done ..")
+                        st.write("☑ Generating Summary Done ..", round((time.time() - start_time_summ) / 60, 2), "minutes")
                         time.sleep(0.1)
+                        start_time_tab = time.time()
                         st.write("☑ Table Extraction in progress ...")
                         # Table Extraction
                         # L = []
@@ -569,7 +572,7 @@ if uploaded_files:
                                         else:
                                             L.append(row)
 
-                    st.write("☑ Table Extraction Done ...")
+                    st.write("☑ Table Extraction Done ...", round((time.time() - start_time_summ) / 60, 2), "minutes")
                     status.update(label="Gene and SNPs succesfully collected.")
                     L = [{key: ''.join(['' if item == 'Unknow' else item for item in value]) for key, value in d.items()} for d in L]
                     L = [{key: ''.join(['Not Available' if item == '' else item for item in value]) for key, value in d.items()} for d in L]
@@ -624,6 +627,7 @@ if uploaded_files:
                         with NamedTemporaryFile(dir='.', suffix=".pdf") as rotated_pdf:
                             pdf_writer.write(rotated_pdf.name)
                             # Entity Extraction
+                            start_time_ext = time.time()
                             st.write("☑ Extracting Entities ...")
                             bytes_data = uploaded_file.read()
                             journal = Journal(uploaded_file.name, bytes_data)
@@ -654,12 +658,14 @@ if uploaded_files:
                                 chunkdf.append(df)
 
                             concat = pd.concat(chunkdf, axis=0).reset_index().drop('index', axis=1).fillna('')
-                            st.write("☑ Entities Extraction Done ..")
+                            st.write("☑ Entities Extraction Done ..", round((time.time() - start_time_ext) / 60, 2), "minutes")
                             time.sleep(0.1)
+                            start_time_summ = time.time()
                             st.write("☑ Generating Summary ...")
                             summary = get_summ(pdf.name)
-                            st.write("☑ Generating Summary Done ..")
+                            st.write("☑ Generating Summary Done ..", round((time.time() - start_time_summ) / 60, 2), "minutes")
                             time.sleep(0.1)
+                            start_time_tab = time.time()
                             st.write("☑ Table Extraction in progress ...")
 
                             # Table Extraction
@@ -930,53 +936,12 @@ if uploaded_files:
                                             })
                                         else:
                                             L.append(row)
-                        # 3
-                        for i in range(len(output_list[2]['result'].split('\n'))):
-                            if output_list[2]['result'].split('\n')[i] != "":
-                                try:
-                                    row = literal_eval(output_list[2]['result'].split('\n')[i])[0]
-                                    row = {**row, **{
-                                                'Title' : concat['title'][0],
-                                                'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
-                                                'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
-                                                'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
-                                                'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
-                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
-                                                'Recommendation' : summary,
-                                            }
-                                    }
-                                    if not row['SNPs'].startswith("rs"):
-                                        row.update({
-                                            'SNPs' : "-"
-                                        })
-                                    else:
-                                        L.append(row)
-                                except KeyError:
-                                    row = literal_eval(output_list[2]['result'].split('\n')[i])
-                                    row = {**row, **{
-                                                'Title' : concat['title'][0],
-                                                'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
-                                                'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
-                                                'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
-                                                'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
-                                                'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
-                                                'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
-                                                'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
-                                                'Recommendation' : summary,
-                                            }
-                                    }
-                                    if not row['SNPs'].startswith("rs"):
-                                        row.update({
-                                            'SNPs' : "-"
-                                        })
-                                    else:
-                                        L.append(row)
-                                except ValueError:
-                                        if type(output_list[2]['result'].split('\n')[i]) is dict:
-                                            row = output_list[2]['result'].split('\n')[i]
-                                            row = {**row, **{
+                            # 3
+                            for i in range(len(output_list[2]['result'].split('\n'))):
+                                if output_list[2]['result'].split('\n')[i] != "":
+                                    try:
+                                        row = literal_eval(output_list[2]['result'].split('\n')[i])[0]
+                                        row = {**row, **{
                                                     'Title' : concat['title'][0],
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
                                                     'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
@@ -987,15 +952,15 @@ if uploaded_files:
                                                     'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
                                                     'Recommendation' : summary,
                                                 }
-                                            }
-                                            if not row['SNPs'].startswith("rs"):
-                                                row.update({
-                                                    'SNPs' : "-"
-                                                })
-                                            else:
-                                                L.append(row)
-                                except SyntaxError:
-                                        row = literal_eval("""{}""".format(output_list[2]['result'].split('\n')[i]))
+                                        }
+                                        if not row['SNPs'].startswith("rs"):
+                                            row.update({
+                                                'SNPs' : "-"
+                                            })
+                                        else:
+                                            L.append(row)
+                                    except KeyError:
+                                        row = literal_eval(output_list[2]['result'].split('\n')[i])
                                         row = {**row, **{
                                                     'Title' : concat['title'][0],
                                                     'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
@@ -1014,7 +979,49 @@ if uploaded_files:
                                             })
                                         else:
                                             L.append(row)
-                    st.write("☑ Table Extraction Done")
+                                    except ValueError:
+                                            if type(output_list[2]['result'].split('\n')[i]) is dict:
+                                                row = output_list[2]['result'].split('\n')[i]
+                                                row = {**row, **{
+                                                        'Title' : concat['title'][0],
+                                                        'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
+                                                        'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
+                                                        'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
+                                                        'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
+                                                        'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
+                                                        'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                        'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
+                                                        'Recommendation' : summary,
+                                                    }
+                                                }
+                                                if not row['SNPs'].startswith("rs"):
+                                                    row.update({
+                                                        'SNPs' : "-"
+                                                    })
+                                                else:
+                                                    L.append(row)
+                                    except SyntaxError:
+                                            row = literal_eval("""{}""".format(output_list[2]['result'].split('\n')[i]))
+                                            row = {**row, **{
+                                                        'Title' : concat['title'][0],
+                                                        'Authors' : concat['authors'][0] if 'authors' in list(concat.columns) else '',
+                                                        'Publisher Name' : concat['publisher_name'][0] if 'publisher_name' in list(concat.columns) else '',
+                                                        'Publication Year' :  get_valid_year(' '.join(concat['year_of_publication'].values.tolist())) if 'year_of_publication' in concat.columns else concat.assign(year_of_publication='')['year_of_publication'],
+                                                        'Population' : upper_abbreviation(' '.join(concat['population_race'].values.tolist()).replace('Unknown', '').title()) if 'population_race' in concat.columns else concat.assign(population_race='')['population_race'],
+                                                        'Sample Size' : sample_size_postproc(' '.join(concat['sample_size'].values.tolist()).replace('Unknown', '').title()) if 'sample_size' in concat.columns else concat.assign(sample_size='')['sample_size'],
+                                                        'Study Methodology' : upper_abbreviation(' '.join(concat['study_methodology'].values.tolist()).replace('Unknown', '').title()) if 'study_methodology' in concat.columns else concat.assign(study_methodology='')['study_methodology'],
+                                                        'Study Level' : upper_abbreviation(' '.join(concat['study_level'].values.tolist()).replace('Unknown', '').title()) if 'study_level' in concat.columns else concat.assign(study_level='')['study_level'],
+                                                        'Recommendation' : summary,
+                                                    }
+                                            }
+                                            if not row['SNPs'].startswith("rs"):
+                                                row.update({
+                                                    'SNPs' : "-"
+                                                })
+                                            else:
+                                                L.append(row)
+
+                    st.write("☑ Table Extraction Done", round((time.time() - start_time_summ) / 60, 2), "minutes")                    
                     status.update(label="Gene and SNPs succesfully collected.")
                     L = [{key: ''.join(['' if item == 'Unknow' else item for item in value]) for key, value in d.items()} for d in L]
                     L = [{key: ''.join(['Not Available' if item == '' else item for item in value]) for key, value in d.items()} for d in L]
@@ -1048,6 +1055,7 @@ if uploaded_files:
                         pdf.write(uploaded_file.getbuffer())
 
                         # Entity Extraction
+                        start_time_ext = time.time()
                         st.write("☑ Extracting Entities ...")
                         bytes_data = uploaded_file.read()
                         journal = Journal(uploaded_file.name, bytes_data)
@@ -1068,8 +1076,9 @@ if uploaded_files:
                             chunkdf.append(df)
 
                         concat = pd.concat(chunkdf, axis=0).reset_index().drop('index', axis=1).fillna('')
-                        st.write("☑ Entities Extraction Done ..")
+                        st.write("☑ Entities Extraction Done ..", round((time.time() - start_time_ext) / 60, 2), "minutes")
                         time.sleep(0.1)
+                        start_time_summ = time.time()
                         st.write("☑ Generating Summary ...")
 
                         if 'SNPs' in list(concat.columns):
@@ -1081,7 +1090,7 @@ if uploaded_files:
 
                         summary = get_summ(pdf.name)
                         time.sleep(0.1)
-                        st.write("☑ Generating Summary Done...")
+                        st.write("☑ Generating Summary Done...", round((time.time() - start_time_summ) / 60, 2), "minutes")
                         for i in range(len(concat)):
                             if (len(concat['genes_locus'][i].split(',')) >= 1) and concat['SNPs'][i] == '':
                                 for g in concat['genes_locus'][i].split(','):