import numpy as np
import pandas as pd 
import glob
import json
import math
import warnings
warnings.filterwarnings('ignore')
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
"""
### Loading Meta Data
"""
meta_df = pd.read_csv('../input/CORD-19-research-challenge/metadata.csv')
meta_df.head()
"""
#### Get papers having covid content
"""
meta_df = meta_df[(meta_df.abstract.str.contains('covid') | meta_df.abstract.str.contains('coronavirus') | meta_df.abstract.str.contains('cov') | meta_df.abstract.str.contains('coronaviruses'))] 
meta_df.drop_duplicates(['sha'], inplace = True)
len(meta_df)
meta_df.info()
"""
### Loading All Json data
"""
all_json = meta_df.pdf_json_files.tolist()
all_json = ['../input/CORD-19-research-challenge/' + str(x) for x in all_json]
_json = glob.glob('../input/CORD-19-research-challenge/document_parses/pdf_json/*.json', recursive=True)
len(all_json)
class FileReader:
    def __init__(self, file_path):
        with open(file_path) as file:
            content = json.load(file)
            self.paper_id = content['paper_id']
            self.abstract = []
            self.body_text = []
            # Abstract
            for entry in content['abstract']:
                self.abstract.append(entry['text'])
            # Body text
            for entry in content['body_text']:
                self.body_text.append(entry['text'])
            self.abstract = '\n'.join(self.abstract)
            self.body_text = '\n'.join(self.body_text)
    def __repr__(self):
        return f'{self.paper_id}: {self.abstract[:200]}... {self.body_text[:200]}...'

first_row = FileReader(all_json[0])
print(first_row)

dict_ = {'paper_id': [], 'abstract': [], 'body_text': []}
for idx, entry in enumerate(all_json):
    try:
        content = FileReader(entry)
        if (idx % 1000 == 0):
            print(f"processing {idx} of {len(all_json)} ")
        dict_['paper_id'].append(content.paper_id)
        dict_['abstract'].append(content.abstract)
        dict_['body_text'].append(content.body_text)
    except:
        pass
df_covid = pd.DataFrame(dict_, columns=['paper_id', 'abstract', 'body_text'])
df_covid.head()
final_df = pd.merge(meta_df, df_covid, how = 'inner', left_on = 'sha', right_on = 'paper_id')
final_df.head()
selected_columns = ['paper_id','title', 'doi', 'abstract_x', 'body_text', 'authors', 'journal', 'publish_time']
final_df = final_df[selected_columns]
final_df.info()
final_df.sample(1).abstract_x.values
final_df.to_csv('selected_data.csv', index = False)
final_df.head()