from datetime import datetime
start = datetime.utcnow() # For measuring the total processing time

import json
from urllib.request import urlopen
import pandas as pd
import numpy as np

"""
Explanation: Languages pivot table
End of explanation
"""


AMC_URL = "http://articlemeta.scielo.org/api/v1/collection/identifiers/"
amc_data = pd.DataFrame(json.load(urlopen(AMC_URL)))

print("Number of collections: " + str(amc_data.shape[0]+1))
amc_data.head(2)

"""
Explanation: Get collection information from ArticleMeta
End of explanation
"""


dont_evaluate = ["bio", "cci", "cic", "ecu", "psi", "pry", "rve", "rvo", "rvt", "sss", "spa", "wid"]
amc_names_map = {"code": "collection"}
amc_pairs = amc_data[(amc_data["acron2"].str.len() == 2) & 
                     ~amc_data["code"].isin(dont_evaluate)]\
                    .rename(columns=amc_names_map)\
#                    "~amc_data["code"].isin(dont_evaluate)]" is denying the list "dont_evaluate"
print("Number of collections: " + str(amc_pairs.shape[0]+1))

collections = amc_pairs[['collection']].copy()
collections

"""
Explanation: Filtering valid collections and renames 'code' to 'collection'
Some collections won't be analyzed, mainly to avoid duplicates
(there are articles in more than one collection).
The spa (Public Health collection) should have part of it
kept in the result, but it's not a collection
whose journals/articles are assigned to a single country.
The collections below are linked to a single country:
End of explanation
"""


spa_issn_country = pd.DataFrame([
    ("0021-2571"),
    ("0042-9686"),
    ("1020-4989"),
    ("1555-7960"),
], columns=["issn"])
spa_issn_country # For collection = "spa", only!

"""
Explanation: ISSN selection from spa
These journals in the spa collection have the following countries:
End of explanation
"""


import zipfile

# Use the Zip file in jcatalog/data/scielo
with zipfile.ZipFile('../../data/scielo/tabs_network_190210.zip', 'r') as zip_ref:
    zip_ref.extract('documents_languages.csv', 'csv_files')


dataset = pd.read_csv("csv_files/documents_languages.csv", keep_default_na=False)
dataset.shape

"""
Explanation: Languages dataset
This dataset is the
Network spreadsheet/CSV pack
 which can be found in the
SciELO Analytics report
web page.
The first two rows of it are:
Unzip the CSV file
End of explanation
"""


names_map = {
    "ISSN SciELO": "issn",
    "collection": "collection",
    "title at SciELO": "title",
    "document publishing ID (PID SciELO)": "docs",
    "document type":"type",
    "document languages": "languages",
    "document is citable": "is_citable",
    "document publishing year": "year",
    "document pt": "document_pt",
    "document es": "document_es",
    "document en": "document_en",
    "document other languages": "document_other_languages"
}
df0 = dataset[list(names_map.keys())].rename(columns=names_map)
df0.head(2)

"""
Explanation: Simplify the column names
End of explanation
"""


df = pd.concat([
    pd.merge(df0[df0["collection"] != "spa"], collections,      how="inner", on="collection"),
    pd.merge(df0[df0["collection"] == "spa"], spa_issn_country, how="inner", on="issn"),
])

df.head(2)

# compare
df0.shape

df.shape

set(df.collection)

"""
Explanation: Creates a new DataFrame: filtering SPA and discarding those collections that are not analyzable
End of explanation
"""


# df["pub_year"] = np.where(df['year'] <= 1996, 'ate_1996', df["year"])
df["pub_year"] = np.where(df['year'] <= 1996, 'anterior', df["year"])

"""
Explanation: Add pub_year (ate_1996)
End of explanation
"""


df["tipo_review"] = np.where(df['type'] == "review-article", 1, 0)

"""
Explanation: Add review type
End of explanation
"""


df["citable_pt"] = np.where((df['document_pt'] == 1) & (df['is_citable'] == 1), 1, 0)
df["citable_es"] = np.where((df['document_es'] == 1) & (df['is_citable'] == 1), 1, 0)
df["citable_en"] = np.where((df['document_en'] == 1) & (df['is_citable'] == 1), 1, 0)
df["citable_other_lang"] = np.where((df['document_other_languages'] == 1) & (df['is_citable'] == 1), 1, 0)

df['sum_to_2_more_lang'] = np.sum([df['document_en'], df['document_pt'], df['document_es'], df['document_other_languages']], axis=0)
df[(df['sum_to_2_more_lang'] == 3)].T

"""
Explanation: Add citable_(language)
End of explanation
"""


df["doc_2_more_lang"] = np.where(((df['sum_to_2_more_lang']) >= 2), 1, 0)
df["citable_doc_2_more_lang"] = np.where((df['sum_to_2_more_lang'] >= 2) & (df['is_citable'] == 1), 1, 0)
# remove sum_to_2_more_lang column
del df['sum_to_2_more_lang']

df[(df["doc_2_more_lang"] == 1)].T

df.columns

values_list = [
    "docs",
    "is_citable", 
    "tipo_review",
    "document_pt", 
    "document_es", 
    "document_en",
    "document_other_languages",
    "doc_2_more_lang",
    "citable_pt",
    "citable_es",
    "citable_en",
    "citable_other_lang",
    "citable_doc_2_more_lang"]

td = df.pivot_table(
     index=["issn"],
     values=values_list,
     columns=["pub_year"],
     aggfunc=np.count_nonzero,
     fill_value=0)

td[:12].T

td.columns.levels

"""
Explanation: Add 2 or more lang
End of explanation
"""


td.keys()
for k in td.keys():
    print(k)

newlabel = []
for k in td.keys():
    newlabel.append(k[0]+'_'+k[1])

newlabel

newlabel[::24]

td.columns = newlabel

td.T

td.to_csv("output/td_documents_languages_network.csv")
#  td.to_csv("output/td_documents_languages_bra_190123.csv")

print(f"Notebook processing duration: {datetime.utcnow() - start}")

"""
Explanation: Renames the labels for CSV
End of explanation
"""


