# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/restrepo/rest-api-doc/blob/master/code/lensapi.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] colab_type="text" id="D_lKr3Avj0Ag"
# # Lens API search for institution and range of years or dates

# + [markdown] colab_type="text" id="fyrZq73Yj0Ak"
# ## Functions and global variables

# + [markdown] colab_type="text" id="T4Rt_kx7j0Al"
# TODO:
# Orcid search: https://www.lens.org/lens/scholar/search/results?q=&orcids=0000-0001-6455-5564

# + colab_type="code" id="Rw3EV881j0Am" outputId="5eb7399d-c4e2-4d2b-e955-5ef7331e8840" colab={"base_uri": "https://localhost:8080/", "height": 103}
import getpass
import pandas as pd
import requests
import sys
import time
import os
if os.getcwd()=='/content':
    # !pip install unidecode
import unidecode    

# + [markdown] colab_type="text" id="TFUR-R4pj0Ap"
# <!-- #mmkP9Ibp7bDpLNLKZF8bdYDxCQmxXhFuJzJViVKH92kyBlGmatE2
#  -->

# + colab_type="code" id="Nl8_4Hyoj0Aq" outputId="fede12fd-4645-4fe0-e263-602d4b1c8ed9" colab={"base_uri": "https://localhost:8080/", "height": 34}
token=getpass.getpass('token:')

# + colab_type="code" id="TwCrMmcRj0Au" colab={}
url = 'https://api.lens.org/scholarly/search'
size=1000


# + colab_type="code" id="vG_EKs5tj0Ay" colab={}
def query(affiliation="University of Antioquia",range_type="date_published",
          date_ini="2003-01-01",
          date_end="2004-12-31",size=1000):
    if range_type=="year_published":
        date_ini=date_ini.split('-')[0]
        date_end=date_end.split('-')[0]
    data='''{
    "query": {
       "bool": {
         "must": [{
           "match_phrase":{
                    "title": "%s"
               }
           },
           {
           "range": {
           "%s": {
                    "gte": "%s",
                    "lte": "%s"
                }
           } 
           }
         ]
       }
     },
     "size": %i
    }''' %(affiliation,range_type,date_ini,date_end,size)
    return data
def query_title(title="Dark Matter",range_type="date_published",
          date_ini="2003-01-01",
          date_end="2004-12-31",size=1000):
    """
    WARNING: Accents needs to be removed!
    """
    title=unidecode.unidecode(title)
    if range_type=="year_published":
        date_ini=date_ini.split('-')[0]
        date_end=date_end.split('-')[0]
    data='''{
    "query": {
       "bool": {
         "must": [{
           "match_phrase":{
                    "title": "%s"
               }
           },
           {
           "range": {
           "%s": {
                    "gte": "%s",
                    "lte": "%s"
                }
           } 
           }
         ]
       }
     },
     "size": %i
    }''' %(title,range_type,date_ini,date_end,size)
    return data


# + [markdown] colab_type="text" id="moXfc414j0BM"
# ## Full search: University of Antioquia
# Maximum number of responses currently is `size=1000` by web is 50000. However the JSON output for the API seem to be more complete: Includes list of citations! no only the number

# + colab_type="code" id="aqlXzSCZj0BN" colab={}
def get_query(token,data,url='https://api.lens.org/scholarly/search',size=1000):
    headers = {'Authorization': '{}'.format(token), 'Content-Type': 'application/json'}
    response = requests.post(url, data=data, headers=headers)
    if response.status_code != requests.codes.ok:
        sys.exit(response.status_code)
    else:
        return response.text    


# + colab_type="code" id="yHUIL4RNj0BS" colab={}
year_ranges=[["1900","1994"],["1995","2000"],["2001","2003"]   ]

# + colab_type="code" id="OP3puxdLj0BU" outputId="3db7862d-414d-4689-e96f-f2a089f48ca7" colab={"base_uri": "https://localhost:8080/", "height": 120}
udea=pd.DataFrame()
for i in range(len(year_ranges)):
    print(year_ranges[i])
    data=query(affiliation="University of Antioquia",range_type="year_published",
      date_ini=year_ranges[i][0],date_end=year_ranges[i][1],size=1000)
    r=get_query(token,data)
    print("query:",pd.read_json(r).shape[0] )
    udea=udea.append( pd.read_json(r),sort=True  ).reset_index(drop=True)
    tmp=udea.copy()
    time.sleep(2)

# + colab_type="code" id="8LesJKD1j0BX" colab={}
date_ranges=[["2004-01-01","2005-11-30"],["2005-12-01","2006-12-31"],["2007-01-01","2007-12-31"],
             ["2008-01-01","2008-10-31"],["2008-11-01","2009-05-31"],['2009-06-01','2009-12-31'],
             ['2010-07-01', '2010-12-31'],['2011-01-01', '2011-05-31'],['2011-06-01', '2011-12-31'],
             ['2012-01-01', '2012-05-31'],['2012-06-01', '2012-11-30'],['2012-12-01', '2013-03-31'],
             ['2013-04-01', '2013-10-31'],['2013-11-01', '2014-02-28'],['2014-03-01', '2014-10-31'],
             ['2014-11-01', '2015-02-28'],['2015-03-01', '2015-10-31'],['2015-11-01', '2016-03-31'],
             ['2016-04-01', '2016-09-30'],['2016-10-01', '2016-12-31'],['2017-01-01', '2017-06-30'],
             ['2017-07-01', '2017-12-31'],['2018-01-01', '2018-08-31'],['2018-09-01', '2019-06-30']
            ]

# + colab_type="code" id="VhzeBRPDj0Bc" colab={}
FIND_DATES=False
if FIND_DATES:
    m=["","01-31","02-28","03-31","04-30","05-31","06-30","07-31","08-31","09-30","10-31","11-30","12-31"]
    dr=date_ranges[0][1].split('-')

    year="2019-"
    month=6
    if int(dr[1])<12:
        date_ranges=[[dr[0]+'-'+m[int(dr[1])+1].split('-')[0]+'-'+'01',year+m[month]]]
    else:
        date_ranges=[[str(int(dr[0])+1)+'-'+'01'+'-'+'01',year+m[month]]]
    print( date_ranges )

# + id="LelhGVCRDkhi" colab_type="code" colab={}
1+1

# + id="KDpSIDSYDUaX" colab_type="code" colab={}
data=query(affiliation="University of Antioquia",range_type="date_published",
      date_ini='2018-01-01',date_end='2018-08-31',size=1000)
r=get_query(token,data)

# + colab_type="code" id="Fbex4g0aj0Be" outputId="c17ab0f4-b2bd-4361-f70d-c391d3478725" colab={"base_uri": "https://localhost:8080/", "height": 974}
#WARNING: Failed after some i-th request: fix initial range to continue
for i in range(0,len(date_ranges)):
    print(date_ranges[i])
    data=query(affiliation="University of Antioquia",range_type="date_published",
      date_ini=date_ranges[i][0],date_end=date_ranges[i][1],size=1000)
    r=get_query(token,data)
    kk=pd.read_json(r)
    print("query:",i,kk.shape[0] )
    if FIND_DATES:
        if kk.shape[0]>=1000 and month>1:
            month=month-1
            date_ranges=[[date_ranges[0][0],year+m[month]]]
        
    udea=udea.append( kk , sort=True ).reset_index(drop=True)
    tmp=udea.copy()

    time.sleep(5)

# + colab_type="code" id="aE9AgET5j0Bg" colab={}
udea=pd.DataFrame( list( udea.data.values ) ).reset_index(drop=True)

# + colab_type="code" id="OrD67YO5j0Bi" outputId="b1453625-cb49-404c-b722-961516180890" colab={"base_uri": "https://localhost:8080/", "height": 34}
udea.shape

# + colab_type="code" id="v4Ve3oPYj0Bj" colab={}
udea.to_json('data/udea.json.gz',compression='gzip')

# + [markdown] colab_type="text" id="YlSjAp7ej0Bl"
# ## Analysis

# + colab_type="code" id="W9KMHUa7j0Bm" outputId="b226044c-d296-47e3-f73f-69bdd8174ada" colab={"base_uri": "https://localhost:8080/", "height": 35}
# %pylab inline

# + colab_type="code" id="g3SlZp7bj0Bo" colab={}
import pandas as pd

# + colab_type="code" id="iyzMs6BXj0Bq" outputId="de1fbbe7-ed6e-4b58-ac57-faed0933e343" colab={}
# ls data/

# + colab_type="code" id="D10s9_5Lj0Bs" colab={}
udea=pd.read_json( 'https://github.com/restrepo/lensapi/raw/master/data/udea.json.gz',compression='gzip')#.reset_index(drop=True) )

# + colab_type="code" id="id4qduLVj0Bu" colab={}
#udea=pd.read_json('data/lens_udea_lines_True_orient_records.json.gz',lines=True,orient='records',compression='gzip')#.reset_index(drop=True)

# + colab_type="code" id="gIYvyYmLj0Bw" outputId="00598ea8-bfa6-41a1-c657-8b9460a14c99" colab={"base_uri": "https://localhost:8080/", "height": 35}
udea.shape

# + id="LgCBtgFPdSv0" colab_type="code" outputId="bd4b4782-5a26-4a54-9607-caf65b7c7ab9" colab={"base_uri": "https://localhost:8080/", "height": 318}
udea[:2]

# + [markdown] colab_type="text" id="-1CuTsL4j0By"
# ### Entries with publication date

# + colab_type="code" id="ZYJgNBiCj0Bz" outputId="594961b6-ba2d-4a8f-bac5-842ba3493208" colab={"base_uri": "https://localhost:8080/", "height": 235}
udea.date_published

# + colab_type="code" id="MFkDDv0sj0B1" colab={}
udeay=udea[~udea.date_published.isna()]

# + colab_type="code" id="Z8SKbQu6j0B3" colab={}
udeay['Year']=udea.date_published.apply(lambda s: s.split('-')[0] if type(s)==str else 
                                          s.get('date').split('-')[0] )

# + colab_type="code" id="wtoo_h1ej0B6" colab={}
udeay=udeay[~udeay['Year'].isna()].reset_index(drop=True)
udeay['Year']=udeay['Year'].astype(int)

# + colab_type="code" id="NNrTL5B0j0B8" outputId="b8b6dec2-6dd2-4648-e8c4-4f9d9651636b" colab={}
udeay[udeay['Year']>2007]['Year'].value_counts().sort_index().plot(kind='bar')

# + colab_type="code" id="_Gc7Iplyj0B-" outputId="58725c45-560c-492f-f874-14941777ee24" colab={}
udeay.shape

# + colab_type="code" id="UtmESdNlj0CA" colab={}
udeay['doi']=udeay['external_ids'].apply(lambda l:[d.get('value') for d in l if d.get('type')=='doi'] if type(l)==list else None)
udeay['doi']=udeay['doi'].apply(lambda l: l[0] if type(l)==list and len(l)>0 else None)

# + colab_type="code" id="rUZKbJR4j0CB" colab={}
udea_doi=udeay[~udeay['doi'].isna()]

# + colab_type="code" id="DTN2anF3j0CD" outputId="08958946-40ca-47e0-8e53-0966067840b9" colab={}
udea_doi.shape

# + colab_type="code" id="ISOl0XUDj0CG" outputId="3c870ffd-6b12-487b-b3b1-d20b95c42d73" colab={}
udea_doi[udea_doi['Year']>2007]['Year'].value_counts().sort_index().plot(kind='bar')

# + colab_type="code" id="LlSIap7Ej0CJ" outputId="16d8e8fa-9167-4d66-9139-96654732f789" colab={}
minyear=2007
totx=list( udeay[udeay['Year']>minyear]['Year'].value_counts().sort_index().index )
toty=list( udeay[udeay['Year']>minyear]['Year'].value_counts().sort_index().values )
doix=list( udea_doi[udea_doi['Year']>minyear]['Year'].value_counts().sort_index().index )
doiy=list( udea_doi[udea_doi['Year']>minyear]['Year'].value_counts().sort_index().values )
plt.plot(totx,toty,label='TOTAL')
plt.plot(totx,toty,'bo')
plt.plot(doix,doiy,'r',label='DOI')
plt.plot(doix,doiy,'ro')
plt.legend(loc='best')
plt.xlim(2008,2018.1)
plt.title('Producción UDEA')
plt.ylabel('Número de artículos cada año',size=15)
plt.savefig('totdoi.pdf')

# + colab_type="code" id="2kIwBP6pj0CK" outputId="f34f2c90-6560-430d-b6da-2a67c5e8bfdf" colab={}
# ls /scratch/restrepo/HUGE/JSON/

# + colab_type="code" id="AspQFy_1j0CM" colab={}


StarcoderdataJupyterScriptsDedupFiltered