#!/usr/bin/env python3

from logging import basicConfig, getLogger, DEBUG, INFO, WARNING, ERROR
from contextlib import suppress
from json import load, dump
from time import time
# from pdb import set_trace
from pprint import PrettyPrinter
from os import makedirs, path # _exit

from requests.packages.urllib3 import disable_warnings

from retr.farm import *
from retr.proxypool import proxypool

from linkedin import linkedin

lg=getLogger(__name__)

getLogger('requests.packages.urllib3').setLevel(ERROR)
disable_warnings()

ll=WARNING
ll=INFO
ll=DEBUG

basicConfig( format='{asctime} {threadName:11s}: {message}', datefmt="%H:%M:%S",
             #filename='lin-{}.log'.format(time()),
             style='{', level=ll)

num_threads=5 #00
basedir='tmp'
# Because getting linkedin profile is troublesome, so in order to test more easily it's possible to cache the profiles downloaded.
cache=basedir # some path to use when caching files, '' to disable it (save MUCH space)
do_filter=0
infile='output_sample.json'

if 0:
    # With proxyrack it's of no big use as the proxies will be rotated there anyway, so no use in marking them bad or good.
    pp=proxypool('proxyrack1.lst', no_disable=True)
    num_threads=1
    timeout=30        
else:
    from pdb import set_trace
    #set_trace()
    pp=proxypool('proxies.lst')
    proxy_headers={}
    
    timeout=180

    if do_filter:
        from retr.proxypool_common import set_plist_for_filter, print_length
        cache='' # It's even harmful when testing proxies
        # od=OrderedDict()    
        # pp.master_plist=od
        # for k, v in  in pp.master_plist.items():
        #     if v.flags != 'G': k:v.replace('G','') for k,v

        old_master_plist=set_plist_for_filter(pp)
    else: # Leave only good or unknown for now
        for k, v in tuple(pp.master_plist.items()):
            if v.flags and 'G' not in v.flags: del pp.master_plist[k]
    
########## No configuration past this line ###############

with suppress(FileExistsError): makedirs(path.join(basedir,'output'))

toget=[]
input_data={}
with open(infile) as ff:
    j=load(ff)
    for i in j:
        url=i['linkedinURL']
        if not url or '/pub/dir/' in url or '/company/' in url:
            lg.warning('Skipping {}'.format(j))
            continue
        toget.append(url)
        input_data[url]=i

if do_filter: # Filtering stuff
    toget=['https://www.linkedin.com/directory/people-a']*len(pp.master_plist)
#toget=['https://www.linkedin.com/in/jefflich']
f=farm(num_threads,
       lambda: linkedin(pp, True, cache, do_filter, False, timeout=timeout),
       toget, extendable=True)

ppr=PrettyPrinter(indent=4)
with open(infile) as fi:
    for it in f.run():
        if do_filter: 
            with pp.lock:
                pp.master_plist[it[0]].addflag(it[1]) # Set good flag
                if it[1] == 'G': lg.info('Good: {}'.format(it[0]))
                print_length()
                continue
        # Convert to the desired format
        cur=it['current']
        out={
	    'id' : input_data[it['profile_url']]['id'],
	    'email' : input_data[it['profile_url']]['email'],
            # https://in.linkedin.com/in/abheshverma
            # https://www.linkedin.com/in/stephen-jo-6325a9/de
	    'UID' : it['profile_url'].split('/')[4],
	    'name' : it['name'],
	    'pic': it['image'],
	    'title' : it['title'],
	    'company' : cur[0][0] if cur else '',
	    'education_latest' : it['education_latest'],
	    'area' : it['location'] or '',
	    #"country": "USA",
	    'summary': it['summary'],
	    'featured_skiils': it['skills']
        }
        out['education']=[
            { 'name' : i['title'], 'degree' : i['degree'] or '',
              'start_dt' : i['start'], 'end_dt' : i['end'] }
            for i in it['education']
        ]
        out['experience']=[
            { 'designation' : i['title'], 'company' : i['company'],
              'area' : i['location'] or '',
              'start_dt' : i['start'], 'end_dt' : i['end'] }
            for i in it['experiences']
        ]
        out['volunteer_experience']=[
            { 'name' : i['name'], 'company' : i['company'],
              'area' : i['area'], 'start_dt' : i['start'], 'end_dt' : i['end'] }
            for i in it['volunteer_experiences']
        ]

        #ppr.pprint(out)
        with open(path.join(basedir, 'output',
                            it['profile_url'].split('/')[-1]+'.json'),
                  'w') as fo:
            dump(out, fo, sort_keys=True, indent=4)
    
f.close()
    
if do_filter:
    old_master_plist.update(pp.master_plist) # Maybe flags have changed?
    pp.master_plist=old_master_plist
    #pp.write()
else: # Update toget array
    if 0: # To rerun with the same file
        with open(infile, 'w') as fo:
            for i in f.arr: fo.write(i+'\n')
    
pp.write()
