#!/usr/bin/python
## Uses a DeferredList to asynchronously fetch a number of pages
## Note: has problems if too much pages are being fetched async
import sys,re,json,feedparser
from time import strftime
from datetime import datetime
# sys.path.append("rawdog")
from   twisted.internet import reactor
from   twisted.internet.task import deferLater
from   twisted.python import log
from   twisted.web.http_headers import Headers
from   twisted.python.util import println
from   txpostgres import txpostgres
import twisted.web.client as client
from   twisted.internet.ssl import ClientContextFactory
from   twisted.internet.defer import setDebugging, Deferred, DeferredList, DeferredSemaphore, gatherResults
import psycopg2
from   twisted.enterprise import adbapi
import treq
# http://txpostgres.readthedocs.org/


# Experimental Twisted version

global DEBUG_MODE
global dbconn
global dbd
global agent
global dbpool
global query_add_to_db

with open('sql/insert_in_db.sql', 'r') as f:
    query_add_to_db = f.read()

DEBUG_MODE=True
setDebugging(DEBUG_MODE)
CUSTOM_USER_AGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/49.0.2623.108 Chrome/49.0.2623.108 Safari/537.36'

def set_up_client():
    # set up client with:
    #
    # set Twisted debug level
    # connect to the db
    global dbconn, dbd, dbpool
    DEBUG_MODE = True
    setDebugging(DEBUG_MODE)
    log.startLogging(sys.stdout)
    # db connection and deferred
    dbconn = txpostgres.Connection()
    dbd = dbconn.connect("dbname=news user=user password=test1 host=127.0.0.1 application_name='news-collect'")
    dbpool = adbapi.ConnectionPool('psycopg2', dbname='news', user='user', password='test1')

def clean_up_and_exit(*args, **kwargs):
    global dbconn
    # print "clean_up_and_exit"
    # print args

    log.msg('clean_up_and_exit')
    dbconn.close()
    dbpool.close()
    reactor.stop()

def parse_date(date1, date2):
    # edge-case for feed
    # pubdate is assumed to be a time.struct_time here
    pubdate_fmt = None

    pubdate = date1
    if pubdate:
        try:
            pubdate_fmt = strftime('%Y-%m-%d %H:%M:%S +0000',pubdate)
        except Exception, e:
            print e
            pubdate = None

    # edge-case for atom feed
    # e.get('updated') 2011-02-01T20:21:42+00:00
    pubdate = date2
    if pubdate and not pubdate_fmt:
        try:
            i1 = pubdate[:19]
            i1 = datetime.strptime(i1, "%Y-%m-%dT%H:%M:%S")
            pubdate_fmt = i1.strftime('%Y-%m-%d %H:%M:%S +0000')
        except Exception, e:
            print e
            pubdate = None

    return pubdate_fmt

# parse feeds and insert them in the db

# fast inserts-related
# https://trvrm.github.io/bulk-psycopg2-inserts.html
# https://nelsonslog.wordpress.com/2015/04/27/inserting-lots-of-data-into-a-remote-postgres-efficiently/
# http://stackoverflow.com/a/23324727/827519
# http://stackoverflow.com/a/10147451/827519
def store_fetched_data(data):
    # log.msg(data)
    # reactor.callLater(0, lambda a,b: None, 0, d)
    global dbconn, dbpool, query_add_to_db
    if data:
        query_data = []
        log.msg('store_fetched_data')
        for f in data:
            if f and 'response' in f and f['response'] is not None:
                try:
                    r = f['response']
                    # log.msg('len response: ' + str(len(r)))
                    feed = feedparser.parse(r)
                    if 'entries' in feed:
                        log.msg('entries: ' + str(len(feed['entries'])))
                        for e in feed['entries']:
                            try:
                                title = e.get('title','')
                                summary = e.get('summary','')

                                date1 = e.get('published_parsed',None)
                                date2 = e.get('updated', None)

                                pubdate_fmt = parse_date(date1, date2)

                                if not pubdate_fmt:
                                    continue

                                # query_data.append((e['title'],e['summary'],pubdate_fmt,))
                                # log.msg('cnt:' + str(cnt))
                                query_data.append({
                                    'title': title,
                                    'summary': summary,
                                    'pubdate': pubdate_fmt,
                                    })
                            except Exception, e:
                                print e
                                pass
                except Exception, e:
                    print e
                    pass
            else:
                print 'feed with empty response'

        len_query_data = len(query_data)
        if len_query_data == 0:
            # there's nothing to insert in the database
            # so we exit early
            pass
        else:
            titles_ = map(lambda e: e['title'], query_data)
            summaries_ = map(lambda e: e['summary'], query_data)
            pubdates_ = map(lambda e: e['pubdate'], query_data)
            query_column_data = {'titles': titles_, 'contents': summaries_, 'pubdates': pubdates_}
            # d = dbpool.runOperation(query_add_to_db, query_column_data)
            # return d
            pass
    else:
        # the request was empty
        print 'store_fetched_data received None'
        pass

def fetch_single(feed_meta=None):

    # TODO: treq.collect is key here. it's called multiple times
    # for each chunk. the response is streaming.

    def fetched_callback(r):
        w = feed_meta
        # print 'here -> ' + str(r.content())
        print r
        # print type(r)
        w['response'] = ''
        return w

    print "scheduling new request ", feed_meta
    d = treq.get(
            feed_meta['feed_url'],
            timeout=3,
            headers={'User-Agent': CUSTOM_USER_AGENT},
            allow_redirects=True,
            persistent=True,
            unbuffered=True
            )
    d.addCallback(treq.collect, fetched_callback)
    # d.addCallback(fetched_callback)
    d.addBoth(lambda x: None)
    # d.addBoth(lambda x: x)
    return d


def batch_gen(data, batch_size):
    for i in range(0, len(data), batch_size):
        yield data[i:i+batch_size]

def fetch_all(feeds):
    BATCH_SIZE=5
    batches = []
    for feeds_batch in batch_gen(feeds, BATCH_SIZE):
        sem = DeferredSemaphore(len(feeds_batch))
        batch = []
        for feed_ in feeds_batch:
            batch.append(sem.run(fetch_single, feed_meta=feed_))
        batchDef = gatherResults(batch, consumeErrors=False)
        batchDef.addCallback(store_fetched_data)
        batches.append(batchDef)

    # rendez-vous for all feeds that were fetched
    batchesDef = gatherResults(batches, consumeErrors=False)

    batchesDef.addCallbacks(
            clean_up_and_exit,
            errback=lambda x: None,
            )
    return batchesDef

def parse_feed_config_line(line):
    parts = re.split(r'\s+', line)
    url = parts[0]
    tags = map(lambda t: re.sub('[\'"]+','', t), parts[1:])
    return {
            'feed_url': url,
            'tags': tags,
            }

def parse_config_data(path=None):
    feeds = []

    with open('./urls','r') as f:
        # entrylim = 99999999
        entrylim = 10
        entrynum = 1
        for line in f:
            if re.match(r'^http', line):
                line = line.rstrip()
                o = parse_feed_config_line(line)
                feeds.append(o)
                entrynum += 1
                if entrynum > entrylim:
                    break

    return feeds

set_up_client()
feeds_metadata = parse_config_data('urls')
d = fetch_all(feeds_metadata)
reactor.run()

