# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem
from hashlib import md5
from scrapy import log
from twisted.enterprise import adbapi
from scrapy_template.items import ScrapyTemplateItem


class ScrapyTemplatePipeline(object):
    def __init__(self, dbpool):
        self.urls_seen = set()
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbargs = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)
        return cls(dbpool)

    def process_item(self, item, spider):
        if isinstance(item, ScrapyTemplateItem):
            if item['url'] in self.urls_seen:
                raise DropItem("Duplicate item found: %s" % item['url'])
            else:
                self.urls_seen.add(item['url'])
                d = self.dbpool.runInteraction(self._do_upsert, item, spider)
                d.addErrback(self._handle_error, item, spider)
                d.addBoth(lambda _: item)
                return d
        else:
            return item

    def _do_upsert(self, conn, item, spider):
        guid = self._get_id(item)

        conn.execute("""SELECT EXISTS(
            SELECT 1 FROM example WHERE guid = %s
        )""", (guid, ))
        ret = conn.fetchone()[0]
        if not ret:
            conn.execute("""
                INSERT INTO example (category, name, color, images, price, url, guid)
                VALUES (%s, %s, %s, %s, %s, %s, %s)
            """, (item['category'], item['name'], item['color'],
                  item['images'], item['price'], item['url'], guid))
            spider.log("Item stored in db: %s %r" % (guid, item))

    def _handle_error(self, failure, item, spider):
        log.err(failure)

    def _get_id(self, item):
        return md5(item['url']).hexdigest()
[((29, 17, 29, 59), 'twisted.enterprise.adbapi.ConnectionPool', 'adbapi.ConnectionPool', ({(29, 39, 29, 48): '"""MySQLdb"""'}, {}), "('MySQLdb', **dbargs)", False, 'from twisted.enterprise import adbapi\n'), ((61, 8, 61, 24), 'scrapy.log.err', 'log.err', ({(61, 16, 61, 23): 'failure'}, {}), '(failure)', False, 'from scrapy import log\n'), ((35, 22, 35, 72), 'scrapy.exceptions.DropItem', 'DropItem', ({(35, 31, 35, 71): "('Duplicate item found: %s' % item['url'])"}, {}), "('Duplicate item found: %s' % item['url'])", False, 'from scrapy.exceptions import DropItem\n'), ((64, 15, 64, 31), 'hashlib.md5', 'md5', ({(64, 19, 64, 30): "item['url']"}, {}), "(item['url'])", False, 'from hashlib import md5\n')]