import json
import re
from string import ascii_uppercase
from time import time
from urllib.parse import urljoin

import scrapy
from more_itertools import first
from scrapy import Request

from product_spider.items import JkProduct, JKPackage
from product_spider.utils.functions import strip


class JkPrdSpider(scrapy.Spider):
    name = "jk"
    allowed_domains = ["jkchemical.com"]
    base_url = "http://www.jkchemical.com"
    start_urls = map(lambda x: "http://www.jkchemical.com/CH/products/index/ProductName/{0}.html".format(x),
                     ascii_uppercase)
    prd_size_url = "http://www.jkchemical.com/Controls/Handler/GetPackAgeJsonp.ashx?callback=py27&value={value}&cid={cid}&type=product&_={ts}"

    def parse(self, response):
        for xp_url in response.xpath("//div[@class='yy toa']//a/@href"):
            tmp_url = self.base_url + xp_url.extract()
            yield Request(tmp_url.replace("EN", "CH"), callback=self.parse_list)

    def parse_list(self, response):
        xp_boxes = response.xpath("//table[@id]//div[@class='PRODUCT_box']")
        for xp_box in xp_boxes:
            div = xp_box.xpath(".//div[2][@class='left_right mulu_text']")
            brand = strip(div.xpath('.//li[@id="ctl00_cph_Content_li_lt_Brand"]/text()').get(), '')
            rel_url = div.xpath('.//a[@class="name"]/@href').get()
            img_url = div.xpath('.//img/@src').get()
            d = {
                'brand': brand.replace('-', '') or None,
                "purity": div.xpath(".//li[1]/text()").get('').split(u"：")[-1].strip(),
                "cas": strip(div.xpath(".//li[2]//a/text()").get()),
                "cat_no": div.xpath(".//li[4]/text()").get().split(u"：")[-1].strip(),
                "en_name": strip(xp_box.xpath(".//a[@class='name']/text()").get()),
                "cn_name": strip(xp_box.xpath(".//a[@class='name']//span[1]/text()").get()),
                'prd_url': rel_url and urljoin(response.url, rel_url),
                'img_url': img_url and urljoin(response.url, img_url),
            }
            data_jkid = xp_box.xpath(".//div[@data-jkid]/@data-jkid").get()
            data_cid = xp_box.xpath(".//div[@data-cid]/@data-cid").get()

            yield Request(self.prd_size_url.format(value=data_jkid, cid=data_cid, ts=int(time())),
                          body=u"",
                          meta={"prd_data": d},
                          callback=self.parse_package)

        next_page = response.xpath('//a[contains(text(), "下一页")]/@href').get()
        if next_page:
            yield Request(urljoin(response.url, next_page), callback=self.parse_list)

    def parse_package(self, response):
        s = re.findall(r"(?<=\().+(?=\))", response.text)[0]
        packages = json.loads(s)
        d = response.meta.get('prd_data', {})
        package = first(packages, {})
        if package:
            d['brand'] = d['brand'] or package.get('Product', {}).get('BrandName')
        yield JkProduct(**d)
        for package_obj in packages:
            catalog_price = package_obj.get("CatalogPrice", {})
            dd = {
                'brand': d.get('brand'),
                'cat_no': d.get('cat_no'),
                'package': package_obj.get("stringFormat"),
                'price': catalog_price and catalog_price.get('Value'),
                'currency': catalog_price and strip(catalog_price.get('Currency')),
                'attrs': json.dumps(package_obj),
            }
            yield JKPackage(**dd)

[((59, 19, 59, 32), 'json.loads', 'json.loads', ({(59, 30, 59, 31): 's'}, {}), '(s)', False, 'import json\n'), ((61, 18, 61, 37), 'more_itertools.first', 'first', ({(61, 24, 61, 32): 'packages', (61, 34, 61, 36): '{}'}, {}), '(packages, {})', False, 'from more_itertools import first\n'), ((58, 12, 58, 57), 're.findall', 're.findall', ({(58, 23, 58, 41): '"""(?<=\\\\().+(?=\\\\))"""', (58, 43, 58, 56): 'response.text'}, {}), "('(?<=\\\\().+(?=\\\\))', response.text)", False, 'import re\n'), ((64, 14, 64, 28), 'product_spider.items.JkProduct', 'JkProduct', ({}, {}), '(**d)', False, 'from product_spider.items import JkProduct, JKPackage\n'), ((73, 25, 73, 48), 'json.dumps', 'json.dumps', ({(73, 36, 73, 47): 'package_obj'}, {}), '(package_obj)', False, 'import json\n'), ((75, 18, 75, 33), 'product_spider.items.JKPackage', 'JKPackage', ({}, {}), '(**dd)', False, 'from product_spider.items import JkProduct, JKPackage\n'), ((42, 39, 42, 69), 'urllib.parse.urljoin', 'urljoin', ({(42, 47, 42, 59): 'response.url', (42, 61, 42, 68): 'rel_url'}, {}), '(response.url, rel_url)', False, 'from urllib.parse import urljoin\n'), ((43, 39, 43, 69), 'urllib.parse.urljoin', 'urljoin', ({(43, 47, 43, 59): 'response.url', (43, 61, 43, 68): 'img_url'}, {}), '(response.url, img_url)', False, 'from urllib.parse import urljoin\n'), ((55, 26, 55, 58), 'urllib.parse.urljoin', 'urljoin', ({(55, 34, 55, 46): 'response.url', (55, 48, 55, 57): 'next_page'}, {}), '(response.url, next_page)', False, 'from urllib.parse import urljoin\n'), ((48, 89, 48, 95), 'time.time', 'time', ({}, {}), '()', False, 'from time import time\n')]