import time
import re
import scrapy
from scrapy.http import Request

from polymerization_spiders.items import ArticleItem


class BaiJiaSpider(scrapy.Spider):
    name = 'baijia'
    download_delay = 2
    allowed_demains = 'baijia.baidu.com'
    start_urls = ['http://baijia.baidu.com/']

    def parse(self, response):
        for item in response.xpath(
                '//div[@class="feeds"]/div[@class="feeds-item hasImg"]'):
            url = item.xpath('h3/a/@href').extract_first().strip()
            yield Request(url, callback=self.parse_context)

    def parse_context(self, response):
        article_item = ArticleItem()
        _article = response.xpath('//div[@class="l-main-col"]/div[@id="page"]')

        title = _article.xpath('h1/text()').extract_first().strip()

        _article_meta = _article.xpath(
            'div[@class="article-infos"]/div[@class="article-info"]')[0]

        _year = time.strftime('%Y', time.localtime(time.time()))
        _date = _article_meta.xpath(
            '//span[@class="time"]/text()').extract_first().strip()
        _tmp = '%s %s' % (re.sub('\W', '-', _date.split(" ")[0])[:-1],
                          _date.split(" ")[1])
        date = '%s-%s' % (_year, _tmp)

        author = _article_meta.xpath('a/text()').extract_first().strip()
        context = _article.xpath('div[@class="article-detail"]').extract_first(
        )

        article_item['title'] = title
        article_item['date'] = date
        article_item['author'] = author
        article_item['context'] = context
        article_item['url'] = response.url
        article_item['iid'] = '%s:::%s' % (self.name, response.url)
        return article_item
