import scrapy
import datetime

class ACLU(scrapy.Spider):
    name = "aclu"
    start_urls = [
        'https://www.aclu.org/action',
    ]

    def parse(self, response):
        topic_urls = response.css('div.field-action-link a::attr(href)').extract()
        for t_url in topic_urls:
            yield scrapy.Request(t_url, callback=self.parse_topic)

    def get_petition_number(self, text):
        text = text.replace('  ','').replace('\n','').replace(',','')
        text = text.split(' ')[0]
        return int(text)

    def parse_topic(self, response):
        result = {'base_url': self.start_urls[0], 'action_url': response.url, 'org': self.name, 'scrape_time': str(datetime.datetime.utcnow())}
        #message type
        result['title'] = response.css('span.xc_largetext b::text').extract_first()
        result['description'] = ' '.join(response.xpath('//span[@class="xc_maintext"]/p//text()').extract())
        result['photo_url'] = response.css('span.xc_maintext p img::attr(src)').extract_first()
        if result['description'] == '':
            #phone call type
            result['title'] = response.xpath('//div[@class="component feature-title"]/h1//text()').extract_first()
            result['description'] = ' '.join(response.xpath('//div[@class="field-media-media-desc text-area"]/p//text()').extract())
            result['photo_url'] = response.css('div.field-media-media-image img::attr(src)').extract_first()
        if result['description'] == '':
            #petition type
            result['title'] = response.css('h1.page-title::text').extract_first()
            result['description'] = ' '.join(response.xpath('//div[@id="node-body-field"]/p//text()').extract())
            result['petition_signed'] = self.get_petition_number(response.css('span.goal-label-signed::text').extract_first())
            result['petition_needed'] = self.get_petition_number(response.css('span.goal-label-needed::text').extract_first())
        return result
