import re
import lxml
from pupa.scrape import Person, Scraper
from openstates.utils import LXMLMixin, validate_email_address
from .utils import LXMLMixinOK


class OKPersonScraper(Scraper, LXMLMixin, LXMLMixinOK):

    _parties = {'R': 'Republican', 'D': 'Democratic', 'I': 'Independent'}

    def _scrub(self, text):
        """Squish whitespace and kill \xa0."""
        return re.sub(r'[\s\xa0]+', ' ', text)

    def _clean_office_info(self, office_info):
        office_info = list(map(self._scrub, office_info.itertext()))
        # Throw away anything after any email address, phone number, or
        # address lines.
        while office_info:
            last = office_info[-1]
            if ('@' not in last
                    and ', OK' not in last
                    and not re.search(r'[\d\-\(\) ]{7,}', last)):
                office_info.pop()
            else:
                break
        return office_info

    def _extract_phone(self, office_info):
        phone = None

        for line in office_info:
            phone_match = re.search(r'''(\(\d{3}\) \d{3}-\d{4}|
                \d{3}.\d{3}.\d{4})''', line)
            if phone_match is not None:
                phone = phone_match.group(1).strip()

        return phone

    def _extract_email(self, doc):
        xpath = '//div[@class="districtheadleft"]' \
                + '/b[contains(text(), "Email:")]' \
                + '/../following-sibling::div' \
                + '/script/text()'
        script = doc.xpath(xpath)
        if not script:
            return ''
        script = script[0]
        line = filter(
            lambda line: '+ "@" +' in line,
            script.split('\r\n'))[0]
        parts = re.findall(r'"(.+?)"', line)

        email = ''.join(parts)

        return email if validate_email_address(email) else ''

    def scrape(self, chamber=None):
        term = self.jurisdiction.legislative_sessions[-1]['identifier']
        chambers = [chamber] if chamber is not None else ['upper', 'lower']
        for chamber in chambers:
            yield from getattr(self, 'scrape_' + chamber + '_chamber')(term)

    def scrape_lower_chamber(self, term):
        url = "https://www.okhouse.gov/Members/Default.aspx"
        page = self.curl_lxmlize(url)

        legislator_nodes = self.get_nodes(
            page,
            '//table[@id="ctl00_ContentPlaceHolder1_RadGrid1_ctl00"]/tbody/tr')

        for legislator_node in legislator_nodes:
            name_node = self.get_node(
                legislator_node,
                './/td[1]/a')

            if name_node is not None:
                name_text = name_node.text.strip()

                # Handle seats with no current representative
                if re.search(r'District \d+', name_text):
                    continue

                last_name, delimiter, first_name = name_text.partition(',')

                if last_name is not None and first_name is not None:
                    first_name = first_name.strip()
                    last_name = last_name.strip()
                    name = ' '.join([first_name, last_name])
                else:
                    raise ValueError('Unable to parse name: {}'.format(
                        name_text))

                if name.startswith('House District'):
                    continue

            district_node = self.get_node(
                legislator_node,
                './/td[3]')

            if district_node is not None:
                district = district_node.text.strip()

            party_node = self.get_node(
                legislator_node,
                './/td[4]')

            if party_node is not None:
                party_text = party_node.text.strip()

            party = self._parties[party_text]

            legislator_url = 'https://www.okhouse.gov/District.aspx?District=' + district
            legislator_page = self.curl_lxmlize(legislator_url)

            photo_url = self.get_node(
                legislator_page,
                '//a[@id="ctl00_ContentPlaceHolder1_imgHiRes"]/@href')

            person = Person(primary_org='lower',
                            district=district,
                            name=name,
                            party=party,
                            image=photo_url)
            person.extras['_scraped_name'] = name_text
            person.add_link(legislator_url)
            person.add_source(url)
            person.add_source(legislator_url)

            # Scrape offices.
            self.scrape_lower_offices(legislator_page, person)

            yield person

    def scrape_lower_offices(self, doc, person):

        # Capitol offices:
        xpath = '//*[contains(text(), "Capitol Address")]'
        for bold in doc.xpath(xpath):

            # Get the address.
            address_div = next(bold.getparent().itersiblings())

            # Get the room number.
            xpath = '//*[contains(@id, "CapitolRoom")]/text()'
            room = address_div.xpath(xpath)
            if room:
                parts = map(self._scrub, list(address_div.itertext()))
                parts = [x.strip() for x in parts if x.strip()]
                phone = parts.pop()
                parts = [parts[0], 'Room ' + room[0], parts[-1]]
                address = '\n'.join(parts)
            else:
                address = None
                phone = None

            if not phone:
                phone = None

            # Get the email address, extracted from a series of JS
            # "document.write" lines.
            email = self._extract_email(doc)
            if email:
                person.add_contact_detail(type='email', value=email,
                                          note='Capitol Office')
                person.extras['email'] = email
            if phone:
                person.add_contact_detail(type='voice', value=str(phone),
                                          note='Capitol Office')
            if address:
                person.add_contact_detail(type='address', value=address,
                                          note='Capitol Office')

        # District offices only have address, no other information
        district_address = doc.xpath('//span[@id="ctl00_Content'
                                     'PlaceHolder1_lblDistrictAddress"]/text()')
        if district_address:
            district_city_state,  = doc.xpath('//span[@id="ctl00_Content'
                                              'PlaceHolder1_lblDistrictCity"]/text()')
            district_address = "{}\n{}".format(district_address[0], district_city_state)
            if district_address:
                person.add_contact_detail(type='address', value=district_address,
                                          note='District Office')

    def scrape_upper_chamber(self, term):
        url = "http://oksenate.gov/Senators/Default.aspx"
        html = self.get(url).text
        doc = lxml.html.fromstring(html)
        doc.make_links_absolute(url)

        for a in doc.xpath('//table[@summary]')[0]. \
                xpath('.//td//a[contains(@href, "biographies")]'):
            tail = a.xpath('..')[0].tail
            if tail:
                district = tail.split()[1]
            else:
                district = a.xpath('../../span')[1].text.split()[1]

            if a.text is None or a.text.strip() == 'Vacant':
                self.warning("District {} appears to be empty".format(district))
                continue
            else:
                match = re.match(r'(.+) \(([A-Z])\)', a.text.strip())
                name, party = match.group(1), self._parties[match.group(2)]

            url = a.get('href')

            person = Person(primary_org='upper',
                            district=district,
                            name=name.strip(),
                            party=party,
                            )
            person.add_link(url)
            person.add_source(url)
            self.scrape_upper_offices(person, url)
            yield person

    def scrape_upper_offices(self, person, url):
        url = url.replace('aspx', 'html')
        html = self.get(url).text
        person.add_source(url)
        doc = lxml.html.fromstring(html)
        doc.make_links_absolute(url)

        try:
            xpath = '//h3[contains(., "Office")]'
            for table in doc.xpath(xpath)[0].itersiblings():
                if table.tag == 'table':
                    break
        except IndexError:
            self.warning('invalid bio page for %s', person)
            return
        col1, col2 = table.xpath('tr[2]/td')
        lxml.etree.strip_tags(col1, 'sup')
        lxml.etree.strip_tags(col2, 'sup')

        capitol_office_info = self._clean_office_info(col1)

        # Set email on the leg object.
        if capitol_office_info:
            if '@' in capitol_office_info[-1]:
                email = capitol_office_info.pop()
                person.extras['email'] = email
            else:
                email = None

            capitol_phone = self._extract_phone(capitol_office_info)

            capitol_address_lines = map(
                lambda line: line.strip(),
                filter(
                    lambda string: re.search(r', OK|Lincoln Blvd|Room \d', string),
                    capitol_office_info))

            if email:
                person.add_contact_detail(type='email', value=email,
                                          note='Capitol Office')
            if capitol_phone:
                person.add_contact_detail(type='voice', value=str(capitol_phone),
                                          note='Capitol Office')

            capitol_address = '\n'.join(capitol_address_lines)
            if capitol_address:
                person.add_contact_detail(type='address', value=capitol_address,
                                          note='Capitol Office')

        district_office_info = self._clean_office_info(col2)
        # This probably isn't a valid district office at less than two lines.
        if len(district_office_info) < 2:
            return

        district_address_lines = []
        for line in district_office_info:
            district_address_lines.append(line.strip())
            if 'OK' in line:
                break

        if 'OK' in district_address_lines[-1]:
            district_address = '\n'.join(filter(lambda line: line,
                                         district_address_lines))
        else:
            district_address = None
        # self.logger.debug(district_address)

        district_phone = self._extract_phone(district_office_info)

        if capitol_phone:
            person.add_contact_detail(type='voice', value=str(district_phone),
                                      note='District Office')
        if capitol_address_lines:
            person.add_contact_detail(type='address', value=district_address,
                                      note='District Office')

openstates/ok/people.py
import re
import lxml
from pupa.scrape import Person, Scraper
from openstates.utils import LXMLMixin, validate_email_address
from .utils import LXMLMixinOK


class OKPersonScraper(Scraper, LXMLMixin, LXMLMixinOK):

    _parties = {'R': 'Republican', 'D': 'Democratic', 'I': 'Independent'}

    def _scrub(self, text):
        """Squish whitespace and kill \xa0."""
        return re.sub(r'[\s\xa0]+', ' ', text)

    def _clean_office_info(self, office_info):
        office_info = list(map(self._scrub, office_info.itertext()))
        # Throw away anything after any email address, phone number, or
        # address lines.
        while office_info:
            last = office_info[-1]
            if ('@' not in last
                    and ', OK' not in last
                    and not re.search(r'[\d\-\(\) ]{7,}', last)):
                office_info.pop()
            else:
                break
        return office_info

    def _extract_phone(self, office_info):
        phone = None

        for line in office_info:
            phone_match = re.search(r'''(\(\d{3}\) \d{3}-\d{4}|
                \d{3}.\d{3}.\d{4})''', line)
            if phone_match is not None:
                phone = phone_match.group(1).strip()

        return phone

    def _extract_email(self, doc):
        xpath = '//div[@class="districtheadleft"]' \
                + '/b[contains(text(), "Email:")]' \
                + '/../following-sibling::div' \
                + '/script/text()'
        script = doc.xpath(xpath)
        if not script:
            return ''
        script = script[0]
        line = filter(
            lambda line: '+ "@" +' in line,
            script.split('\r\n'))[0]
        parts = re.findall(r'"(.+?)"', line)

        email = ''.join(parts)

        return email if validate_email_address(email) else ''

    def scrape(self, chamber=None):
        term = self.jurisdiction.legislative_sessions[-1]['identifier']
        chambers = [chamber] if chamber is not None else ['upper', 'lower']
        for chamber in chambers:
            yield from getattr(self, 'scrape_' + chamber + '_chamber')(term)

    def scrape_lower_chamber(self, term):
        url = "https://www.okhouse.gov/Members/Default.aspx"
        page = self.curl_lxmlize(url)

        legislator_nodes = self.get_nodes(
            page,
            '//table[@id="ctl00_ContentPlaceHolder1_RadGrid1_ctl00"]/tbody/tr')

        for legislator_node in legislator_nodes:
            name_node = self.get_node(
                legislator_node,
                './/td[1]/a')

            if name_node is not None:
                name_text = name_node.text.strip()

                # Handle seats with no current representative
                if re.search(r'District \d+', name_text):
                    continue

                last_name, delimiter, first_name = name_text.partition(',')

                if last_name is not None and first_name is not None:
                    first_name = first_name.strip()
                    last_name = last_name.strip()
                    name = ' '.join([first_name, last_name])
                else:
                    raise ValueError('Unable to parse name: {}'.format(
                        name_text))

                if name.startswith('House District'):
                    continue

            district_node = self.get_node(
                legislator_node,
                './/td[3]')

            if district_node is not None:
                district = district_node.text.strip()

            party_node = self.get_node(
                legislator_node,
                './/td[4]')

            if party_node is not None:
                party_text = party_node.text.strip()

            party = self._parties[party_text]

            legislator_url = 'https://www.okhouse.gov/Members/District.aspx?District=' + district
            legislator_page = self.curl_lxmlize(legislator_url)

            photo_url = self.get_node(
                legislator_page,
                '//a[@id="ctl00_ContentPlaceHolder1_imgHiRes"]/@href')

            person = Person(primary_org='lower',
                            district=district,
                            name=name,
                            party=party,
                            image=photo_url)
            person.extras['_scraped_name'] = name_text
            person.add_link(legislator_url)
            person.add_source(url)
            person.add_source(legislator_url)

            # Scrape offices.
            self.scrape_lower_offices(legislator_page, person)

            yield person

    def scrape_lower_offices(self, doc, person):

        # Capitol offices:
        xpath = '//*[contains(text(), "Capitol Address")]'
        for bold in doc.xpath(xpath):

            # Get the address.
            address_div = next(bold.getparent().itersiblings())

            # Get the room number.
            xpath = '//*[contains(@id, "CapitolRoom")]/text()'
            room = address_div.xpath(xpath)
            if room:
                parts = map(self._scrub, list(address_div.itertext()))
                parts = [x.strip() for x in parts if x.strip()]
                phone = parts.pop()
                parts = [parts[0], 'Room ' + room[0], parts[-1]]
                address = '\n'.join(parts)
            else:
                address = None
                phone = None

            if not phone:
                phone = None

            # Get the email address, extracted from a series of JS
            # "document.write" lines.
            email = self._extract_email(doc)
            if email:
                person.add_contact_detail(type='email', value=email,
                                          note='Capitol Office')
                person.extras['email'] = email
            if phone:
                person.add_contact_detail(type='voice', value=str(phone),
                                          note='Capitol Office')
            if address:
                person.add_contact_detail(type='address', value=address,
                                          note='Capitol Office')

        # District offices only have address, no other information
        district_address = doc.xpath('//span[@id="ctl00_Content'
                                     'PlaceHolder1_lblDistrictAddress"]/text()')
        if district_address:
            district_city_state,  = doc.xpath('//span[@id="ctl00_Content'
                                              'PlaceHolder1_lblDistrictCity"]/text()')
            district_address = "{}\n{}".format(district_address[0], district_city_state)
            if district_address:
                person.add_contact_detail(type='address', value=district_address,
                                          note='District Office')

    def scrape_upper_chamber(self, term):
        url = "http://oksenate.gov/Senators/Default.aspx"
        html = self.get(url).text
        doc = lxml.html.fromstring(html)
        doc.make_links_absolute(url)

        for a in doc.xpath('//table[@summary]')[0]. \
                xpath('.//td//a[contains(@href, "biographies")]'):
            tail = a.xpath('..')[0].tail
            if tail:
                district = tail.split()[1]
            else:
                district = a.xpath('../../span')[1].text.split()[1]

            if a.text is None or a.text.strip() == 'Vacant':
                self.warning("District {} appears to be empty".format(district))
                continue
            else:
                match = re.match(r'(.+) \(([A-Z])\)', a.text.strip())
                name, party = match.group(1), self._parties[match.group(2)]

            url = a.get('href')

            person = Person(primary_org='upper',
                            district=district,
                            name=name.strip(),
                            party=party,
                            )
            person.add_link(url)
            person.add_source(url)
            self.scrape_upper_offices(person, url)
            yield person

    def scrape_upper_offices(self, person, url):
        url = url.replace('aspx', 'html')
        html = self.get(url).text
        person.add_source(url)
        doc = lxml.html.fromstring(html)
        doc.make_links_absolute(url)

        try:
            xpath = '//h3[contains(., "Office")]'
            for table in doc.xpath(xpath)[0].itersiblings():
                if table.tag == 'table':
                    break
        except IndexError:
            self.warning('invalid bio page for %s', person)
            return
        col1, col2 = table.xpath('tr[2]/td')
        lxml.etree.strip_tags(col1, 'sup')
        lxml.etree.strip_tags(col2, 'sup')

        capitol_office_info = self._clean_office_info(col1)

        # Set email on the leg object.
        if capitol_office_info:
            if '@' in capitol_office_info[-1]:
                email = capitol_office_info.pop()
                person.extras['email'] = email
            else:
                email = None

            capitol_phone = self._extract_phone(capitol_office_info)

            capitol_address_lines = map(
                lambda line: line.strip(),
                filter(
                    lambda string: re.search(r', OK|Lincoln Blvd|Room \d', string),
                    capitol_office_info))

            if email:
                person.add_contact_detail(type='email', value=email,
                                          note='Capitol Office')
            if capitol_phone:
                person.add_contact_detail(type='voice', value=str(capitol_phone),
                                          note='Capitol Office')

            capitol_address = '\n'.join(capitol_address_lines)
            if capitol_address:
                person.add_contact_detail(type='address', value=capitol_address,
                                          note='Capitol Office')

        district_office_info = self._clean_office_info(col2)
        # This probably isn't a valid district office at less than two lines.
        if len(district_office_info) < 2:
            return

        district_address_lines = []
        for line in district_office_info:
            district_address_lines.append(line.strip())
            if 'OK' in line:
                break

        if 'OK' in district_address_lines[-1]:
            district_address = '\n'.join(filter(lambda line: line,
                                         district_address_lines))
        else:
            district_address = None
        # self.logger.debug(district_address)

        district_phone = self._extract_phone(district_office_info)

        if capitol_phone:
            person.add_contact_detail(type='voice', value=str(district_phone),
                                      note='District Office')
        if capitol_address_lines:
            person.add_contact_detail(type='address', value=district_address,
                                      note='District Office')

openstates/ok/people.py
diff --git a/openstates/ok/people.py b/openstates/ok/people.py
index 4364385747..e6c154711a 100644
--- a/openstates/ok/people.py
+++ b/openstates/ok/people.py
@@ -111,7 +111,7 @@ def scrape_lower_chamber(self, term):
 
             party = self._parties[party_text]
 
-            legislator_url = 'https://www.okhouse.gov/District.aspx?District=' + district
+            legislator_url = 'https://www.okhouse.gov/Members/District.aspx?District=' + district
             legislator_page = self.curl_lxmlize(legislator_url)
 
             photo_url = self.get_node(

OK failing since at least 2017-07-29
OK has been failing since 2017-07-29

Based on automated runs it appears that OK has not run successfully in 2 days (2017-07-29).


```
  02:10:09 INFO pupa: save person Roger Thompson as person_1e8475c0-74f6-11e7-858b-0242ac110005.json
02:10:09 INFO pupa: save membership 1e8475c0-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "upper"} as membership_1e84796c-74f6-11e7-858b-0242ac110005.json
02:10:09 INFO pupa: save membership 1e8475c0-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "party", "name": "Republican"} as membership_1e847b92-74f6-11e7-858b-0242ac110005.json
02:10:09 INFO scrapelib: GET - http://oksenate.gov/Senators/biographies/treat_bio.html
02:10:10 INFO pupa: save person Greg Treat as person_1f1d4a98-74f6-11e7-858b-0242ac110005.json
02:10:10 INFO pupa: save membership 1f1d4a98-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "upper"} as membership_1f1d4e58-74f6-11e7-858b-0242ac110005.json
02:10:10 INFO pupa: save membership 1f1d4a98-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "party", "name": "Republican"} as membership_1f1d5074-74f6-11e7-858b-0242ac110005.json
02:10:10 INFO scrapelib: GET - http://oksenate.gov/Senators/biographies/yen_bio.html
02:10:11 INFO pupa: save membership 1fb7ab60-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "upper"} as membership_1fb7b04c-74f6-11e7-858b-0242ac110005.json
02:10:11 INFO pupa: save person Ervin Yen as person_1fb7ab60-74f6-11e7-858b-0242ac110005.json
02:10:11 INFO pupa: save membership 1fb7ab60-74f6-11e7-858b-0242ac110005 membership in ~{"classification": "party", "name": "Republican"} as membership_1fb7b326-74f6-11e7-858b-0242ac110005.json
02:10:11 INFO pupa: GET via curl subprocess: https://www.okhouse.gov/Members/Default.aspx
02:10:12 INFO pupa: GET via curl subprocess: https://www.okhouse.gov/District.aspx?District=33
02:10:12 INFO pupa: save person Greg Babinec as person_2113c200-74f6-11e7-858b-0242ac110005.json
02:10:12 WARNING pupa: validation of Person 2113c200-74f6-11e7-858b-0242ac110005 failed: 1 validation errors:
Value None for field '<obj>.image' is not of type string
no pupa_settings on path, using defaults
ok (scrape, import)
  bills: {}
  people: {}
  committees: {}
Value None for field '<obj>.image' is not of type string
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 175, in validate
    validator.validate(self.as_dict(), schema)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/validictory/validator.py", line 620, in validate
    raise MultipleValidationError(self._errors)
validictory.validator.MultipleValidationError: 1 validation errors:
Traceback (most recent call last):
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/__main__.py", line 67, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 260, in handle
    return self.do_handle(args, other, juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
    report['scrape'] = self.do_scrape(juris, args, scrapers)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
    report[scraper_name] = scraper.do_scrape(**scrape_args)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 104, in do_scrape
    self.save_object(obj)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 89, in save_object
    raise ve
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 85, in save_object
    obj.validate()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 178, in validate
    self.__class__.__name__, self._id, ve)
pupa.exceptions.ScrapeValueError: validation of Person 2113c200-74f6-11e7-858b-0242ac110005 failed: 1 validation errors:
Value None for field '<obj>.image' is not of type string
```

Visit http://bobsled.openstates.org for more info.

