text: from congress.tasks import utils
  import os.path
  import os
  import re
  import datetime
  import json
  import lxml.etree
  import uuid
  import logging
  import mechanize
  import zipfile
  import io
  import requests
  import subprocess
  
  from email.utils import parsedate
  from time import mktime
  
  # to get text files their is a new dependency; you need to have pdftotext. 
  # On Ubuntu, apt-get install poppler-utils. On OS X, install it via MacPorts 
  # with port install poppler, or via Homebrew with brew install poppler.
  
  # options:
  #
  #    --chamber: "house" or "senate" to limit the parse to a single chamber
  #    --load_by: Takes a range of House Event IDs. Give it the beginning and end IDs with a dash between, otherwise, it goes by the committee feeds.
  #    --docs=False: Don't download (& convert to text) House committee documents
  
  def run(options):
      # can limit it to one chamber
      chamber = options.get("chamber", None)
      if chamber and (chamber in ("house", "senate")):
          chambers = (chamber)
      else:
          chambers = ("house", "senate")
  
      load_by = options.get("load_by", None)
  
      # Load the committee metadata from the congress-legislators repository and make a
      # mapping from thomas_id and house_id to the committee dict. For each committee,
      # replace the subcommittees list with a dict from thomas_id to the subcommittee.
      utils.require_congress_legislators_repo()
      committees = {}
      for c in utils.yaml_load("congress-legislators/committees-current.yaml"):
          committees[c["thomas_id"]] = c
          if "house_committee_id" in c:
              committees[c["house_committee_id"] + "00"] = c
          c["subcommittees"] = dict((s["thomas_id"], s) for s in c.get("subcommittees", []))
  
      if "senate" in chambers:
          print("Fetching Senate meetings...")
          meetings = fetch_senate_committee_meetings(committees, options)
          print("Writing Senate meeting data to disk.")
          utils.write_json(meetings, output_for("senate"))
  
      if "house" in chambers:
          if load_by == None:
              print("Fetching House meetings...")
              meetings = fetch_house_committee_meetings(committees, options)
          else:
              print("Fetching House meetings by event_id...")
              meetings = fetch_meeting_from_event_id(committees, options, load_by)
  
          print("Writing House meeting data to disk.")
          utils.write_json(meetings, output_for("house"))
  
      # Write all meetings to a single file on disk.
  
  
  # TODO: if these have unique IDs, maybe worth storing a file per-meeting.
  def output_for(chamber):
      return utils.data_dir() + "/committee_meetings_%s.json" % chamber
  
  
  # Parse the Senate committee meeting XML feed for meetings.
  # To aid users of the data, attempt to assign GUIDs to meetings.
  def fetch_senate_committee_meetings(committees, options):
      # Load any existing meetings file so we can recycle any GUIDs.
      existing_meetings = []
      output_file = output_for("senate")
      if os.path.exists(output_file):
          existing_meetings = json.load(open(output_file))
  
      options = dict(options)  # clone
      options["binary"] = True #
      options["force"] = True
  
      meetings = []
  
      dom = lxml.etree.fromstring(utils.download(
          "http://www.senate.gov/general/committee_schedules/hearings.xml",
          "committee_schedule/senate.xml",
          options))
  
      for node in dom.xpath("meeting"):
          committee_id = str(node.xpath('string(cmte_code)'))
          if committee_id.strip() == "":
              continue  # "No committee hearings scheduled" placeholder
          occurs_at = str(node.xpath('string(date)'))
          room = str(node.xpath('string(room)'))
          topic = str(node.xpath('string(matter)'))
  
          occurs_at = datetime.datetime.strptime(occurs_at, "%d-%b-%Y %I:%M %p")
          topic = re.sub(r"\s+", " ", topic).strip()
  
          # Validate committee code.
          try:
              committee_code, subcommittee_code = re.match(r"(\D+)(\d+)$", committee_id).groups()
              if committee_code not in committees:
                  raise ValueError(committee_code)
              if subcommittee_code == "00":
                  subcommittee_code = None
              if subcommittee_code and subcommittee_code not in committees[committee_code]["subcommittees"]:
                  raise ValueError(subcommittee_code)
          except:
              print("Invalid committee code", committee_id)
              continue
  
          # See if this meeting already exists. If so, take its GUID.
          # Assume meetings are the same if they are for the same committee/subcommittee and
          # at the same time.
          for mtg in existing_meetings:
              if mtg["committee"] == committee_code and mtg.get("subcommittee", None) == subcommittee_code and mtg["occurs_at"] == occurs_at.isoformat():
                  if options.get("debug", False):
                      print("[%s] Reusing gUID." % mtg["guid"])
                  guid = mtg["guid"]
                  break
          else:
              # Not found, so create a new ID.
              # TODO: Can we make this a human-readable ID?
              guid = str(uuid.uuid4())
  
          # Scrape the topic text for mentions of bill numbers.
          congress = utils.congress_from_legislative_year(utils.current_legislative_year(occurs_at))
          bills = []
          bill_number_re = re.compile(r"(hr|s|hconres|sconres|hjres|sjres|hres|sres)\s?(\d+)", re.I)
          for bill_match in bill_number_re.findall(topic.replace(".", "")):
              bills.append(bill_match[0].lower() + bill_match[1] + "-" + str(congress))
  
          # Create the meeting event.
          if options.get("debug", False):
              print("[senate][%s][%s] Found meeting in room %s at %s." % (committee_code, subcommittee_code, room, occurs_at.isoformat()))
  
          meetings.append({
              "chamber": "senate",
              "congress": congress,
              "guid": guid,
              "committee": committee_code,
              "subcommittee": subcommittee_code,
              "occurs_at": occurs_at.isoformat(),
              "room": room,
              "topic": topic,
              "bill_ids": bills,
          })
  
      print("[senate] Found %i meetings." % len(meetings))
      return meetings
  
  # House
  
  # Scrape docs.house.gov for meetings.
  # To aid users of the data, assign GUIDs to meetings piggy-backing off of the provided EventID.
  def fetch_house_committee_meetings(committees, options):
      # Load any existing meetings file so we can recycle any GUIDs.
      existing_meetings = []
      output_file = output_for("house")
      if os.path.exists(output_file):
          existing_meetings = json.load(open(output_file))
  
      opts = dict(options)
      opts["binary"] = True
      opts["force"] = True
  
      meetings = []
      seen_meetings = set()
  
      # Scrape the committee listing page for a list of committees with scrapable events.
      committee_html = utils.download("http://docs.house.gov/Committee/Committees.aspx", "committee_schedule/house_overview.html", opts)
      for cmte in re.findall(r'<option value="(....)">', committee_html.decode('utf-8')):
          if cmte not in committees:
              logging.error("Invalid committee code: " + cmte)
              continue
  
          # Download the feed for this committee.
          logging.info("Fetching events for committee " + cmte)
          html = utils.download(
              "http://docs.house.gov/Committee/RSS.ashx?Code=%s" % cmte,
              "committee_schedule/house_%s.xml" % cmte,
              opts)
  
          # It's not really valid?
          html = html.replace(b"&nbsp;", b" ")  # who likes nbsp's? convert to spaces. but otherwise, entity is not recognized.
          #print(html)
          # Parse and loop through the meetings listed in the committee feed.
          dom = lxml.etree.fromstring(html)
          
          # original start to loop
          for mtg in dom.xpath("channel/item"):
  
              eventurl = str(mtg.xpath("string(link)"))
              event_id = re.search(r"EventID=(\d+)$", eventurl)
              if not event_id: continue # weird empty event showed up
              event_id = event_id.group(1)
              pubDate = datetime.datetime.fromtimestamp(mktime(parsedate(mtg.xpath("string(pubDate)"))))
              # skip old records of meetings, some of which just give error pages
              if pubDate < (datetime.datetime.now() - datetime.timedelta(days=60)):
                  continue
  
              # Events can appear in multiple committee feeds if it is a joint meeting.
              if event_id in seen_meetings:
                  logging.info("Duplicated multi-committee event: " + event_id)
                  continue
              seen_meetings.add(event_id)
  
              # this loads the xml from the page and sends the xml to parse_house_committee_meeting
              load_xml_from_page(eventurl, options, existing_meetings, committees, event_id, meetings)
              # if bad zipfile
              if load_xml_from_page == False: continue
  
      print("[house] Found %i meetings." % len(meetings))
      return meetings
  
  
  ## load House meeting sequentially from event_id
  def fetch_meeting_from_event_id(committees, options, load_id):
      existing_meetings = []
      output_file = output_for("house")
      if os.path.exists(output_file):
          existing_meetings = json.load(open(output_file))
  
      opts = dict(options)
      opts["binary"] = True
      opts["force"] = True
  
      meetings = []
      ids = load_id.split('-')
      current_id = int(ids[0])
      end_id = int(ids[1])
  
      while current_id <= end_id:
          event_id = str(current_id)
          event_url = "http://docs.house.gov/Committee/Calendar/ByEvent.aspx?EventID=" + event_id
          load_xml_from_page(event_url, options, existing_meetings, committees, event_id, meetings)
          # bad zipfile
          if load_xml_from_page == False: continue
          current_id += 1
      
      print("[house] Found %i meetings." % len(meetings))
      return meetings
  
  
  def load_xml_from_page(eventurl, options, existing_meetings, committees, event_id, meetings):  
      # Load the HTML page for the event and use the mechanize library to
      # submit the form that gets the meeting XML. TODO Simplify this when
      # the House makes the XML available at an actual URL.
  
      logging.info(eventurl)
      package_info = extract_meeting_package(eventurl, event_id, options)
      if package_info == False: return False
      witnesses = package_info["witnesses"]
      uploaded_documents = package_info["uploaded_documents"]
      dom = package_info["dom"]
  
      # Parse the XML.
      try:
          meeting = parse_house_committee_meeting(event_id, dom, existing_meetings, committees, options, witnesses, uploaded_documents)
          if meeting != None: # an active meeting record
              meetings.append(meeting)
     
      except Exception as e:
          logging.error("Error parsing " + eventurl, exc_info=e)
  
  
  #look for witnesses and documents in the house meeting package    
  def extract_meeting_package(eventurl, event_id, options):
      br = mechanize.Browser()
      # open committee event page
      br.open(eventurl)
  
      br.select_form(nr=0)
  
      # mechanize parser failed to find these fields
      br.form.new_control("hidden", "__EVENTTARGET", {})
      br.form.new_control("hidden", "__EVENTARGUMENT", {})
      br.form.set_all_readonly(False)
  
      # set field values
      if options.get("docs", True):
          # When we want documents, download the whole ZIP package.
          br["__EVENTTARGET"] = "ctl00$MainContent$LinkButtonDownloadMtgPackage"
      else:
          # Otherwise, just download the metadata XML.
          br["__EVENTTARGET"] = "ctl00$MainContent$LinkButtonDownloadMtgXML"
      br["__EVENTARGUMENT"] = ""
      
      # get the info
      request = br.submit()
  
      # when just downloading the metadata XML, return the DOM and no other info
      if not options.get("docs", True):
          try:
              dom = lxml.etree.fromstring(request.read())
          except lxml.etree.XMLSyntaxError as e:
              print(event_id, e)
              return False
          return {"witnesses": None, "uploaded_documents": [], "dom": dom}
  
      ## read zipfile
      try:
          request_bytes = io.BytesIO(request.read())
          package = zipfile.ZipFile(request_bytes)
      except:
          message = "Problem downloading zipfile: %s" % (event_id)
          print(message)
          return False
  
      # save documents in meeting package
      uploaded_documents = save_documents(package, event_id)
      witnesses = None
      # find meeting and witness xml
      for name in package.namelist():
          if ".xml" in name:
              if "WList" in name:
                  bytes = package.read(name)
                  witness_tree = lxml.etree.fromstring(bytes)
                  witness_info = parse_witness_list(witness_tree, uploaded_documents, event_id)
                  witnesses = witness_info["hearing_witness_info"]
              else:
                  bytes = package.read(name)
                  dom = lxml.etree.fromstring(bytes)
  
      # it will return none if there is no witness list in the file
      return {"witnesses": witnesses, "uploaded_documents": uploaded_documents, "dom": dom}
  
  
  # parse xml for urls to testimony and witness information
  def parse_witness_list(witness_tree, uploaded_documents, event_id):
      hearing_id = witness_tree.xpath("//@meeting-id")[0]
      hearing_witness_info = []
      #basic witness information
      for witness in witness_tree.xpath("panel/witness"):
          record = {"house_event_id": hearing_id}
          record["first_name"] =  witness.xpath("string(firstname)")
          if record["first_name"] == '':
              record["first_name"] = None
          record["middle_name"] = witness.xpath("string(middlename)")
          if record["middle_name"] == '':
              record["middle_name"] = None
          record["last_name"] = witness.xpath("string(lastname)")
          if record["last_name"] == '':
              record["last_name"] = None
          record["honorific"] = witness.xpath("string(honorific)")
          if record["honorific"] == '':
              record["honorific"] = None
          record["position"] = witness.xpath("string(position)")
          if record["position"] == '':
              record["position"] = None
          record["organization"] = witness.xpath("string(organization)")
          if record["organization"] == '':
              record["organization"] = None
          record["witness_type"] = witness.xpath("string(witness-type)")
          if record["witness_type"] == '':
              record["witness_type"] = None
          record["documents"] = []
          # documents related to that witness
          for doc in witness.xpath("witness-documents/witness-document"):
              document = {}
              published_on = doc.xpath("string(@publish-date)")
              try:
                  document["published_on"] = datetime.datetime.strptime(published_on, "%Y-%m-%dT%H:%M:%S.%f")
              except:
                  document["published_on"] = datetime.datetime.strptime(published_on, "%Y-%m-%dT%H:%M:%S")
  
              document["description"] = doc.xpath("string(description)")
              if document["description"] == '':
                  document["description"] = None
              
              doc_type = doc.xpath("string(type)")
              if doc_type == '':
                  document["type"] = None
                  document["type_name"] = None
              else:
                  document["type"] = doc_type
                  types = {"CV": "Committee vote", "WS": "Witness statement", 
                      "WT": "Witness truth statement", "WB": "Witness biography",
                      "CR": "Committee report", "BR": "Bill", "FA": "Floor amendment",
                      "CA": "Committee amendment", "HT": "Transcript", "WD": "Witness document"}
                          # "SD": "" I don't know this one, the SD category covers a lot
                  if doc_type in types:
                    document["type_name"] = types[doc_type]
                  else:
                    document["type_name"] = None
              
              urls = []
              for files in doc.xpath("files/file"):
                  url = files.xpath("string(@doc-url)")
                  splinter = url.split('/')
                  doc_name = splinter[-1]
                  if doc_name not in uploaded_documents:
                      file_found = save_file(url, event_id)
                  else:
                      file_found = True
                  urls.append({"url":url, "file_found": file_found})
              
              document["urls"] = urls
              record["documents"].append(document)
          hearing_witness_info.append(record)
      return {"hearing_witness_info": hearing_witness_info}
  
  
  # Grab a House meeting out of the DOM for the XML feed.
  def parse_house_committee_meeting(event_id, dom, existing_meetings, committees, options, witnesses, uploaded_documents):
      try:
          congress = int(dom.xpath("//@congress-num")[0])
          occurs_at = dom.xpath("string(meeting-details/meeting-date/calendar-date)") + " " + dom.xpath("string(meeting-details/meeting-date/start-time)")
          occurs_at = datetime.datetime.strptime(occurs_at, "%Y-%m-%d %H:%M:%S")
      except:
          raise ValueError("Invalid meeting data (probably server error) in %s." % event_id)
  
      current_status = str(dom.xpath("string(current-status)"))
      if current_status not in ("S", "R"):
          # If status is "P" (postponed and not yet rescheduled) or "C" (cancelled),
          # don't include in output.
          return
  
      topic = dom.xpath("string(meeting-details/meeting-title)")
  
      committee_names = []
      for com in dom.xpath("meeting-details/committees"):
          comte = com.xpath("string(committee-name)")
          if comte != None:
              committee_names.append(com.xpath("string(committee-name)"))
      for scom in dom.xpath("meeting-details/subcommittees"):
          scomte = scom.xpath("string(committee-name)")
          if scomte != None:
              committee_names.append(scom.xpath("string(committee-name)"))
  
      room = None
      for n in dom.xpath("meeting-details/meeting-location/capitol-complex"):
          room = n.xpath("string(building)") + " " + n.xpath("string(room)")
  
      bills = []
      for bill_id in dom.xpath("meeting-documents/meeting-document[@type='BR']/legis-num"):
          # validating bill ids
          bill_id = house_bill_id_formatter(bill_id.text, congress)
          if bill_id != None:
              bills.append(bill_id)
  
      meeting_type = dom.xpath("//@meeting-type")[0]
  
      # Meeting documents include legislation, reports, etc.
      meeting_documents = []
      for doc in dom.xpath("//meeting-document"):
          document = {}
          published_on = doc.xpath("string(@publish-date)")
          try:
              document["published_on"] = datetime.datetime.strptime(published_on, "%Y-%m-%dT%H:%M:%S.%f")
          except:
              document["published_on"] = datetime.datetime.strptime(published_on, "%Y-%m-%dT%H:%M:%S")
          document["description"] = doc.xpath("string(description)")
          if document["description"] == '':
              document["description"] = None
          
          doc_type = doc.xpath("string(filename-metadata/doc-type)")
          if doc_type == '':
              document["type"] = None
              document["type_name"] = None
          else:
              document["type"] = doc_type
              types = { "CV": "Committee vote", "WS": "Witness statement", 
                      "WT": "Witness truth statement", "WB": "Witness biography",
                      "CR": "Committee report", "BR": "Bill", "FA": "Floor amendment",
                      "CA": "Committee amendment", "HT": "Transcript", "WD": "Witness document"}
                      # "SD": "" I don't know this one, the SD category covers a lot
              if doc_type in types:
                document["type_name"] = types[doc_type]
              else:
                document["type_name"] = None
  
          document["bioguide_id"] = doc.xpath("string(filename-metadata/bioguideID)")
          if document["bioguide_id"] == '':
              document["bioguide_id"] = None
  
          document["amendmendment_number"]= doc.xpath("string(filename-metadata/amdt-num)")
  
          bill_id = doc.xpath("string(filename-metadata/legis-num)")
          document["bill_id"] = house_bill_id_formatter(bill_id, congress)
  
          document["version_code"] = doc.xpath("string(filename-metadata/legis-stage)")
          if document["version_code"] == '':
              document["version_code"] = None
          
          urls = []
          for u in doc.xpath("files/file"):
              url = u.xpath("string(@doc-url)")
              splinter = url.split('/')
              doc_name = splinter[-1]
              file_found = False
              if doc_name in uploaded_documents:
                  file_found = True
              elif options.get("docs", True):
                  file_found = save_file(url, event_id)
              urls.append({"url":url, "file_found": file_found})
  
          if len(urls) > 0:
              document["urls"] = urls
          meeting_documents.append(document)
  
      # Repeat the event for each listed committee or subcommittee, since our
      # data model supports only a single committee/subcommittee ID per event.
  
      orgs = []
      for c in dom.xpath("meeting-details/committees/committee-name"):
          if c.get("id") not in committees:
              raise ValueError("Invalid committee ID: " + c.get("id"))
          orgs.append((committees[c.get("id")]["thomas_id"], None))
      for sc in dom.xpath("meeting-details/subcommittees/committee-name"):
          if sc.get("id")[0:2] + "00" not in committees:
              raise ValueError("Invalid committee ID: " + sc.get("id"))
          c = committees[sc.get("id")[0:2] + "00"]
          if sc.get("id")[2:] not in c["subcommittees"]:
              logging.error("Invalid subcommittee code: " + sc.get("id"))
              continue
          orgs.append((c["thomas_id"], sc.get("id")[2:]))
  
      for committee_code, subcommittee_code in orgs:
          # See if this meeting already exists. If so, take its GUID.
          # Assume meetings are the same if they are for the same event ID and committee/subcommittee.
          for mtg in existing_meetings:
              if mtg["house_event_id"] == int(event_id) and mtg.get("committee", None) == committee_code and mtg.get("subcommittee", None) == subcommittee_code:
                  guid = mtg["guid"]
                  break
          else:
              # Not found, so create a new ID.
              # TODO: when does this happen?
              guid = str(uuid.uuid4())
  
          url = "http://docs.house.gov/Committee/Calendar/ByEvent.aspx?EventID=" + event_id
  
          # return the parsed meeting
          if options.get("debug", False):
              print("[house][%s][%s] Found meeting in room %s at %s" % (committee_code, subcommittee_code, room, occurs_at.isoformat()))
  
          results = {
              "chamber": "house",
              "congress": congress,
              "guid": guid,
              "committee": committee_code,
              "committee_names": committee_names,
              "subcommittee": subcommittee_code,
              "occurs_at": occurs_at.isoformat(),
              "room": room,
              "topic": topic,
              "bill_ids": bills,
              "house_meeting_type": meeting_type,
              "house_event_id": int(event_id),
              "url": url,
          }
  
          # witness information and documents are only added if there was a result
          if witnesses != None:
              results["witnesses"] = witnesses
          if len(meeting_documents) > 0:
              results["meeting_documents"] = meeting_documents 
  
          return results
  
  
  # saves documents to disk, called in extract_meeting_package
  def save_documents(package, event_id):
      uploaded_documents = []
  
      # find/create directory
      folder = str(int(event_id)/100)
      output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (folder, event_id)
      if not os.path.exists(output_dir): os.makedirs(output_dir)
  
      # loop through package and save documents    
      for name in package.namelist():
          # for documents that are not xml
          if ".xml" not in name:
              try:
                  bytes = package.read(name)
              except:
                  print("Did not save to disk: file %s" % (name))
                  continue
              file_name = "%s/%s" % (output_dir, name)
              
              # save document
              logging.info("saved " + file_name)
              with open(file_name, 'wb') as document_file:
                  document_file.write(bytes)
              # try to make a text version
              text_doc = text_from_pdf(file_name)
              if text_doc != None:
                  uploaded_documents.append(text_doc)
              uploaded_documents.append(name)
  
      return uploaded_documents
  
  # Code from IG scraper project
  # uses pdftotext to get text out of PDFs,
  # then writes it and returns the /data-relative path.
  def text_from_pdf(pdf_path):
    try:
      subprocess.Popen(["pdftotext", "-v"], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
    except:
      logging.warn("Not processing PDF. Have you installed pdftotext?")
      return None
  
    real_pdf_path = pdf_path
    real_text_path = pdf_path.replace(".pdf", ".txt")
  
    try:
      subprocess.check_call("pdftotext -layout \"%s\" \"%s\"" % (real_pdf_path, real_text_path), shell=True)
    except subprocess.CalledProcessError as exc:
      logging.warn("Error extracting text for %s\n" % (real_text_path))
      return None
  
    if os.path.exists(real_text_path):
      file_name = os.path.basename(real_text_path)
      return file_name
    else:
      logging.warn("Text not extracted to %s" % real_text_path)
      return None     
  
  
  # this is for files mentioned in the xml that do not appear in the meeting packet
  def save_file(url, event_id):
      # not saving xml but I cold be convinced otherwise
      if ".xml" in url: return False
  
      r = requests.get(url, stream=True)
  
      if r.status_code == requests.codes.ok: 
          # find or create directory
          folder = str(int(event_id)/100)
          output_dir = utils.data_dir() + "/committee/meetings/house/%s/%s" % (folder, event_id)
          if not os.path.exists(output_dir): os.makedirs(output_dir)
          # get file name
          splinter = url.split('/')
          name = splinter[-1]
          file_name = "%s/%s" % (output_dir, name)
          # try to save
  
          try:
              logging.info("saved " + url + " to " + file_name)
              with open(file_name, 'wb') as document_file:
                  document_file.write(r.content)
              if ".pdf" in file_name:
                  text_doc = text_from_pdf(file_name)
              return True
          except:
              print("Failed to save- %s" % (url))
              return False
      else:
          logging.info("failed to fetch: " + url)
          return False
              
  
  def house_bill_id_formatter(bill_id, congress):
      # make sure there is a number
      if bill_id == None or bill_id == '':
          return None
      else:
          bill_id = bill_id.strip()
          digit = False
          alpha = False
          for char in bill_id:
              if char.isdigit():
                  digit = True
              if char.isalpha():
                  alpha = True
          
          if digit == False:
              return None
          # look for missing hr, though this risks mislabeling continuing and joint resolutions
          if alpha == False:
              bill_id = "hr" + bill_id
          else:
              bill_id = bill_id.replace(".", "").replace(" ", "").lower()
          
          bill_id = bill_id + "-" + str(congress)
          return bill_id
  
  
  
  
  
  
  