#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from bs4 import BeautifulSoup
from utils import utils, inspector

# https://www.cpsc.gov/en/about-cpsc/inspector-general/
# Oldest report: 2003

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - Fix the links for BLACKLIST_REPORT_URLS

REPORTS_URL = "https://www.cpsc.gov/en/about-cpsc/inspector-general/"

BLACKLIST_REPORT_URLS = [
  'https://www.cpsc.gov/Media/Documents/About/OIG/Audits/CPSC-Fiscal-Year-2009-Financial-Statements-released-November-13-2009/',
]

def run(options):
  year_range = inspector.year_range(options)

  doc = BeautifulSoup(utils.download(REPORTS_URL))
  results = doc.select("ul.summary-list li")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_from(result, year_range):
  link = result.find("a")
  report_url = urljoin(REPORTS_URL, link.get('href'))
  if report_url in BLACKLIST_REPORT_URLS:
    return
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)
  if not report_id:
    import pdb;pdb.set_trace()

  title = link.text
  published_on_text = result.select("span.date")[0].text
  published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'cpsc',
    'inspector_url': 'https://www.cpsc.gov/en/about-cpsc/inspector-general/',
    'agency': 'cpsc',
    'agency_name': 'Consumer Product Safety Commission',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None

Add Consumer Product Safety Commission.
Add Consumer Product Safety Commission.
