text: from logging.handlers import TimedRotatingFileHandler
  from time import sleep, localtime, mktime
  from datetime import datetime
  from pytz import timezone
  from os import remove, listdir, mkdir, stat
  from os.path import exists
  from zipfile import ZipFile
  from tqdm import tqdm
  from pyppeteer import launch
  from boto3 import Session
  from botocore.exceptions import ClientError
  from subprocess import run
  from requests import get, RequestException
  import pandas as pd
  import sys
  import logging
  import asyncio
  
  
  def main():
      log.info("Starting main()")
  
      # Setup for aws connection w/boto3
      session = Session(
          # Default will search ~/.aws/ for your credentials
          # profile_name = "default",
          # aws_access_key_id = "",
          # aws_secret_access_key = "",
          # region_name = "us-west-2",
      )
  
      # Server is running in UTC but we want friendly Alberta times
      ab_now = datetime.now(timezone("America/Edmonton"))
      today = ab_now.strftime("%Y-%m-%d")
  
      # Our bucket for Petrinex Data
      petrinex_bucket = "petrinex.bieberstein.ca"
      folder = "petrinex/ab/" + today +"/"
  
      # Local filesystem
      path = "/tmp/"
      primary_zip = "download.zip"
  
      # Our external IP
      ip = get_external_ip()
      log.info("External IP:" + ip)
  
      log.info("Installing Headless Chrome")
      install_chrome_from_s3(session)
  
      # Navigate the website and download our zip file of data
      log.info("Pyppeteer Asyncio loop: main")
      asyncio.get_event_loop().run_until_complete(async_pyppeteer_script())
  
      # 1st Level unzip and cleanup
      if not exists("/tmp/zips"):
          mkdir("/tmp/zips")
  
      tmpfile = path + primary_zip
  
      path = "/tmp/zips/"
      with ZipFile(tmpfile, "r") as zip:
          zip.extractall(path)
      remove(tmpfile)
  
      # 2nd level unzip and cleanup
      for filename in listdir(path):
          tmpfile = path + filename
          if filename.endswith(".zip"):
              with ZipFile(tmpfile) as innerzip:
                  innerzip.extractall(path)
              remove(tmpfile)
  
      for filename in listdir(path):
          if filename.endswith(".CSV"):
              if filename.startswith("Vol_"):
                  subfolder = "conventional-volumetrics/"
              elif filename.startswith("Business Associate"):
                  subfolder = "business-associate/"
              elif filename.startswith("Facility Infrastructure"):
                  subfolder = "facility-infrastructure/"
              elif filename.startswith("Facility Licence"):
                  subfolder = "facility-licence/"
              elif filename.startswith("Facility Operator History"):
                  subfolder = "facility-operator-history/"
              elif filename.startswith("Well Infrastructure"):
                  subfolder = "well-infrastructure/"
              elif filename.startswith("Well Licence"):
                  subfolder = "well-licence/"
              elif filename.startswith("Well to Facility Link"):
                  subfolder = "well-to-facility-link/"
  
              # Upload CSV
              tmpkey = folder + subfolder + filename
              tmpfile = path + filename
              log.info("Uploading CSV " + tmpfile)
              upload_file_to_s3(
                  session,
                  bucket=petrinex_bucket,
                  key=tmpkey,
                  filename=tmpfile,
              )
  
  
  async def async_pyppeteer_script():
      # Our URL for the Petrinex data
      petrinex_ab_pubdata = "https://www.petrinex.gov.ab.ca/PublicData"
      # Where we receive the download from the petrinex site
      zip_file_path = "/tmp/download.zip"
  
      log.info("Setup browser")
      browser = await launch(
          args=[
              "--no-sandbox",
              "--single-process",
              "--disable-dev-shm-usage",
              "--disable-gpu",
              "--no-zygote",
          ],
          devtools=False,
          headless=True,
          executablePath="/tmp/headless-chromium",
          userDataDir="/tmp",
      )
  
      page = await browser.newPage()
      await page.setViewport({"width": 1280, "height": 800})
      # Enable downloading and set default folder to /tmp/
      cdp = await page.target.createCDPSession()
      await cdp.send(
          "Page.setDownloadBehavior", {"behavior": "allow", "downloadPath": "/tmp/"}
      )
  
      log.info("Load Page/Complete Form: " + petrinex_ab_pubdata)
      await page.goto(petrinex_ab_pubdata)
      await page.click('[data-control-element="Infrastructure_All"]')
      await page.click('[data-control-element="Volumetric_All"]')
      await page.click('[value="csv"]')
      log.info("Click Download Button")
      await page.click('[data-control-element="Download_Button"]')
      log.info("Sleep 2")
      sleep(2)
      log.info("Click Yes on popup (has focus)")
      await page.keyboard.press("Enter")
      log.info("Sleep 120 to let download stream start")
      sleep(120)
  
      log.info("Sleeping 10min or until file exists: " + zip_file_path)
      before = localtime()
      while not exists(zip_file_path):
          sleep(30)
          # Timelimit 600 seconds - 10 minutes
          if mktime(localtime()) - mktime(before) > 600:
              exit(1)
      log.info("Close Browser")
      await browser.close()
  
  
  def download_object_from_s3(session, *, bucket, key, version_id=None, filename):
      """Download an object from S3 with a progress bar.
  
      session = Session()
      download_object_from_s3(session, bucket="bucket", key="key", filename="file.txt", )  # optional version_id="versionstring"
      """
      # Use existing session to create an S3 client
      s3 = session.client("s3")
  
      # Handle versioning if it exists
      kwargs = {"Bucket": bucket, "Key": key}
      if version_id is not None:
          kwargs["VersionId"] = version_id
  
      # How big is the object (race condition)
      object_size = s3.head_object(**kwargs)["ContentLength"]
  
      # Handle versioning if it exists
      if version_id is not None:
          ExtraArgs = {"VersionId": version_id}
      else:
          ExtraArgs = None
  
      # Download object with ASCII progress bar
      with tqdm(
          ascii=True,
          desc=filename,
          total=object_size,
          unit="B",  # unit string = B (bytes)
          unit_scale=True,  # Use metric prefixes for unit
      ) as progress:
          try:
              s3.download_file(
                  Bucket=bucket,
                  Key=key,
                  ExtraArgs=ExtraArgs,
                  Filename=filename,
                  Callback=lambda xferd: progress.update(xferd),
              )
          except ClientError as e:
              logging.error(e)
              return False
      return True
  
  
  def upload_file_to_s3(session, *, bucket, key, filename):
      """Upload a file to S3 with a progress bar.
  
      session = boto3.Session()
      success = upload_file_to_s3(session, bucket="bucket", key="key", filename="file.txt")
      """
      # Use existing session to create an S3 client
      s3 = session.client("s3")
  
      # How big is the file (race condition)
      file_size = stat(filename).st_size
  
      # Upload the file with ASCII progress bar
      with tqdm(
          ascii=True, total=file_size, unit="B", unit_scale=True, desc=filename
      ) as progress:
          try:
              response = s3.upload_file(
                  Filename=filename,
                  Bucket=bucket,
                  Key=key,
                  Callback=lambda xferd: progress.update(xferd),
              )
          except ClientError as e:
              logging.error(e)
              return False
      return True
  
  
  def install_chrome_from_s3(
      session,
      bucket_name="assets.bieberstein.ca",
      key="serverless-chrome/stable-headless-chromium-amazonlinux-2017-03.zip",
  ):
      """
      Install headless chromium browser from zipfile stored in s3bucket/object
      """
      zip_file_path = "/tmp/chrome.zip"
  
      download_object_from_s3(
          session,
          bucket=bucket_name,
          key=key,
          filename=zip_file_path,
      )
      with ZipFile(zip_file_path, "r") as zip_ref:
          zip_ref.extractall("/tmp")
      run(["/bin/chmod", "+x", "/tmp/headless-chromium"], check=True)
  
  
  def get_external_ip():
      # Get my External IP
      try:
          # Get external IP using AWS service.
          ip = (get("http://checkip.amazonaws.com/")).text.replace("\n", "")
          # Another public service to get the same thing
          # ip = (get("https://ifconfig.me/ip")).text.replace("\n", "")
      except RequestException as e:
          # Send some context about this error to Lambda Logs
          print(e)
          raise e
      return ip
  
  
  def configure_logging(logger_name):
      FORMATTER = logging.Formatter(
          "%(asctime)s — %(name)s — %(levelname)s — %(message)s"
      )
      LOG_FILE = logger_name + ".log"
  
      logger = logging.getLogger(logger_name)
      logger.setLevel(logging.DEBUG)  # better to have too much log than not enough
  
      # Attach console to log
      console_handler = logging.StreamHandler(sys.stdout)
      console_handler.setFormatter(FORMATTER)
      logger.addHandler(console_handler)
  
      # Attach file to log
      file_handler = TimedRotatingFileHandler(LOG_FILE, when="midnight")
      file_handler.setFormatter(FORMATTER)
      logger.addHandler(file_handler)
  
      # with this pattern, it's rarely necessary to propagate the error up to parent
      logger.propagate = False
      return logger
  
  
  if __name__ == "__main__":
      log = configure_logging("app")
      main()
  