text: #!/usr/bin/env python
  
  from __future__ import print_function, absolute_import
  
  # -*- coding: utf-8 -*-
  __author__ = "perfguru87@gmail.com"
  __copyright__ = "Copyright 2018, The PerfTracker project"
  __license__ = "MIT"
  
  """
  A library to scan a control panel menu items, click on them, wait for request completion and report
  performance metrics.
  
  Features:
  - Native Chrome and Firefox browsers support
  - Automatic menu and sub-menu discovery (by xpath patterns)
  - Login form detection and authentication
  - Run multiple native browsers in background under different users
  - Automatic ajax activity detection in browser and wait for click completion
  - Python-based browser repeaters to simulate bigger load
  - Waterfall-like requests view
  - Page click requests errors tracking
  - Cached/Uncached clicks/pages
  - Direct URL click / xpath click
  - Headless / traditional browsers
  """
  
  import os
  import sys
  import logging
  import traceback
  import time
  import re
  import shutil
  import copy
  import datetime
  import socket
  import platform
  import math
  from tempfile import gettempdir
  from optparse import OptionParser, OptionGroup
  from multiprocessing import Process, Queue, cpu_count
  from psutil import virtual_memory
  
  from perftrackerlib import __version__ as __version__
  from perftrackerlib.client import ptSuite, ptTest, ptVM, ptComponent
  from .browser_base import BrowserExc, DEFAULT_NAV_TIMEOUT, DEFAULT_AJAX_THRESHOLD
  from .browser_python import BrowserPython
  from .browser_chrome import BrowserChrome
  from .browser_firefox import BrowserFirefox
  from .html_report import ptBrowserHtmlReport
  from .page import PageStats, PageStatsSummary
  from .utils import gen_urls_from_index_file
  from .cp_engine import CPEngineBase
  from perftrackerlib.helpers.texttable import TextTable
  from perftrackerlib.helpers.ptshell import ptShell
  from selenium.webdriver.remote.remote_connection import LOGGER as selenium_logger
  
  bindir, basename = os.path.split(sys.argv[0])
  basename = basename.split(".")[0]
  
  BROWSERS = (BrowserChrome, BrowserFirefox, BrowserPython)
  
  OPT_DELAY_BETWEEN_CLICK_SEC = 1.0
  OPT_DEFAULT_AJAX_THRESHOLD = DEFAULT_AJAX_THRESHOLD
  OPT_DEFAULT_NAV_TIMEOUT = DEFAULT_NAV_TIMEOUT
  
  
  class CPLogFormatter(logging.Formatter):
      LOG_LVL_MAP = {logging.CRITICAL: "CRI",
                     logging.ERROR: "ERR",
                     logging.WARNING: "WRN",
                     logging.INFO: "INF",
                     logging.DEBUG: "DBG",
                     logging.NOTSET: "n/a"
                     }
  
      def __init__(self, *args, **kwargs):
          return super(CPLogFormatter, self).__init__("%(message)s", *args, **kwargs)
  
      def format(self, record):
          if 'browser' in record.__dict__:
              name = record.__dict__['browser']
          else:
              name = record.name[0:10].split(".")[0]
          record.msg = '%s [%3s] %-10s - %s' % (datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3],
                                                self.LOG_LVL_MAP[record.levelno], name, record.msg)
          return super(CPLogFormatter, self).format(record)
  
  
  class CPCrawlerException(RuntimeError):
      pass
  
  
  class CPBrowserRunner:
      def __init__(self, cp_engines, opts, urls, users, browser_id, logfile, workdir, pt_suite):
          self.cp_engines = cp_engines
          self.opts = opts
          self.urls = urls
          self.browser_id = int(browser_id)
          self.workdir = workdir
          self.page_stats_summary = PageStatsSummary()
          self.user = users[(self.browser_id - 1) % len(users)] if users else None
          self.pt_suite = pt_suite
          self._html_report = None
  
          self.browser_class = BROWSERS[0]
          for b in BROWSERS:
              if b.engine == self.opts.browser:
                  self.browser_class = b
                  break
  
          self.logdir = os.path.join(workdir, "browser.%d" % browser_id)
          self.crawler_logfile = os.path.join(self.logdir, logfile if logfile else "%s.log" % basename)
          self.browser_logfile = os.path.join(self.logdir, "%s.log" % self.browser_class.engine)
  
          self.stdout_fname = os.path.join(self.logdir, "%s.stdout" % basename) if self.browser_id else None
          self.stderr_fname = os.path.join(self.logdir, "%s.stderr" % basename) if self.browser_id else None
  
          self._stdout_orig = sys.stdout
          self._stderr_orig = sys.stderr
  
          if self.browser_id:
              self.opts.log2file = True
  
      def init(self):
          shutil.rmtree(self.logdir, ignore_errors=True)
          os.makedirs(self.logdir, mode=0o777)
  
          self.browser = self.browser_class(headless=not self.opts.view, cleanup=False,
                                            telemetry_fname=self.opts.telemetry,
                                            log_path=self.browser_logfile,
                                            nav_timeout=self.opts.nav_timeout,
                                            ajax_threshold=self.opts.ajax_threshold,
                                            remote_connstring=self.opts.remote_connstring)
  
          if self.browser_id:
              sys.stdout = open(self.stdout_fname, 'w')
              sys.stderr = open(self.stderr_fname, 'w')
  
              self.opts.telemetry = os.path.join(self.workdir, "telemetry.log")
  
          if self.opts.html_report and self.browser_id == 0:
              self._html_report = ptBrowserHtmlReport(self.opts.html_report, title=self.urls[0])
              self._html_report.gen_index_html()
  
      def fini(self):
          if self.browser_id:
              sys.stdout = self._stdout_orig
              sys.stderr = self._stderr_orig
  
          if self._html_report:
              p = self._html_report.gen_index_html()
              print("See HTML report here: %s" % p)
  
      def _detect_cp_type(self):
          for cp in self.cp_engines:
              self.browser.log_info("Trying to recognize: %s ..." % cp.type)
              c = cp(self.browser, self.user, self.opts.password, self._html_report)
              if not c.cp_init_context():
                  continue
              if not len(c.menu_xpaths):
                  continue
              for cpmx in c.menu_xpaths[0]:
                  c.switch_to_frame(cpmx.frame, verbose=False)
                  if self.browser.driver.find_elements_by_xpath(cpmx.link_xpath):
                      self.browser.log_info("%s control panel detected" % c.type)
                      c.switch_to_default_content()
                      return c
                  c.switch_to_default_content()
          return None
  
      def _login(self, url, wait_completion=True):
          self.browser.navigate_to(url, cached=False)
          for cp in self.cp_engines:
              self.browser.log_info("Trying to login into: %s ..." % cp.type)
              c = cp(self.browser, self.user, self.opts.password)
              if not c.cp_init_context():
                  continue
              if c.cp_do_login(url, timeout_s=None if wait_completion else 2.0):
                  return True
          logging.error("Login to %s under %s:%s failed" % (url, self.user, self.opts.password))
          sys.exit(-1)
  
      def _pt_suite_init(self, cp):
          if not self.opts.pt_project:
              return
  
          suite_ver = self.opts.pt_version if self.opts.pt_version else __version__
          suite_name = "cp_crawler"
          product_ver = self.opts.pt_product_version if self.opts.pt_product_version else cp.cp_get_product_version()
          product_name = self.opts.pt_product_name if self.opts.pt_product_name else cp.cp_get_product_name()
  
          self.pt_suite.suite_ver = suite_ver
          self.pt_suite.suite_name = suite_name
          self.pt_suite.product_ver = product_ver
          self.pt_suite.product_name = product_name
  
          if not self.opts.pt_title and product_name and product_ver:
              browser_name = self.browser.browser_get_name()
              browser_name += self.browser.browser_get_version().split(".")[0]
              ghz = ptShell().hw_info.cpu_freq_ghz
              self.pt_suite.job_title = "%s %s @ %s %1.fGHz" % (product_name, product_ver, browser_name, ghz)
  
          ram = virtual_memory()
  
          s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
          s.connect(("8.8.8.8", 80))
          ip = s.getsockname()[0]
          s.close()
  
          client = self.pt_suite.addNode(ptVM("client", version=platform.platform(), ip=ip, cpus=cpu_count(),
                                         ram_mb=int(ram.total / (1024 * 1024)) if ram else 0))
          client.addNode(ptComponent(self.browser.browser_get_name(), self.browser.browser_get_version()))
  
      def _pt_upload(self):
          if not self.opts.pt_project:
              return
  
          self.pt_suite.upload()
  
      def _pt_update_results(self, page):
          if self.opts.pt_project is None or page is None:
              return
  
          basename = page.get_full_name()
          if basename is None:
              return
  
          for s, m, f, g in (("Rendering", "msec", "dur", "Page rendering"),
                             ("Footprint", "KB", "ram_usage_kb", "Browser memory footprint")):
              name = "%s [%s, %s]" % (basename, s, m)
              group = "%s (%s)" % (g, m)
              test = self.pt_suite.getTest(name, group=group)
              if test is None:
                  test = ptTest(name, less_better=True, group=group, metrics=m, description=page.url)
                  self.pt_suite.addTest(test)
  
              test.add_score(page.__dict__[f])
              test.duration_sec += int(math.ceil(page.dur))
  
          self._pt_upload()
  
      def _run(self):
  
          self.browser.print_browser_info()
  
          urls = copy.copy(self.urls)
  
          CP = None
  
          if self.user:
              if self.opts.uncached:
                  logging.error("Can't combine --user and --uncached mode")
                  return None
              if len(self.urls) == 0:
                  logging.error("Login URL is not specified")
                  return None
  
              self._login(urls[0][1], wait_completion=self.opts.login_wait and self.opts.add_login_landing)
  
              CP = self._detect_cp_type()
  
              if self.opts.add_login_landing:
                  curr_url = self.browser.browser_get_current_url()
                  if curr_url not in [u[1] for u in urls]:
                      urls = [('Login landing page', curr_url)] + urls
  
          if self.opts.menu_walk:
              if not self.user:
                  name, url = urls[0]
                  page = self.browser.navigate_to(url, name=name)
  
              if CP is None:
                  CP = self._detect_cp_type()
  
              if not CP:
                  logging.error("Can't recognize Control Panel, aborting")
                  sys.exit(-1)
              items = CP.cp_do_scan_menu()
              if items:
                  urls = items
  
          if CP is None:
              CP = CPEngineBase(self.browser, html_report=self._html_report)
  
          CP.cp_handle_opts(self.opts)
          self._pt_suite_init(CP)
  
          if self.opts.randomize_urls:
              import random
              for i in range(len(urls)):
                  j = random.randint(0, len(urls) - 1)
                  urls[i], urls[j] = urls[j], urls[i]
  
          print("")
          PageStats.print_title("URLs to navigate")
          print("  " + "\n  ".join(["%s%s" % (("%-25s - " % u[0]) if u[0] else "", u[1]) for u in urls]))
  
          pages = {}
  
          cached = not self.opts.uncached
  
          print_title = True
          for name, url in urls:
              try:
                  page = CP.cp_do_navigate(url, cached=cached, name=name)
                  self._pt_update_results(page)
                  if not page:
                      raise CPCrawlerException("Page navigation (%s) failed, aborting" % url)
  
                  pages[url] = page
              except BrowserExc as e:
                  logging.error("Browser exception @ %s: %s\n%s" % (name, url, e))
                  continue
  
              if self.opts.html_report:
                  img_path = self._html_report.add_page(url, page=page)
                  self.browser.browser_get_screenshot_as_file(img_path)
                  self._html_report.gen_thumbnails(url)
  
              if self.opts.requests:
                  description = ["SCREEN: %s" % page.get_full_name(), "URL: %s" % page.url]
                  if print_title:
                      page.print_page_req_groups_stats(title=print_title, description=description)
                  else:
                      page.print_page_req_groups_stats(title=print_title, description=description)
                  print_title = False
  
          if self.opts.requests:
              print_title = True
              for page in pages.values():
                  description = ["SCREEN: %s" % page.get_full_name(), "URL: %s" % page.url]
                  page.print_page_requests_stats(title=print_title, description=description)
                  print_title = False
  
          need_to_exit = False
  
          browser_page_stats = []
          simulators_page_stats = []
  
          if self.opts.python_browsers:
              simulators = []
              for n in range(0, self.opts.python_browsers):
                  log_path = os.path.join(self.logdir, "%s.%d.log" % (BrowserPython.engine, n))
                  simulators.append(BrowserPython(log_path=log_path))
  
          for name, url in urls:
              if url not in pages:
                  continue
  
              page = pages[url]
  
              if self.opts.python_browsers:
                  map(lambda x: x.loop_start([page], sleep_sec=self.opts.delay), simulators)
  
              description = ["BROWSER:      %s" % self.browser.browser_get_name(),
                             "SIMULATORS:   %d browser(s) in background" % (self.opts.python_browsers),
                             ] if self.opts.python_browsers else []
  
              page_full_name = page.get_full_name()
              page_url = page.url
  
              description.append("%11s %s" % ("SCREEN (CACHED):" if cached else "SCREEN (UNCACHED):", page_full_name))
              if page_url != page_full_name:
                  description.append("%11s %s" % ("URL:", page_url))
  
              self.browser.page_stats[page.get_key()].print_page_timeline_header(title=not len(browser_page_stats),
                                                                                 description=description)
  
              try:
                  for n in range(0, self.opts.loops):
                      if n == 0 and not self.opts.python_browsers:
                          # use already fetched page
                          self.browser.page_stats[page.get_key()].print_page_timeline(pages[url], title=str(n + 1))
                          continue
  
                      time.sleep(self.opts.delay)
                      try:
                          if self.opts.reset_dom:
                              CP.browser.navigation_reset()
                          page = CP.cp_do_navigate(url, cached=cached, name=name)
                          self._pt_update_results(page)
                          if not page:
                              raise CPCrawlerException("Page navigation (%s) failed, aborting" % url)
  
                          self.browser.page_stats[page.get_key()].print_page_timeline(page, title=str(n + 1))
                      except BrowserExc as e:
                          logging.error(e)
                          # break
              except KeyboardInterrupt:
                  need_to_exit = True
  
              br_ps = self.browser.page_stats[page.get_key()]
              br_ps.id = self.browser.browser_get_name()
              avg = br_ps.get_avg()
              self.browser.page_stats[page.get_key()].print_page_timeline(avg, title="Average", hr=True)
              browser_page_stats.append(br_ps)
  
              if self.opts.python_browsers:
                  map(lambda x: x.loop_stop(), simulators)
                  map(lambda x: x.loop_wait(), simulators)
  
                  py_ps = PageStats("%d python simulator(s)" % (self.opts.python_browsers))
                  for s in simulators:
                      py_ps.iterations += s.page_stats[page.get_key()].iterations
  
                  simulators_page_stats.append(py_ps)
  
              if need_to_exit:
                  break
  
          for ps in browser_page_stats:
              self.page_stats_summary.add_page_stats(ps)
          for ps in simulators_page_stats:
              self.page_stats_summary.add_page_stats(ps)
  
      def run(self):
  
          if self.opts.log2file or self.opts.log_file:
              logging.basicConfig(filename=self.crawler_logfile, level=logging.DEBUG)
              print("Redirecting %sverbose logs to %s" %
                    ("browser.%d " % self.browser_id if self.browser_id is not None else "", self.crawler_logfile))
          else:
              level = logging.DEBUG if self.opts.verbose > 1 else logging.INFO if self.opts.verbose else logging.WARNING
              logging.basicConfig(level=level)
  
          logger = logging.getLogger()
          for handler in logger.root.handlers:
              handler.setFormatter(CPLogFormatter())
  
          self.init()
  
          if self.opts.session:
              self.browser.domain_set_session(self.urls[0], self.opts.session)
  
          try:
              self._run()
              print("")
          except KeyboardInterrupt:
              pass
          except BrowserExc as e:
              print(e)
          except RuntimeError:
              self.fini()
              raise
          finally:
              self.browser.browser_stop()
              self.fini()
  
          if not self.browser_id:
              if self.page_stats_summary:
                  self.page_stats_summary.print_summary(title='Final summary')
  
              if self.opts.wait:
                  print("Press Ctrl+C to exit...")
                  try:
                      time.sleep(10000)
                  except KeyboardInterrupt:
                      pass
          else:
              print("browser.%d done" % self.browser_id)
  
  
  class CPCrawler:
      def __init__(self, workdir=None, logfile=None, pt_job_title=None, pt_server_url=None):
  
          self.workdir = workdir if workdir else os.path.join(gettempdir(), "%s.%d" % (basename, os.getpid()))
          self.logfile = logfile
          self.opts = None
          self.urls = []
  
          self.pt_suite = ptSuite("UI crawler job" if pt_job_title is None else pt_job_title,
                                  pt_server_url=pt_server_url)
  
      def _gen_users(self, users_ar):
          if not users_ar:
              return None
  
          users = []
          for u in users_ar:
              for _u in u.split(","):
                  if "{" in _u:
                      m = re.search(r"(?P<pfx>.*?){(?P<from>\d+)-(?P<to>\d+)}(?P<sfx>.*)", _u)
                      if not m:
                          raise CPCrawlerException("can't parse users range from '%s',"
                                                   "valid pattern is 'prefix{from-to}suffix'" % _u)
                      for n in range(int(m.group('from')), int(m.group('to')) + 1):
                          users.append(m.group('pfx') + str(n) + m.group('sfx'))
                  else:
                      users.append(_u)
          return users
  
      def add_options(self, op, passwd="password",
                      nav_timeout=OPT_DEFAULT_NAV_TIMEOUT,
                      ajax_threshold=OPT_DEFAULT_AJAX_THRESHOLD,
                      delay_between_click_sec=OPT_DELAY_BETWEEN_CLICK_SEC):
  
          og = OptionGroup(op, "Control Panel crawler options")
          og.add_option("-v", "--verbose", action="count", default=0,
                        help="run in verbose mode, use -vvv for max verbosity")
          og.add_option("", "--log-file", type="string", help="log into the file (default %s)" % self.logfile)
          og.add_option("", "--log2file", action="store_true", help="enable --verbose mode and log to the --log-file")
          og.add_option("-V", "--view", action="store_true", help="Show browser screen")
          og.add_option("--remote-connstring", default=None, type="string",
                        help="Connect to remote selenium. ex. http://{SELENIUM_IP}:{SELENIUM_PORT}/wd/hub")
          og.add_option("-g", "--html-report", type="string",
                        help="generate HTML report with screenshots and other information")
          og.add_option("-t", "--telemetry", type="string",
                        help="log pages to given file (append only, concurrent-process-safe)")
          og.add_option("-w", "--wait", action="store_true", help="don\'t close the browser and wait till test is killed")
          og.add_option("-l", "--loops", type="int", default=7, help="number of iterations, default %default")
  
          og.add_option("-b", "--browser", choices=[b.engine for b in BROWSERS], default=BROWSERS[0].engine,
                        help="browser to use: %s (default is '%%default')" %
                        ",".join(['\'%s\'' % b.engine for b in BROWSERS]))
          og.add_option("-r", "--requests", action="store_true",
                        help="print information about individual network requests")
          og.add_option("-o", "--perf-atomic-format", action="store_true", help="perf-atomic output format")
          op.add_option_group(og)
  
          og = OptionGroup(op, "Authentication options")
          og.add_option("-U", "--user", action="append", type="string", default=None,
                        help="try to login with given user name before the test (comma-separated list accepted)")
          og.add_option("-P", "--password", type="string", default=passwd, help="password, default: %default")
          op.add_option_group(og)
  
          og = OptionGroup(op, "Navigation options")
          og.add_option("-m", "--menu-walk", action="store_true",
                        help="search for menu items and click on every menu item")
          og.add_option("-L", "--skip-login-landing-url", action="store_false", dest="add_login_landing", default=True,
                        help="skip login landing page URL")
          og.add_option("", "--dont-wait-login-landing", action="store_false", dest="login_wait", default=True,
                        help="do not wait for full login landing page load")
          og.add_option("-R", "--randomize-urls", action="store_true", help="randomize menu items sequence")
          og.add_option("-f", "--urls-file", type='string', help="get URLs from file (instead of command line)")
          og.add_option("-i", "--dir-index", action="store_true",
                        help="treat given page as apache directory listing index page and parse URLs from there")
          og.add_option("-u", "--uncached", action="store_true", help="invalidate browser cache before each request")
          og.add_option("-s", "--session", type="string", help="session ID")
          og.add_option("-e", "--reset-dom", action="store_true", help="reset DOM model after every nav click")
  
          op.add_option_group(og)
  
          og = OptionGroup(op, "Page navigation timings")
          og.add_option("-d", "--delay", type="float", default=delay_between_click_sec,
                        help="delay between GET requests, default %default sec")
          og.add_option("-T", "--nav-timeout", type="float", dest='nav_timeout', default=nav_timeout,
                        help="navigation timeout, sec (default %default)")
          og.add_option("-A", "--ajax-threshold", type="float", dest='ajax_threshold', default=ajax_threshold,
                        help="page considered as complete if no ajax requests issued after the "
                             "AJAX_THRESHOLD, sec (default %default)")
          op.add_option_group(og)
  
          og = OptionGroup(op, "Mass load mode")
          og.add_option("-X", "--real-browsers", type="int", default=0,
                        help="run REAL_BROWSERS instances of the real browsers, users (if any) "
                             "will be spread equally by the instances")
          og.add_option("-x", "--python-browsers", type="int", default=0,
                        help="run PYTHON_BROWSERS satellite/clone python browsers per every real "
                             "browser to increase load on server")
          og.add_option("-D", "--instances-delay", type="float", default=0.3,
                        help="delay between instances start (sec) (default %default)")
          og.add_option("-W", "--work-dir", type="string",
                        help="work directory with the tool logs and final HTML report, default %default")
          op.add_option_group(og)
  
          self.pt_suite.addOptions(op)
  
      def init_opts(self, opts, urls, init_logging=True):
  
          if opts.urls_file:
              if not urls:
                  urls = []
              try:
                  f = open(opts.urls_file)
                  for line in f.readlines():
                      urls.append(line.strip())
                  f.close()
              except Exception as e:
                  raise CPCrawlerException("ERROR: %s" % e)
  
          if not urls:
              raise CPCrawlerException("URL is not specified")
  
          if opts.work_dir:
              self.workdir = opts.work_dir
              self.logfile = basename + ".log"
  
          if opts.log_file:
              self.logfile = opts.log_file
          self.opts = opts
          if opts.dir_index:
              self.urls = [(None, u) for u in gen_urls_from_index_file(urls)]
          else:
              self.urls = [(None, u) for u in urls]
  
          shutil.rmtree(self.workdir, ignore_errors=True)
          os.makedirs(self.workdir, mode=0o777)
  
          if init_logging:
              self.init_logging(opts)
  
          self.pt_suite.handleOptions(opts)
  
      def init_logging(self, opts):
          if opts.log2file or opts.log_file:
              logging.basicConfig(filename=self.logfile, level=logging.DEBUG)
              print("Redirecting verbose logs to %s" % self.logfile)
          else:
              level = logging.DEBUG if opts.verbose > 1 else logging.INFO if opts.verbose else logging.WARNING
              logging.basicConfig(level=level)
              if opts.verbose < 3:
                  selenium_logger.setLevel(logging.WARNING)
  
          logger = logging.getLogger()
          for handler in logger.root.handlers:
              handler.setFormatter(CPLogFormatter())
  
      def crawl(self, cp_engines=None):
  
          def _browser_launch(queue, cp_engines, opts, urls, users, browser_id, logfile, workdir, pt_suite):
              cpbr = CPBrowserRunner(cp_engines, opts, urls, users, browser_id, logfile, workdir, pt_suite)
              if browser_id:
                  try:
                      cpbr.run()
                  except RuntimeError:
                      logging.error(traceback.format_exc())
              else:
                  cpbr.run()
              if queue:
                  queue.put(cpbr)
              return cpbr
  
          if not cp_engines:
              cp_engines = [CPEngineBase]
  
          if self.opts is None:
              raise CPCrawlerException("the init_opts() method must be called before run()")
  
          users = self._gen_users(self.opts.user)
  
          if users and len(users) > 1 and len(users) > self.opts.real_browsers:
              raise CPCrawlerException("ERROR: number of users (%d) can't be higher than number of browsers (%d)"
                                       % (len(users), self.opts.real_browsers))
  
          if self.opts.real_browsers > 0:
              real_browsers = []
              cpbr_objs = {}
  
              for i in range(1, self.opts.real_browsers + 1):
                  cpbr_objs[i] = Queue()
                  b = Process(target=_browser_launch, args=(cpbr_objs[i], cp_engines, self.opts, self.urls, users, i,
                                                            self.logfile, self.workdir, self.pt_suite))
                  b.start()
                  real_browsers.append(b)
                  time.sleep(self.opts.instances_delay)
  
              delay = 3
              telemetry_fname = os.path.join(self.workdir, "telemetry.log")
              report_fname = os.path.join(self.workdir, "report.html")
  
              print("")
              print("Notes:")
              print("   * Tests executed by %d %s browsers%s%s" %
                    (self.opts.real_browsers, self.opts.browser,
                     " and %d python browsers" % (self.opts.real_browsers * self.opts.python_browsers)
                     if self.opts.python_browsers else "",
                     " on %d users:\n       %s" % (len(users), "\n       ".join(users)) if users else ""))
              print("   * All data is stored in %s" % self.workdir)
              print("   * The %s file is updated every %d sec" % (report_fname, delay))
              print("")
  
              while True:
                  time.sleep(delay)
                  all_dead = True
                  for b in real_browsers:
                      if not b.is_alive():
                          continue
                      all_dead = False
  
                  if all_dead:
                      break
  
              page_stats_summary = PageStatsSummary()
  
              for i in range(1, self.opts.real_browsers + 1):
                  cpbr = cpbr_objs[i].get()
                  if os.path.exists(cpbr.stderr_fname):
                      f = open(cpbr.stderr_fname, 'r')
                      data = f.read()
                      f.close()
                      if data:
                          print("browser.%d stderr:\n%s" % (i, data))
  
                  for ps in cpbr.page_stats_summary.page_stats:
                      page_stats_summary.add_page_stats(ps)
  
              report_fname = os.path.join(self.workdir, "report.html")
  
          else:
              cpbr = _browser_launch(None, cp_engines, self.opts, self.urls, users, 0, self.logfile,
                                     self.workdir, self.pt_suite)
              page_stats_summary = cpbr.page_stats_summary
  
          return page_stats_summary.page_stats
  
  
  ##############################################################################
  # Autotests
  ##############################################################################
  
  
  if __name__ == "__main__":
      usage = "usage: %prog [options] URL [URL2 [URL3 ...]]"
  
      op = OptionParser(usage=usage)
      og = OptionGroup(op, "My specific options")
      op.add_option("-c", "--config", type="string", help="Some config file")
      op.add_option_group(og)
  
      cpc = CPCrawler()
      cpc.add_options(op)
  
      opts, urls = op.parse_args()
  
      # override options to simplify the test
      opts.loops = 1
      urls = ["https://example.com/"]
  
      cpc.init_opts(opts, urls)
  
      cpc.crawl()
  