text: """Script to download the references used by the project
     Note 19/01/22: many of the functions here no longer work due to the urls may now be directing to a different page.
     The reference files should therefore be obtained manually and store in the the {App}\References folder instead.
  """
  
  import sys, os, time, types, re, traceback, threading, io, datetime, csv, json, urllib, requests, zipfile, tarfile
  from win32api import GetFileVersionInfo, LOWORD, HIWORD
  
  __dir__ = os.path.dirname(os.path.realpath(__file__))
  
  def main():
      set_working_dir(__dir__ + r'\References\\')
      
      print(__doc__)
      print('Last update : ' + file_datetime('references.json', format='%Y-%m-%d %H:%M:%S'))
      print('')
      
      Log('Update references ...')
      
      #run tasks in parallel
      with ConfigFile('references.json') as config:
          tasks = Tasks(config)
          exitcode = ParallelWorker(tasks).run(pattern='^update_')
      
      if exitcode:
          print('\nFailed!')
          sys.stderr = ''
          sys.exit(1)
      else:
          print('\nDone')
  
  class Tasks():
      
      def __init__(self, configs):
          self.cfgs = configs
      
      def update_FirefoxDriver(self):
          page = r"https://pypi.python.org/pypi/selenium"
          pattern = 'selenium-([\d\.]+).tar.gz'
          value, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          url = r'https://pypi.python.org/packages/source/s/selenium/' + value
          cfg = self.cfgs.get('FirefoxDriver')
          if cfg.get('version') != version or not file_exists('firefoxdriver.xpi'):
              with WebGZip(url) as gzip:
                  #open the xpi file as a zip
                  xpi_bytes = gzip.read(r'.*\webdriver.xpi')
                  zip_in = zipfile.ZipFile(xpi_bytes)
                  zip_out = zipfile.ZipFile('firefoxdriver.xpi', mode='w', compression=0)
                  #copy all the files except the linux ones and remove their references from the manifest
                  for fileinfo in zip_in.infolist():
                      filename = fileinfo.filename
                      if filename.endswith(r'chrome.manifest'): #edit manifest
                          manifest_txt = zip_in.read(filename)
                          manifest_txt = re.sub(r'^binary-component platform/Linux.*$\s*', \
                              '', manifest_txt, flags=re.MULTILINE)
                          zip_out.writestr(fileinfo, manifest_txt, compress_type=0)
                      elif not filename.endswith('.so'): #skip linux files
                          bytes = zip_in.read(filename)
                          zip_out.writestr(fileinfo, bytes, compress_type=0)
                  zip_out.close()
                  zip_in.close()
              cfg.update({'version': version, 'url': url})
              Log("Updated FirefoxDriver to version " + version)
      
      def update_FirefoxPrefs(self):
          url = r"https://raw.githubusercontent.com/SeleniumHQ/selenium/master/javascript/firefox-driver/webdriver.json"
          version = WebSource(url).getEtag()
          cfg = self.cfgs.get('FirefoxPrefs')
          if cfg.get('version') != version or not file_exists('firefox-prefs.js'):
              source = WebSource(url).gettext().decode('utf-8')
              content = json.loads(source)
              with open("firefox-prefs.js", 'w') as file:
                  for mainkey in ["frozen", "mutable"]:
                      mainobj = content[mainkey]
                      for key in sorted(mainobj.keys()):
                          value = mainobj[key]
                          if isinstance(value, basestring):
                              txt = 'user_pref("' + key + '", "' + value + '");\n'
                          else:
                              txt = 'user_pref("' + key + '", ' + str(value).lower() + ');\n'
                          file.write(txt)
                      file.write('\n')
              cfg.update({'version': version, 'url': url})
              Log("Updated FirefoxPrefs to version " + version)
      
      def skip_SeleniumLibraries(self):
          page = r"http://selenium-release.storage.googleapis.com"
          pattern = r'<Key>([\d\.]+/selenium-dotnet-([\d\.]+).zip)'
          value, version = WebSource(page).findlastversion(pattern, group_value=1, group_version=2)
          url = page + '/' + value
          cfg = self.cfgs.get('.NetLibraries')
          if cfg.get('version') != version or not file_exists('WebDriver.dll'):
              with WebZip(url) as zip:
                  zip.extract(r'^net35/.')
              WebFile('http://selenium.googlecode.com/git/dotnet/CHANGELOG') \
                  .save('WebDriver.changelog.txt')
              cfg.update({'version': version, 'url': url})
              Log("Updated Selenium .Net to version " + version)
      
      def update_IE32(self):
          url = r"https://github.com/SeleniumHQ/selenium/raw/master/cpp/prebuilt/Win32/Release/IEDriverServer.exe"
          version = WebSource(url).getEtag()
          cfg = self.cfgs.get('IEDriver')
          if cfg.get('version') != version or not file_exists('iedriver.exe'):
              WebFile(url).save('iedriver.exe')
              cfg.update({'version': version, 'url': url})
              file_version = get_version_number(r'iedriver.exe')
              Log("Updated IE32 driver to version " + file_version)
      
      def update_IE64(self):
          page = r"http://selenium-release.storage.googleapis.com/"
          pattern = r'<Key>([\d\.]+/IEDriverServer_x64_([\d\.]+).zip)'
          value, version = WebSource(page).findlastversion(pattern, group_value=1, group_version=2)
          url = page + value
          cfg = self.cfgs.get('IE64Driver')
          if cfg.get('version') != version or not file_exists('iedriver64.exe'):
              with WebZip(url) as zip:
                  zip.extract(r'IEDriverServer.exe', 'iedriver64.exe')
              cfg.update({'version': version, 'url': url})
              Log("Updated IE64 driver to version " + version)
      
      def update_SeleniumIDE(self):
          page = r'https://addons.mozilla.org/en-US/firefox/addon/selenium-ide/'
          pattern = r'https://addons.mozilla.org/firefox/downloads/file/\d+/selenium_ide-(\d\.\d\.\d)[^?"]+'
          url, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          cfg = self.cfgs.get('SeleniumIDE')
          if cfg.get('version') != version or not file_exists('selenium-ide.xpi'):
              WebFile(url).save('selenium-ide.xpi')
              cfg.update({'version': version, 'url': url})
              Log("Updated Selenium IDE to version " + version)
      
      def update_ChromeDriver(self):
          page = r"http://chromedriver.storage.googleapis.com/"
          version = WebSource(page + r'LATEST_RELEASE').gettext().strip()
          url = page + version + r'/chromedriver_win32.zip'
          cfg = self.cfgs.get('ChromeDriver')
          if cfg.get('version') != version or not file_exists('chromedriver.exe'):
              with WebZip(url) as zip:
                  zip.extract(r'chromedriver.exe')
              cfg.update({'version': version, 'url': url})
              Log("Updated Chrome driver to version " + version)
      
      def update_PhantomJS(self):
          page = r'https://bitbucket.org/ariya/phantomjs/downloads/'
          pattern = r'phantomjs-([\d\.]+)-windows.zip'
          value, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          url = page + value
          cfg = self.cfgs.get('PhantomJSDriver')
          if cfg.get('version') != version or not file_exists('phantomjs.exe'):
              with WebZip(url) as zip:
                  zip.extract(r'.*/phantomjs.exe')
              cfg.update({'version': version, 'url': url})
              Log("Updated PhantomJS to version " + version)
      
      def skip_Safari(self):
          page = r"http://selenium-release.storage.googleapis.com/"
          pattern = r'<Key>(([\d\.]+)/SafariDriver.safariextz)'
          value, version = WebSource(page).findlastversion(pattern, group_value=1, group_version=2)
          url = page + value
          cfg = self.cfgs.get('SafariDriver')
          if cfg.get('version') != version or not file_exists('SafariDriver.safariextz'):
              WebFile(url).save('SafariDriver.safariextz')
              cfg.update({'version': version, 'url': url})
              Log("Updated Safari driver to version " + version)
      
      def update_Opera(self):
          page = r'https://api.github.com/repos/operasoftware/operachromiumdriver/releases'
          pattern = r'/v([\d\.]+)/operadriver_win32.zip'
          value, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          url = r'https://github.com/operasoftware/operachromiumdriver/releases/download' + value
          cfg = self.cfgs.get('OperaDriver')
          if cfg.get('version') != version or not file_exists('operadriver.exe'):
              with WebZip(url) as zip:
                  zip.extract(r'operadriver.exe')
              cfg.update({'version': version, 'url': url})
              Log("Updated Opera driver to version " + version)
              
      def update_FirefoxWires(self):
          page = r'https://api.github.com/repos/jgraham/wires/releases'
          pattern = r'/([\d\.]+)/wires-[\d\.]+-windows.zip'
          value, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          url = r'https://github.com/jgraham/wires/releases/download' + value
          cfg = self.cfgs.get('FirefoxWiresDriver')
          if cfg.get('version') != version or not file_exists('wires.exe'):
              with WebZip(url) as zip:
                  zip.extract(r'wires.exe')
              cfg.update({'version': version, 'url': url})
              Log("Updated Firefox Wire driver to version " + version)
      
      def skip_PdfSharp(self):
          page1 = r'http://sourceforge.net/projects/pdfsharp/files/pdfsharp'
          pattern = r'/PDFsharp%20([\d\.]+)'
          value, version = WebSource(page1).findlastversion(pattern, group_value=0, group_version=1)
          page2 = page1 + value
          pattern = r'([^/]+/[^/]*Assemblies[^/]*\.zip)/download'
          value = WebSource(page2).findfirst(pattern)
          url = r'http://sunet.dl.sourceforge.net/project/pdfsharp/pdfsharp/' + value
          cfg = self.cfgs.get('PDFsharp')
          if cfg.get('version') != version or not file_exists( 'PdfSharp.dll'):
              with WebZip(url) as zip:
                  zip.extract(r'.*/PdfSharp.dll')
              cfg.update({'version': version, 'url': url})
              Log("Updated PDF Sharp to version " + version)
      
      def skip_DotNetZip(self):
          page = r'https://olex-secure.openlogic.com/packages/dotnetzip'
          pattern = r'https[^"]+dotnetzip-([\d.]+)[^"]+\.zip'
          value, version = WebSource(page).findlastversion(pattern, group_value=0, group_version=1)
          url = urllib.unquote(value)
          cfg = self.cfgs.get('DotNetZip')
          if cfg.get('version') != version or not file_exists('Ionic.Zip.dll'):
              with WebZip(url) as zip:
                  zip.extract(r'.*/Release/Ionic.Zip.dll')
              cfg.update({'version': version, 'url': url})
              Log("Updated DotNetZip to version " + version)
  
  
  def set_working_dir(folder):
      if not os.path.isdir(folder):
          os.makedirs(folder)
      os.chdir(folder)
  
  def file_exists(file_path):
      return os.path.isfile(file_path);
  
  def file_datetime(filepath, format='%c', default='none'):
      if not os.path.isfile(filepath) :
          return default
      return datetime.datetime.fromtimestamp(os.path.getmtime(filepath)).strftime(format)
  
  def format_ex(e_type, e_value, e_trace):
      lines = []
      for filename, lineno, name, line in traceback.extract_tb(e_trace):
          lines.append(' in %s() line %d in %s\n' % (name, lineno, os.path.basename(filename)))
          if line:
              lines.append('  ' + line.strip() + '\n')
      return '\n#%s:\n%s\n\n%s' %  (e_type.__name__, str(e_value), ''.join(lines))
  
  def get_version_number(filename):
      try:
      	info = GetFileVersionInfo(os.path.realpath(filename), "\\")
      	ms = info['FileVersionMS']
      	ls = info['FileVersionLS']
      	return '.'.join([str(v) for v in (HIWORD(ms), LOWORD(ms), HIWORD(ls), LOWORD(ls))])
      except:
      	return '0.0.0.0'
  
  class ConfigFile(dict):
  
      def __init__(self, filepath):
          self.lock = threading.Lock()
          self.filepath = filepath
          if os.path.isfile(filepath):
              with open(filepath, 'r') as file:
                  self.data = json.load(file)
          else:
              self.data = {}
  
      def __enter__(self):
          return self
  
      def __exit__(self, type, value, traceback):
          with open(self.filepath, 'w') as file:
              json.dump(self.data, file, sort_keys=False, indent=4, ensure_ascii=False)
  
      def get(self, key):
          with self.lock:
              return self.data.setdefault(key, {})
  
  from queue import Queue
  
  class ParallelWorker(Queue):
  
      def __init__(self, instance, max_workers=10):
          Queue.__init__(self)
          self.instance = instance
          self.exitcode = 0
          self.max_workers = max_workers
  
      def __run__(self):
          while not self.empty():
              method = self.get()
              try:
                  method()
              except Exception as ex:
                  self.exitcode = 1
                  e_type, e_value, e_trace = sys.exc_info()
                  sys.stderr.write(format_ex(e_type, e_value, e_trace.tb_next))
              finally:
                  self.task_done()                
  
      def run(self, pattern=''):
          methods = [getattr(self.instance, k) for k, v in self.instance.__class__.__dict__.items() \
              if isinstance(v, types.FunctionType) and re.search(pattern, k)]
          nb_workers = min(self.max_workers, len(methods))
          for method in methods:
              self.put(method)
          for i in range(nb_workers):
              t = threading.Thread(target=self.__run__)
              t.daemon = True
              t.start()
          self.join()
          return self.exitcode
  
  class WebGZip:
  
      def __init__(self, url):
          response = requests.get(url)
          response.raise_for_status()
          buffer = io.BytesIO(response.content)
          self.tar = tarfile.open(fileobj=buffer, mode='r:gz')
  
      def __enter__(self):
          return self
  
      def __exit__(self, type, value, traceback):
          self.tar.close()
  
      def extract(self, pattern, dest = '.'):
          p = re.compile(pattern)
          destIsdir = os.path.isdir(dest)
          for tarinfo in self.tar.getmembers():
              name = tarinfo.name
              if p.match(name) and not tarinfo.isdir():
                  dest_file = dest + '\\' + os.path.basename(name) if destIsdir else dest
                  with open(dest_file , 'wb') as file:
                      file.write(self.tar.extractfile(tarinfo).read())
                  if not destIsdir:
                      return
      
      def read(self, pattern):
          p = re.compile(pattern)
          for tarinfo in self.tar.getmembers():
              if p.match(tarinfo.name) and not tarinfo.isdir():
                  return self.tar.extractfile(tarinfo)
  
  class WebZip:
  
      def __init__(self, url):
          response = requests.get(url)
          response.raise_for_status()
          buffer = io.BytesIO(response.content)
          self.zip = zipfile.ZipFile(buffer)
  
      def __enter__(self):
          return self
  
      def __exit__(self, type, value, traceback):
          self.zip.close()
  
      def extract(self, pattern, dest='.'):
          p = re.compile(pattern)
          destIsdir = os.path.isdir(dest)
          for name in self.zip.namelist():
              if p.match(name):
                  dest_file = dest + '\\' + os.path.basename(name) if destIsdir else dest
                  with open(dest_file , 'wb') as file:
                      file.write(self.zip.read(name))
                  if not destIsdir:
                      return
                      
  class WebFile:
  
      def __init__(self, url):
          self.url = url
  
      def save(self, file):
          response = requests.get(self.url, stream=True)
          response.raise_for_status()
          with open(file, 'wb') as handle:
              for block in response.iter_content(1024):
                  if not block:
                      break
                  handle.write(block)
  
  
  class WebSource:
  
      def __init__(self, url):
          self.url = url
          self.text = None
  
      def gettext(self):
          if self.text is None:
              self.text = requests.get(self.url).text
          return self.text
  
      def findfirst(self, pattern, group=-1):
          res = re.search(pattern, self.gettext())
          if not res:
              raise PatternNotFound(pattern, self.url)
          return res.group(res.re.groups if group == -1 else group)
  
      def getEtag(self, default='none'):
          try:
              etag = requests.head(self.url).headers['etag']
              return re.search(r'[\w-]+', etag).group(0)
          except:
              return default
  
      def findlastversion(self, pattern, group_value=1, group_version=2):
          match = re.finditer(pattern, self.gettext())
          lst_versions = [(m.group(group_value), m.group(group_version), \
              map(int, m.group(group_version).split('.'))) for m in match]
          if not lst_versions:
              raise PatternNotFound(pattern, self.url)
          lst_versions.sort(key=lambda m: m[2], reverse=True)
          return (lst_versions[0][0], lst_versions[0][1])
  
      def findlastdate(self, pattern, group_value=1, group_datetime=2, datetime_format='%Y-%m-%dT%H:%M:%S'):
          match = re.finditer(pattern, self.gettext())
          lst_dates = [(m.group(group_value), time.strptime(m.group(group_datetime), datetime_format) \
              ) for m in match]
          if not lst_dates:
              raise PatternNotFound(pattern, self.url)
          lst_dates.sort(key=lambda m: m[1], reverse=True)
          return lst_dates[0]
  
  class PatternNotFound(Exception):
  
      def __init__(self, pattern, source):
          self.__data__ = (pattern, source)
  
      def __str__(self):
          return 'Pattern "%s" not found in %s' % self.__data__
  
  def Log(message):
      with Logger.lock:
          print(message)
          Logger.write(message)
  
  class Logger:
      
      @staticmethod
      def write(message, header = ''):
          header = time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + header
          txt = header + message.replace('\r\n', '\n').replace('\n', '\n' + header) + '\n'
          Logger.logfile.write(txt)
      
      def __init__(self):
          Logger.lock = threading.Lock()
          filename = re.sub(r'\.[^.]+$', '.log', __file__)
          Logger.logfile = open(filename, mode='a', buffering = 1)
      
      def __enter__(self):
          return self
      
      def __exit__(self, e_type, e_value, e_trace):
          if e_type:
              Logger.write(''.join(traceback.format_exception(e_type, e_value, e_trace)), 'Err')
          Logger.logfile.close()
  
  if __name__ == '__main__':
      with Logger() as log:
          main()
  