text: # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
  # for details. All rights reserved. Use of this source code is governed by a
  # BSD-style license that can be found in the LICENSE file.
  
  # This file contains a set of utilities functions used by other Python-based
  # scripts.
  
  import contextlib
  import datetime
  import glob
  import imp
  import json
  import os
  import platform
  import re
  import shutil
  import subprocess
  import sys
  import tarfile
  import tempfile
  import uuid
  
  try:
      # Not available on Windows.
      import resource
  except:
      pass
  
  DART_DIR = os.path.abspath(os.path.normpath(os.path.join(__file__, '..', '..')))
  
  
  def GetBotUtils():
      '''Dynamically load the tools/bots/bot_utils.py python module.'''
      return imp.load_source(
          'bot_utils', os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py'))
  
  
  def GetMinidumpUtils():
      '''Dynamically load the tools/minidump.py python module.'''
      return imp.load_source('minidump',
                             os.path.join(DART_DIR, 'tools', 'minidump.py'))
  
  
  class Version(object):
  
      def __init__(self, channel, major, minor, patch, prerelease,
                   prerelease_patch, abi_version, oldest_supported_abi_version):
          self.channel = channel
          self.major = major
          self.minor = minor
          self.patch = patch
          self.prerelease = prerelease
          self.prerelease_patch = prerelease_patch
          self.abi_version = abi_version
          self.oldest_supported_abi_version = oldest_supported_abi_version
  
  
  # Try to guess the host operating system.
  def GuessOS():
      os_id = platform.system()
      if os_id == "Linux":
          return "linux"
      elif os_id == "Darwin":
          return "macos"
      elif os_id == "Windows" or os_id == "Microsoft":
          # On Windows Vista platform.system() can return "Microsoft" with some
          # versions of Python, see http://bugs.python.org/issue1082 for details.
          return "win32"
      elif os_id == 'FreeBSD':
          return 'freebsd'
      elif os_id == 'OpenBSD':
          return 'openbsd'
      elif os_id == 'SunOS':
          return 'solaris'
      else:
          return None
  
  
  # Try to guess the host architecture.
  def GuessArchitecture():
      os_id = platform.machine()
      if os_id.startswith('armv5te'):
          return 'armv5te'
      elif os_id.startswith('armv6'):
          return 'armv6'
      elif os_id.startswith('arm'):
          return 'arm'
      elif os_id.startswith('aarch64'):
          return 'arm64'
      elif '64' in os_id:
          return 'x64'
      elif (not os_id) or (not re.match('(x|i[3-6])86', os_id) is None):
          return 'ia32'
      elif os_id == 'i86pc':
          return 'ia32'
      else:
          guess_os = GuessOS()
          print ("Warning: Guessing architecture %s based on os %s\n"\
                % (os_id, guess_os))
          if guess_os == 'win32':
              return 'ia32'
          return None
  
  
  # Try to guess the number of cpus on this machine.
  def GuessCpus():
      if os.getenv("DART_NUMBER_OF_CORES") is not None:
          return int(os.getenv("DART_NUMBER_OF_CORES"))
      if os.path.exists("/proc/cpuinfo"):
          return int(
              subprocess.check_output(
                  "grep -E '^processor' /proc/cpuinfo | wc -l", shell=True))
      if os.path.exists("/usr/bin/hostinfo"):
          return int(
              subprocess.check_output(
                  '/usr/bin/hostinfo |'
                  ' grep "processors are logically available." |'
                  ' awk "{ print \$1 }"',
                  shell=True))
      win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS")
      if win_cpu_count:
          return int(win_cpu_count)
      return 2
  
  
  def GetWindowsRegistryKeyName(name):
      import win32process
      # Check if python process is 64-bit or if it's 32-bit running in 64-bit OS.
      # We need to know whether host is 64-bit so that we are looking in right
      # registry for Visual Studio path.
      if sys.maxsize > 2**32 or win32process.IsWow64Process():
          wow6432Node = 'Wow6432Node\\'
      else:
          wow6432Node = ''
      return r'SOFTWARE\%s%s' % (wow6432Node, name)
  
  
  # Try to guess Visual Studio location when buiding on Windows.
  def GuessVisualStudioPath():
      defaultPath = r"C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7" \
                    r"\IDE"
      defaultExecutable = "devenv.com"
  
      if not IsWindows():
          return defaultPath, defaultExecutable
  
      keyNamesAndExecutables = [
          # Pair for non-Express editions.
          (GetWindowsRegistryKeyName(r'Microsoft\VisualStudio'), 'devenv.com'),
          # Pair for 2012 Express edition.
          (GetWindowsRegistryKeyName(r'Microsoft\VSWinExpress'),
           'VSWinExpress.exe'),
          # Pair for pre-2012 Express editions.
          (GetWindowsRegistryKeyName(r'Microsoft\VCExpress'), 'VCExpress.exe')
      ]
  
      bestGuess = (0.0, (defaultPath, defaultExecutable))
  
      import _winreg
      for (keyName, executable) in keyNamesAndExecutables:
          try:
              key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, keyName)
          except WindowsError:
              # Can't find this key - moving on the next one.
              continue
  
          try:
              subkeyCounter = 0
              while True:
                  try:
                      subkeyName = _winreg.EnumKey(key, subkeyCounter)
                      subkeyCounter += 1
                  except WindowsError:
                      # Reached end of enumeration. Moving on the next key.
                      break
  
                  match = re.match(r'^\d+\.\d+$', subkeyName)
                  if match:
                      with _winreg.OpenKey(key, subkeyName) as subkey:
                          try:
                              (installDir, registrytype) = _winreg.QueryValueEx(
                                  subkey, 'InstallDir')
                          except WindowsError:
                              # Can't find value under the key - continue to the next key.
                              continue
                          isExpress = executable != 'devenv.com'
                          if not isExpress and subkeyName == '14.0':
                              # Stop search since if we found non-Express VS2015 version
                              # installed, which is preferred version.
                              return installDir, executable
                          else:
                              version = float(subkeyName)
                              # We prefer higher version of Visual Studio and given equal
                              # version numbers we prefer non-Express edition.
                              if version > bestGuess[0]:
                                  bestGuess = (version, (installDir, executable))
          finally:
              _winreg.CloseKey(key)
      return bestGuess[1]
  
  
  # Returns true if we're running under Windows.
  def IsWindows():
      return GuessOS() == 'win32'
  
  
  # Reads a text file into an array of strings - one for each
  # line. Strips comments in the process.
  def ReadLinesFrom(name):
      result = []
      for line in open(name):
          if '#' in line:
              line = line[:line.find('#')]
          line = line.strip()
          if len(line) == 0:
              continue
          result.append(line)
      return result
  
  
  # Filters out all arguments until the next '--' argument
  # occurs.
  def ListArgCallback(option, value, parser):
      if value is None:
          value = []
  
      for arg in parser.rargs:
          if arg[:2].startswith('--'):
              break
          value.append(arg)
  
      del parser.rargs[:len(value)]
      setattr(parser.values, option.dest, value)
  
  
  # Filters out all argument until the first non '-' or the
  # '--' argument occurs.
  def ListDartArgCallback(option, value, parser):
      if value is None:
          value = []
  
      for arg in parser.rargs:
          if arg[:2].startswith('--') or arg[0] != '-':
              break
          value.append(arg)
  
      del parser.rargs[:len(value)]
      setattr(parser.values, option.dest, value)
  
  
  # Mapping table between build mode and build configuration.
  BUILD_MODES = {
      'debug': 'Debug',
      'release': 'Release',
      'product': 'Product',
  }
  
  # Mapping table between OS and build output location.
  BUILD_ROOT = {
      'win32': os.path.join('out'),
      'linux': os.path.join('out'),
      'freebsd': os.path.join('out'),
      'macos': os.path.join('xcodebuild'),
  }
  
  ARCH_FAMILY = {
      'ia32': 'ia32',
      'x64': 'ia32',
      'arm': 'arm',
      'armv6': 'arm',
      'armv5te': 'arm',
      'arm64': 'arm',
      'simarm': 'ia32',
      'simarmv6': 'ia32',
      'simarmv5te': 'ia32',
      'simarm64': 'ia32',
      'simdbc': 'ia32',
      'simdbc64': 'ia32',
      'armsimdbc': 'arm',
      'armsimdbc64': 'arm',
      'simarm_x64': 'ia32',
  }
  
  ARCH_GUESS = GuessArchitecture()
  BASE_DIR = os.path.abspath(os.path.join(os.curdir, '..'))
  DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
  VERSION_FILE = os.path.join(DART_DIR, 'tools', 'VERSION')
  
  
  def GetBuildbotGSUtilPath():
      gsutil = '/b/build/scripts/slave/gsutil'
      if platform.system() == 'Windows':
          gsutil = 'e:\\\\b\\build\\scripts\\slave\\gsutil'
      return gsutil
  
  
  def GetBuildMode(mode):
      return BUILD_MODES[mode]
  
  
  def GetArchFamily(arch):
      return ARCH_FAMILY[arch]
  
  
  def IsCrossBuild(target_os, arch):
      host_arch = ARCH_GUESS
      return ((GetArchFamily(host_arch) != GetArchFamily(arch)) or
              (target_os != GuessOS()))
  
  
  def GetBuildConf(mode, arch, conf_os=None):
      if conf_os == 'android':
          return '%s%s%s' % (GetBuildMode(mode), conf_os.title(), arch.upper())
      else:
          # Ask for a cross build if the host and target architectures don't match.
          host_arch = ARCH_GUESS
          cross_build = ''
          if GetArchFamily(host_arch) != GetArchFamily(arch):
              cross_build = 'X'
          return '%s%s%s' % (GetBuildMode(mode), cross_build, arch.upper())
  
  
  def GetBuildDir(host_os):
      return BUILD_ROOT[host_os]
  
  
  def GetBuildRoot(host_os, mode=None, arch=None, target_os=None):
      build_root = GetBuildDir(host_os)
      if mode:
          build_root = os.path.join(build_root, GetBuildConf(
              mode, arch, target_os))
      return build_root
  
  
  def GetBuildSdkBin(host_os, mode=None, arch=None, target_os=None):
      build_root = GetBuildRoot(host_os, mode, arch, target_os)
      return os.path.join(build_root, 'dart-sdk', 'bin')
  
  
  def GetBaseDir():
      return BASE_DIR
  
  
  def GetShortVersion():
      version = ReadVersionFile()
      return ('%s.%s.%s.%s.%s' % (version.major, version.minor, version.patch,
                                  version.prerelease, version.prerelease_patch))
  
  
  def GetSemanticSDKVersion(no_git_hash=False, version_file=None):
      version = ReadVersionFile(version_file)
      if not version:
          return None
  
      if version.channel == 'be':
          postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision()
      elif version.channel == 'dev':
          postfix = '-dev.%s.%s' % (version.prerelease, version.prerelease_patch)
      else:
          assert version.channel == 'stable'
          postfix = ''
  
      return '%s.%s.%s%s' % (version.major, version.minor, version.patch, postfix)
  
  
  def GetVersion(no_git_hash=False, version_file=None):
      return GetSemanticSDKVersion(no_git_hash, version_file)
  
  
  # The editor used to produce the VERSION file put on gcs. We now produce this
  # in the bots archiving the sdk.
  # The content looks like this:
  #{
  #  "date": "2015-05-28",
  #  "version": "1.11.0-edge.131653",
  #  "revision": "535394c2657ede445142d8a92486d3899bbf49b5"
  #}
  def GetVersionFileContent():
      result = {
          "date": str(datetime.date.today()),
          "version": GetVersion(),
          "revision": GetGitRevision()
      }
      return json.dumps(result, indent=2)
  
  
  def GetChannel():
      version = ReadVersionFile()
      return version.channel
  
  
  def GetUserName():
      key = 'USER'
      if sys.platform == 'win32':
          key = 'USERNAME'
      return os.environ.get(key, '')
  
  
  def GetAbiVersion(version_file=None):
      version = ReadVersionFile(version_file)
      return version.abi_version
  
  
  def GetOldestSupportedAbiVersion(version_file=None):
      version = ReadVersionFile(version_file)
      return version.oldest_supported_abi_version
  
  
  def ReadVersionFile(version_file=None):
  
      def match_against(pattern, file_content):
          match = re.search(pattern, file_content, flags=re.MULTILINE)
          if match:
              return match.group(1)
          return None
  
      if version_file == None:
          version_file = VERSION_FILE
  
      try:
          fd = open(version_file)
          content = fd.read()
          fd.close()
      except:
          print("Warning: Couldn't read VERSION file (%s)" % version_file)
          return None
  
      channel = match_against('^CHANNEL ([A-Za-z0-9]+)$', content)
      major = match_against('^MAJOR (\d+)$', content)
      minor = match_against('^MINOR (\d+)$', content)
      patch = match_against('^PATCH (\d+)$', content)
      prerelease = match_against('^PRERELEASE (\d+)$', content)
      prerelease_patch = match_against('^PRERELEASE_PATCH (\d+)$', content)
      abi_version = match_against('^ABI_VERSION (\d+)$', content)
      oldest_supported_abi_version = match_against(
          '^OLDEST_SUPPORTED_ABI_VERSION (\d+)$', content)
  
      if channel and major and minor and prerelease and prerelease_patch and \
          abi_version and oldest_supported_abi_version:
          return Version(channel, major, minor, patch, prerelease,
                         prerelease_patch, abi_version,
                         oldest_supported_abi_version)
      else:
          print("Warning: VERSION file (%s) has wrong format" % version_file)
          return None
  
  
  # Our schema for releases and archiving is based on an increasing
  # sequence of numbers. In the svn world this was simply the revision of a
  # commit, which would always give us a one to one mapping between the number
  # and the commit. This was true across branches as well, so a number used
  # to archive a build was always unique and unambiguous.
  # In git there is no such global number, so we loosen the requirement a bit.
  # We only use numbers on the master branch (bleeding edge). On branches
  # we use the version number instead for archiving purposes.
  # The number on master is the count of commits on the master branch.
  def GetArchiveVersion():
      version = ReadVersionFile()
      if not version:
          raise 'Could not get the archive version, parsing the version file failed'
      if version.channel in ['be', 'integration']:
          return GetGitNumber()
      return GetSemanticSDKVersion()
  
  
  def GetGitRevision():
      # When building from tarball use tools/GIT_REVISION
      git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION')
      try:
          with open(git_revision_file) as fd:
              return fd.read()
      except:
          pass
      p = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
                           shell=IsWindows(),
                           cwd=DART_DIR)
      output, _ = p.communicate()
      output = output.strip()
      # We expect a full git hash
      if len(output) != 40:
          print("Warning: could not parse git commit, output was %s" % output)
          return None
      return output
  
  
  def GetShortGitHash():
      p = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
                           shell=IsWindows(),
                           cwd=DART_DIR)
      output, _ = p.communicate()
      output = output.strip()
      if p.wait() != 0:
          return None
      return output
  
  
  def GetLatestDevTag():
      cmd = [
          'git',
          'for-each-ref',
          'refs/tags/*dev*',
          '--sort=-taggerdate',
          "--format=%(refname:lstrip=2)",
          '--count=1',
      ]
      p = subprocess.Popen(
          cmd,
          stdout=subprocess.PIPE,
          stderr=subprocess.STDOUT,
          shell=IsWindows(),
          cwd=DART_DIR)
      output, _ = p.communicate()
      if p.wait() != 0:
          print(
              "Warning: Could not get the most recent dev branch tag %s" % output)
          return None
      return output.strip()
  
  
  def GetGitTimestamp():
      p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cd'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
                           shell=IsWindows(),
                           cwd=DART_DIR)
      output, _ = p.communicate()
      if p.wait() != 0:
          return None
      return output
  
  
  # To eliminate clashing with older archived builds on bleeding edge we add
  # a base number bigger the largest svn revision (this also gives us an easy
  # way of seeing if an archive comes from git based or svn based commits).
  GIT_NUMBER_BASE = 100000
  
  
  def GetGitNumber():
      p = subprocess.Popen(['git', 'rev-list', 'HEAD', '--count'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
                           shell=IsWindows(),
                           cwd=DART_DIR)
      output, _ = p.communicate()
      try:
          number = int(output)
          return number + GIT_NUMBER_BASE
      except:
          print("Warning: could not parse git count, output was %s" % output)
      return None
  
  
  def ParseGitInfoOutput(output):
      """Given a git log, determine the latest corresponding svn revision."""
      for line in output.split('\n'):
          tokens = line.split()
          if len(tokens) > 0 and tokens[0] == 'git-svn-id:':
              return tokens[1].split('@')[1]
      return None
  
  
  def ParseSvnInfoOutput(output):
      revision_match = re.search('Last Changed Rev: (\d+)', output)
      if revision_match:
          return revision_match.group(1)
      return None
  
  
  def RewritePathSeparator(path, workspace):
      # Paths in test files are always specified using '/'
      # as the path separator. Replace with the actual
      # path separator before use.
      if '/' in path:
          split_path = path.split('/')
          path = os.sep.join(split_path)
          path = os.path.join(workspace, path)
          if not os.path.exists(path):
              raise Exception(path)
      return path
  
  
  def ParseTestOptions(pattern, source, workspace):
      match = pattern.search(source)
      if match:
          return [
              RewritePathSeparator(o, workspace)
              for o in match.group(1).split(' ')
          ]
      else:
          return None
  
  
  def ParseTestOptionsMultiple(pattern, source, workspace):
      matches = pattern.findall(source)
      if matches:
          result = []
          for match in matches:
              if len(match) > 0:
                  result.append([
                      RewritePathSeparator(o, workspace) for o in match.split(' ')
                  ])
              else:
                  result.append([])
          return result
      else:
          return None
  
  
  def Daemonize():
      """
    Create a detached background process (daemon). Returns True for
    the daemon, False for the parent process.
    See: http://www.faqs.org/faqs/unix-faq/programmer/faq/
    "1.7 How do I get my program to act like a daemon?"
    """
      if os.fork() > 0:
          return False
      os.setsid()
      if os.fork() > 0:
          exit(0)
          raise
      return True
  
  
  def PrintError(string):
      """Writes and flushes a string to stderr."""
      sys.stderr.write(string)
      sys.stderr.write('\n')
  
  
  def CheckedUnlink(name):
      """Unlink a file without throwing an exception."""
      try:
          os.unlink(name)
      except OSError as e:
          PrintError("os.unlink() " + str(e))
  
  
  def Main():
      print("GuessOS() -> ", GuessOS())
      print("GuessArchitecture() -> ", GuessArchitecture())
      print("GuessCpus() -> ", GuessCpus())
      print("IsWindows() -> ", IsWindows())
      print("GuessVisualStudioPath() -> ", GuessVisualStudioPath())
      print("GetGitRevision() -> ", GetGitRevision())
      print("GetGitTimestamp() -> ", GetGitTimestamp())
      print("GetVersionFileContent() -> ", GetVersionFileContent())
      print("GetGitNumber() -> ", GetGitNumber())
  
  
  class Error(Exception):
      pass
  
  
  class ToolError(Exception):
      """Deprecated exception, use Error instead."""
  
      def __init__(self, value):
          self.value = value
  
      def __str__(self):
          return repr(self.value)
  
  
  def IsCrashExitCode(exit_code):
      if IsWindows():
          return 0x80000000 & exit_code
      else:
          return exit_code < 0
  
  
  def DiagnoseExitCode(exit_code, command):
      if IsCrashExitCode(exit_code):
          sys.stderr.write('Command: %s\nCRASHED with exit code %d (0x%x)\n' %
                           (' '.join(command), exit_code, exit_code & 0xffffffff))
  
  
  def Touch(name):
      with file(name, 'a'):
          os.utime(name, None)
  
  
  def ExecuteCommand(cmd):
      """Execute a command in a subprocess."""
      print('Executing: ' + ' '.join(cmd))
      pipe = subprocess.Popen(
          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=IsWindows())
      output = pipe.communicate()
      if pipe.returncode != 0:
          raise Exception('Execution failed: ' + str(output))
      return pipe.returncode, output
  
  
  # The checked-in SDKs are documented at
  #     https://github.com/dart-lang/sdk/wiki/The-checked-in-SDK-in-tools
  def CheckedInSdkPath():
      # We don't use the normal macos, linux, win32 directory names here, instead,
      # we use the names that the download_from_google_storage script uses.
      osdict = {'Darwin': 'mac', 'Linux': 'linux', 'Windows': 'win'}
      system = platform.system()
      try:
          osname = osdict[system]
      except KeyError:
          sys.stderr.write('WARNING: platform "%s" not supported\n' % system)
          return None
      tools_dir = os.path.dirname(os.path.realpath(__file__))
      return os.path.join(tools_dir, 'sdks', 'dart-sdk')
  
  
  def CheckedInSdkExecutable():
      name = 'dart'
      if IsWindows():
          name = 'dart.exe'
      return os.path.join(CheckedInSdkPath(), 'bin', name)
  
  
  def CheckedInSdkCheckExecutable():
      executable = CheckedInSdkExecutable()
      canary_script = os.path.join(
          os.path.dirname(os.path.realpath(__file__)), 'canary.dart')
      try:
          with open(os.devnull, 'wb') as silent_sink:
              if 0 == subprocess.call([executable, canary_script],
                                      stdout=silent_sink):
                  return True
      except OSError as e:
          pass
      return False
  
  
  def CheckLinuxCoreDumpPattern(fatal=False):
      core_pattern_file = '/proc/sys/kernel/core_pattern'
      core_pattern = open(core_pattern_file).read()
  
      expected_core_pattern = 'core.%p'
      if core_pattern.strip() != expected_core_pattern:
          message = (
              'Invalid core_pattern configuration. '
              'The configuration of core dump handling is *not* correct for '
              'a buildbot. The content of {0} must be "{1}" instead of "{2}".'.
              format(core_pattern_file, expected_core_pattern, core_pattern))
          if fatal:
              raise Exception(message)
          else:
              print(message)
              return False
      return True
  
  
  class TempDir(object):
  
      def __init__(self, prefix=''):
          self._temp_dir = None
          self._prefix = prefix
  
      def __enter__(self):
          self._temp_dir = tempfile.mkdtemp(self._prefix)
          return self._temp_dir
  
      def __exit__(self, *_):
          shutil.rmtree(self._temp_dir, ignore_errors=True)
  
  
  class ChangedWorkingDirectory(object):
  
      def __init__(self, working_directory):
          self._working_directory = working_directory
  
      def __enter__(self):
          self._old_cwd = os.getcwd()
          print("Enter directory = ", self._working_directory)
          os.chdir(self._working_directory)
  
      def __exit__(self, *_):
          print("Enter directory = ", self._old_cwd)
          os.chdir(self._old_cwd)
  
  
  class UnexpectedCrash(object):
  
      def __init__(self, test, pid, *binaries):
          self.test = test
          self.pid = pid
          self.binaries = binaries
  
      def __str__(self):
          return "Crash(%s: %s %s)" % (self.test, self.pid, ', '.join(
              self.binaries))
  
  
  class PosixCoreDumpEnabler(object):
  
      def __init__(self):
          self._old_limits = None
  
      def __enter__(self):
          self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
          resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
  
      def __exit__(self, *_):
          resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
  
  
  class LinuxCoreDumpEnabler(PosixCoreDumpEnabler):
  
      def __enter__(self):
          # Bump core limits to unlimited if core_pattern is correctly configured.
          if CheckLinuxCoreDumpPattern(fatal=False):
              super(LinuxCoreDumpEnabler, self).__enter__()
  
      def __exit__(self, *args):
          CheckLinuxCoreDumpPattern(fatal=False)
          super(LinuxCoreDumpEnabler, self).__exit__(*args)
  
  
  class WindowsCoreDumpEnabler(object):
      """This enabler assumes that Dart binary was built with Crashpad support.
    In this case DART_CRASHPAD_CRASHES_DIR environment variable allows to
    specify the location of Crashpad crashes database. Actual minidumps will
    be written into reports subfolder of the database.
    """
      CRASHPAD_DB_FOLDER = os.path.join(DART_DIR, r'crashes')
      DUMPS_FOLDER = os.path.join(CRASHPAD_DB_FOLDER, r'reports')
  
      def __init__(self):
          pass
  
      def __enter__(self):
          print("INFO: Enabling coredump archiving into %s" %
                (WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER))
          os.environ[
              'DART_CRASHPAD_CRASHES_DIR'] = WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER
  
      def __exit__(self, *_):
          del os.environ['DART_CRASHPAD_CRASHES_DIR']
  
  
  def TryUnlink(file):
      try:
          os.unlink(file)
      except Exception as error:
          print("ERROR: Failed to remove %s: %s" % (file, error))
  
  
  class BaseCoreDumpArchiver(object):
      """This class reads coredumps file written by UnexpectedCrashDumpArchiver
    into the current working directory and uploads all cores and binaries
    listed in it into Cloud Storage (see
    pkg/test_runner/lib/src/test_progress.dart).
    """
  
      # test.dart will write a line for each unexpected crash into this file.
      _UNEXPECTED_CRASHES_FILE = "unexpected-crashes"
  
      def __init__(self, search_dir, output_directory):
          self._bucket = 'dart-temp-crash-archive'
          self._binaries_dir = os.getcwd()
          self._search_dir = search_dir
          self._output_directory = output_directory
  
      def _safe_cleanup(self):
          try:
              return self._cleanup()
          except Exception as error:
              print("ERROR: Failure during cleanup: %s" % error)
              return False
  
      def __enter__(self):
          print("INFO: Core dump archiving is activated")
  
          # Cleanup any stale files
          if self._safe_cleanup():
              print("WARNING: Found and removed stale coredumps")
  
      def __exit__(self, *_):
          try:
              crashes = self._find_unexpected_crashes()
              if crashes:
                  # If we get a ton of crashes, only archive 10 dumps.
                  archive_crashes = crashes[:10]
                  print('Archiving coredumps for crash (if possible):')
                  for crash in archive_crashes:
                      print('----> %s' % crash)
  
                  sys.stdout.flush()
  
                  self._archive(archive_crashes)
              else:
                  print("INFO: No unexpected crashes recorded")
                  dumps = self._find_all_coredumps()
                  if dumps:
                      print("INFO: However there are %d core dumps found" %
                            len(dumps))
                      for dump in dumps:
                          print("INFO:        -> %s" % dump)
                      print()
          except Exception as error:
              print("ERROR: Failed to archive crashes: %s" % error)
              raise
  
          finally:
              self._safe_cleanup()
  
      def _archive(self, crashes):
          files = set()
          missing = []
          for crash in crashes:
              files.update(crash.binaries)
              core = self._find_coredump_file(crash)
              if core:
                  files.add(core)
              else:
                  missing.append(crash)
          if self._output_directory is not None and self._is_shard():
              print(
                  "INFO: Moving collected dumps and binaries into output directory\n"
                  "INFO: They will be uploaded to isolate server. Look for \"isolated"
                  " out\" under the failed step on the build page.\n"
                  "INFO: For more information see runtime/docs/infra/coredumps.md"
              )
              self._move(files)
          else:
              print(
                  "INFO: Uploading collected dumps and binaries into Cloud Storage\n"
                  "INFO: Use `gsutil.py cp from-url to-path` to download them.\n"
                  "INFO: For more information see runtime/docs/infra/coredumps.md"
              )
              self._upload(files)
  
          if missing:
              self._report_missing_crashes(missing, throw=True)
  
      # todo(athom): move the logic to decide where to copy core dumps into the recipes.
      def _is_shard(self):
          return 'BUILDBOT_BUILDERNAME' not in os.environ
  
      def _report_missing_crashes(self, missing, throw=True):
          missing_as_string = ', '.join([str(c) for c in missing])
          other_files = list(glob.glob(os.path.join(self._search_dir, '*')))
          sys.stderr.write(
              "Could not find crash dumps for '%s' in search directory '%s'.\n"
              "Existing files which *did not* match the pattern inside the search "
              "directory are are:\n  %s\n" % (missing_as_string, self._search_dir,
                                              '\n  '.join(other_files)))
          # TODO: Figure out why windows coredump generation does not work.
          # See http://dartbug.com/36469
          if throw and GuessOS() != 'win32':
              raise Exception('Missing crash dumps for: %s' % missing_as_string)
  
      def _get_file_name(self, file):
          # Sanitize the name: actual cores follow 'core.%d' pattern, crashed
          # binaries are copied next to cores and named
          # 'binary.<mode>_<arch>_<binary_name>'.
          # This should match the code in testing/dart/test_progress.dart
          name = os.path.basename(file)
          (prefix, suffix) = name.split('.', 1)
          is_binary = prefix == 'binary'
          if is_binary:
              (mode, arch, binary_name) = suffix.split('_', 2)
              name = binary_name
          return (name, is_binary)
  
      def _move(self, files):
          for file in files:
              print('+++ Moving %s to output_directory (%s)' %
                    (file, self._output_directory))
              (name, is_binary) = self._get_file_name(file)
              destination = os.path.join(self._output_directory, name)
              shutil.move(file, destination)
              if is_binary and os.path.exists(file + '.pdb'):
                  # Also move a PDB file if there is one.
                  pdb = os.path.join(self._output_directory, name + '.pdb')
                  shutil.move(file + '.pdb', pdb)
  
      def _tar(self, file):
          (name, is_binary) = self._get_file_name(file)
          tarname = '%s.tar.gz' % name
  
          # Compress the file.
          tar = tarfile.open(tarname, mode='w:gz')
          tar.add(file, arcname=name)
          if is_binary and os.path.exists(file + '.pdb'):
              # Also add a PDB file if there is one.
              tar.add(file + '.pdb', arcname=name + '.pdb')
          tar.close()
          return tarname
  
      def _upload(self, files):
          bot_utils = GetBotUtils()
          gsutil = bot_utils.GSUtil()
          storage_path = '%s/%s/' % (self._bucket, uuid.uuid4())
          gs_prefix = 'gs://%s' % storage_path
          http_prefix = 'https://storage.cloud.google.com/%s' % storage_path
  
          print('\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix))
          for file in files:
              tarname = self._tar(file)
  
              # Remove / from absolute path to not have // in gs path.
              gs_url = '%s%s' % (gs_prefix, tarname)
              http_url = '%s%s' % (http_prefix, tarname)
  
              try:
                  gsutil.upload(tarname, gs_url)
                  print('+++ Uploaded %s (%s)' % (gs_url, http_url))
              except Exception as error:
                  print('!!! Failed to upload %s, error: %s' % (tarname, error))
  
              TryUnlink(tarname)
  
          print('--- Done ---\n')
  
      def _find_all_coredumps(self):
          """Return coredumps that were recorded (if supported by the platform).
      This method will be overriden by concrete platform specific implementations.
      """
          return []
  
      def _find_unexpected_crashes(self):
          """Load coredumps file. Each line has the following format:
  
          test-name,pid,binary-file1,binary-file2,...
      """
          try:
              with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f:
                  return [
                      UnexpectedCrash(*ln.strip('\n').split(','))
                      for ln in f.readlines()
                  ]
          except:
              return []
  
      def _cleanup(self):
          found = False
          if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE):
              os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE)
              found = True
          for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')):
              found = True
              TryUnlink(binary)
  
          return found
  
  
  class PosixCoreDumpArchiver(BaseCoreDumpArchiver):
  
      def __init__(self, search_dir, output_directory):
          super(PosixCoreDumpArchiver, self).__init__(search_dir,
                                                      output_directory)
  
      def _cleanup(self):
          found = super(PosixCoreDumpArchiver, self)._cleanup()
          for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
              found = True
              TryUnlink(core)
          return found
  
      def _find_coredump_file(self, crash):
          core_filename = os.path.join(self._search_dir, 'core.%s' % crash.pid)
          if os.path.exists(core_filename):
              return core_filename
  
  
  class LinuxCoreDumpArchiver(PosixCoreDumpArchiver):
  
      def __init__(self, output_directory):
          super(LinuxCoreDumpArchiver, self).__init__(os.getcwd(),
                                                      output_directory)
  
  
  class MacOSCoreDumpArchiver(PosixCoreDumpArchiver):
  
      def __init__(self, output_directory):
          super(MacOSCoreDumpArchiver, self).__init__('/cores', output_directory)
  
  
  class WindowsCoreDumpArchiver(BaseCoreDumpArchiver):
  
      def __init__(self, output_directory):
          super(WindowsCoreDumpArchiver, self).__init__(
              WindowsCoreDumpEnabler.DUMPS_FOLDER, output_directory)
          self._dumps_by_pid = None
  
      # Find CDB.exe in the win_toolchain that we are using.
      def _find_cdb(self):
          win_toolchain_json_path = os.path.join(DART_DIR, 'build',
                                                 'win_toolchain.json')
          if not os.path.exists(win_toolchain_json_path):
              return None
  
          with open(win_toolchain_json_path, "r") as f:
              win_toolchain_info = json.loads(f.read())
  
          win_sdk_path = win_toolchain_info['win_sdk']
  
          # We assume that we are running on 64-bit Windows.
          # Note: x64 CDB can work with both X64 and IA32 dumps.
          cdb_path = os.path.join(win_sdk_path, 'Debuggers', 'x64', 'cdb.exe')
          if not os.path.exists(cdb_path):
              return None
  
          return cdb_path
  
      CDBG_PROMPT_RE = re.compile(r'^\d+:\d+>')
  
      def _dump_all_stacks(self):
          # On Windows due to crashpad integration crashes do not produce any
          # stacktraces. Dump stack traces from dumps Crashpad collected using
          # CDB (if available).
          cdb_path = self._find_cdb()
          if cdb_path is None:
              return
  
          dumps = self._find_all_coredumps()
          if not dumps:
              return
  
          print("### Collected %d crash dumps" % len(dumps))
          for dump in dumps:
              print()
              print("### Dumping stacks from %s using CDB" % dump)
              cdb_output = subprocess.check_output(
                  '"%s" -z "%s" -kqm -c "!uniqstack -b -v -p;qd"' % (cdb_path,
                                                                     dump),
                  stderr=subprocess.STDOUT)
              # Extract output of uniqstack from the whole output of CDB.
              output = False
              for line in cdb_output.split('\n'):
                  if re.match(WindowsCoreDumpArchiver.CDBG_PROMPT_RE, line):
                      output = True
                  elif line.startswith("quit:"):
                      break
                  elif output:
                      print(line)
          print()
          print("#############################################")
          print()
  
      def __exit__(self, *args):
          try:
              self._dump_all_stacks()
          except Exception as error:
              print("ERROR: Unable to dump stacks from dumps: %s" % error)
  
          super(WindowsCoreDumpArchiver, self).__exit__(*args)
  
      def _cleanup(self):
          found = super(WindowsCoreDumpArchiver, self)._cleanup()
          for core in glob.glob(os.path.join(self._search_dir, '*')):
              found = True
              TryUnlink(core)
          return found
  
      def _find_all_coredumps(self):
          pattern = os.path.join(self._search_dir, '*.dmp')
          return [core_filename for core_filename in glob.glob(pattern)]
  
      def _find_coredump_file(self, crash):
          if self._dumps_by_pid is None:
              # If this function is invoked the first time then look through the directory
              # that contains crashes for all dump files and collect pid -> filename
              # mapping.
              self._dumps_by_pid = {}
              minidump = GetMinidumpUtils()
              pattern = os.path.join(self._search_dir, '*.dmp')
              for core_filename in glob.glob(pattern):
                  pid = minidump.GetProcessIdFromDump(core_filename)
                  if pid != -1:
                      self._dumps_by_pid[str(pid)] = core_filename
          if crash.pid in self._dumps_by_pid:
              return self._dumps_by_pid[crash.pid]
  
      def _report_missing_crashes(self, missing, throw=True):
          # Let's only print the debugging information and not throw. We'll do more
          # validation for werfault.exe and throw afterwards.
          super(WindowsCoreDumpArchiver, self)._report_missing_crashes(
              missing, throw=False)
  
          if throw:
              missing_as_string = ', '.join([str(c) for c in missing])
              raise Exception('Missing crash dumps for: %s' % missing_as_string)
  
  
  class IncreasedNumberOfFileDescriptors(object):
  
      def __init__(self, nofiles):
          self._old_limits = None
          self._limits = (nofiles, nofiles)
  
      def __enter__(self):
          self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE)
          resource.setrlimit(resource.RLIMIT_NOFILE, self._limits)
  
      def __exit__(self, *_):
          resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
  
  
  @contextlib.contextmanager
  def NooptContextManager():
      yield
  
  
  def CoreDumpArchiver(args):
      enabled = '--copy-coredumps' in args
      prefix = '--output-directory='
      output_directory = next(
          (arg[len(prefix):] for arg in args if arg.startswith(prefix)), None)
  
      if not enabled:
          return NooptContextManager()
  
      osname = GuessOS()
      if osname == 'linux':
          return contextlib.nested(LinuxCoreDumpEnabler(),
                                   LinuxCoreDumpArchiver(output_directory))
      elif osname == 'macos':
          return contextlib.nested(PosixCoreDumpEnabler(),
                                   MacOSCoreDumpArchiver(output_directory))
      elif osname == 'win32':
          return contextlib.nested(WindowsCoreDumpEnabler(),
                                   WindowsCoreDumpArchiver(output_directory))
      else:
          # We don't have support for MacOS yet.
          return NooptContextManager()
  
  
  def FileDescriptorLimitIncreaser():
      osname = GuessOS()
      if osname == 'macos':
          return IncreasedNumberOfFileDescriptors(nofiles=10000)
      else:
          assert osname in ('linux', 'win32')
          # We don't have support for MacOS yet.
          return NooptContextManager()
  
  
  if __name__ == "__main__":
      import sys
      Main()
  