text: #!/usr/bin/env vpython
  #
  # Copyright 2015 The Chromium Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  
  """Install *_incremental.apk targets as well as their dependent files."""
  
  import argparse
  import collections
  import functools
  import glob
  import json
  import logging
  import os
  import posixpath
  import shutil
  import sys
  
  sys.path.append(
      os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
  import devil_chromium
  from devil.android import apk_helper
  from devil.android import device_utils
  from devil.utils import reraiser_thread
  from devil.utils import run_tests_helper
  from pylib import constants
  from pylib.utils import time_profile
  
  prev_sys_path = list(sys.path)
  sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
  import dex
  from util import build_utils
  sys.path = prev_sys_path
  
  
  _R8_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib',
                          'r8.jar')
  
  
  def _DeviceCachePath(device):
    file_name = 'device_cache_%s.json' % device.adb.GetDeviceSerial()
    return os.path.join(constants.GetOutDirectory(), file_name)
  
  
  def _Execute(concurrently, *funcs):
    """Calls all functions in |funcs| concurrently or in sequence."""
    timer = time_profile.TimeProfile()
    if concurrently:
      reraiser_thread.RunAsync(funcs)
    else:
      for f in funcs:
        f()
    timer.Stop(log=False)
    return timer
  
  
  def _GetDeviceIncrementalDir(package):
    """Returns the device path to put incremental files for the given package."""
    return '/data/local/tmp/incremental-app-%s' % package
  
  
  def _IsStale(src_paths, dest):
    """Returns if |dest| is older than any of |src_paths|, or missing."""
    if not os.path.exists(dest):
      return True
    dest_time = os.path.getmtime(dest)
    for path in src_paths:
      if os.path.getmtime(path) > dest_time:
        return True
    return False
  
  
  def _AllocateDexShards(dex_files):
    """Divides input dex files into buckets."""
    # Goals:
    # * Make shards small enough that they are fast to merge.
    # * Minimize the number of shards so they load quickly on device.
    # * Partition files into shards such that a change in one file results in only
    #   one shard having to be re-created.
    shards = collections.defaultdict(list)
    # As of Oct 2019, 10 shards results in a min/max size of 582K/2.6M.
    NUM_CORE_SHARDS = 10
    # As of Oct 2019, 17 dex files are larger than 1M.
    SHARD_THRESHOLD = 2**20
    for src_path in dex_files:
      if os.path.getsize(src_path) >= SHARD_THRESHOLD:
        # Use the path as the name rather than an incrementing number to ensure
        # that it shards to the same name every time.
        name = os.path.relpath(src_path, constants.GetOutDirectory()).replace(
            os.sep, '.')
        shards[name].append(src_path)
      else:
        name = 'shard{}.dex.jar'.format(hash(src_path) % NUM_CORE_SHARDS)
        shards[name].append(src_path)
    logging.info('Sharding %d dex files into %d buckets', len(dex_files),
                 len(shards))
    return shards
  
  
  def _CreateDexFiles(shards, dex_staging_dir, min_api, use_concurrency):
    """Creates dex files within |dex_staging_dir| defined by |shards|."""
    tasks = []
    for name, src_paths in shards.iteritems():
      dest_path = os.path.join(dex_staging_dir, name)
      if _IsStale(src_paths, dest_path):
        tasks.append(
            functools.partial(dex.MergeDexForIncrementalInstall, _R8_PATH,
                              src_paths, dest_path, min_api))
  
    # TODO(agrieve): It would be more performant to write a custom d8.jar
    #     wrapper in java that would process these in bulk, rather than spinning
    #     up a new process for each one.
    _Execute(use_concurrency, *tasks)
  
    # Remove any stale shards.
    for name in os.listdir(dex_staging_dir):
      if name not in shards:
        os.unlink(os.path.join(dex_staging_dir, name))
  
  
  def Uninstall(device, package, enable_device_cache=False):
    """Uninstalls and removes all incremental files for the given package."""
    main_timer = time_profile.TimeProfile()
    device.Uninstall(package)
    if enable_device_cache:
      # Uninstall is rare, so just wipe the cache in this case.
      cache_path = _DeviceCachePath(device)
      if os.path.exists(cache_path):
        os.unlink(cache_path)
    device.RunShellCommand(['rm', '-rf', _GetDeviceIncrementalDir(package)],
                           check_return=True)
    logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
  
  
  def Install(device, install_json, apk=None, enable_device_cache=False,
              use_concurrency=True, permissions=()):
    """Installs the given incremental apk and all required supporting files.
  
    Args:
      device: A DeviceUtils instance (to install to).
      install_json: Path to .json file or already parsed .json object.
      apk: An existing ApkHelper instance for the apk (optional).
      enable_device_cache: Whether to enable on-device caching of checksums.
      use_concurrency: Whether to speed things up using multiple threads.
      permissions: A list of the permissions to grant, or None to grant all
                   non-denylisted permissions in the manifest.
    """
    if isinstance(install_json, basestring):
      with open(install_json) as f:
        install_dict = json.load(f)
    else:
      install_dict = install_json
  
    main_timer = time_profile.TimeProfile()
    install_timer = time_profile.TimeProfile()
    push_native_timer = time_profile.TimeProfile()
    merge_dex_timer = time_profile.TimeProfile()
    push_dex_timer = time_profile.TimeProfile()
  
    def fix_path(p):
      return os.path.normpath(os.path.join(constants.GetOutDirectory(), p))
  
    if not apk:
      apk = apk_helper.ToHelper(fix_path(install_dict['apk_path']))
    split_globs = [fix_path(p) for p in install_dict['split_globs']]
    native_libs = [fix_path(p) for p in install_dict['native_libs']]
    dex_files = [fix_path(p) for p in install_dict['dex_files']]
    show_proguard_warning = install_dict.get('show_proguard_warning')
  
    apk_package = apk.GetPackageName()
    device_incremental_dir = _GetDeviceIncrementalDir(apk_package)
    dex_staging_dir = os.path.join(constants.GetOutDirectory(),
                                   'incremental-install',
                                   install_dict['apk_path'])
    device_dex_dir = posixpath.join(device_incremental_dir, 'dex')
  
    # Install .apk(s) if any of them have changed.
    def do_install():
      install_timer.Start()
      if split_globs:
        splits = []
        for split_glob in split_globs:
          splits.extend((f for f in glob.glob(split_glob)))
        device.InstallSplitApk(
            apk,
            splits,
            allow_downgrade=True,
            reinstall=True,
            allow_cached_props=True,
            permissions=permissions)
      else:
        device.Install(
            apk, allow_downgrade=True, reinstall=True, permissions=permissions)
      install_timer.Stop(log=False)
  
    # Push .so and .dex files to the device (if they have changed).
    def do_push_files():
  
      def do_push_native():
        push_native_timer.Start()
        if native_libs:
          with build_utils.TempDir() as temp_dir:
            device_lib_dir = posixpath.join(device_incremental_dir, 'lib')
            for path in native_libs:
              # Note: Can't use symlinks as they don't work when
              # "adb push parent_dir" is used (like we do here).
              shutil.copy(path, os.path.join(temp_dir, os.path.basename(path)))
            device.PushChangedFiles([(temp_dir, device_lib_dir)],
                                    delete_device_stale=True)
        push_native_timer.Stop(log=False)
  
      def do_merge_dex():
        merge_dex_timer.Start()
        shards = _AllocateDexShards(dex_files)
        build_utils.MakeDirectory(dex_staging_dir)
        _CreateDexFiles(shards, dex_staging_dir, apk.GetMinSdkVersion(),
                        use_concurrency)
        merge_dex_timer.Stop(log=False)
  
      def do_push_dex():
        push_dex_timer.Start()
        device.PushChangedFiles([(dex_staging_dir, device_dex_dir)],
                                delete_device_stale=True)
        push_dex_timer.Stop(log=False)
  
      _Execute(use_concurrency, do_push_native, do_merge_dex)
      do_push_dex()
  
    def check_device_configured():
      target_sdk_version = int(apk.GetTargetSdkVersion())
      # Beta Q builds apply allowlist to targetSdk=28 as well.
      if target_sdk_version >= 28 and device.build_version_sdk >= 28:
        # In P, there are two settings:
        #  * hidden_api_policy_p_apps
        #  * hidden_api_policy_pre_p_apps
        # In Q, there is just one:
        #  * hidden_api_policy
        if device.build_version_sdk == 28:
          setting_name = 'hidden_api_policy_p_apps'
        else:
          setting_name = 'hidden_api_policy'
        apis_allowed = ''.join(
            device.RunShellCommand(['settings', 'get', 'global', setting_name],
                                   check_return=True))
        if apis_allowed.strip() not in '01':
          msg = """\
  Cannot use incremental installs on Android P+ without first enabling access to
  non-SDK interfaces (https://developer.android.com/preview/non-sdk-q).
  
  To enable access:
     adb -s {0} shell settings put global {1} 0
  To restore back to default:
     adb -s {0} shell settings delete global {1}"""
          raise Exception(msg.format(device.serial, setting_name))
  
    cache_path = _DeviceCachePath(device)
    def restore_cache():
      if not enable_device_cache:
        return
      if os.path.exists(cache_path):
        logging.info('Using device cache: %s', cache_path)
        with open(cache_path) as f:
          device.LoadCacheData(f.read())
        # Delete the cached file so that any exceptions cause it to be cleared.
        os.unlink(cache_path)
      else:
        logging.info('No device cache present: %s', cache_path)
  
    def save_cache():
      if not enable_device_cache:
        return
      with open(cache_path, 'w') as f:
        f.write(device.DumpCacheData())
        logging.info('Wrote device cache: %s', cache_path)
  
    # Create 2 lock files:
    # * install.lock tells the app to pause on start-up (until we release it).
    # * firstrun.lock is used by the app to pause all secondary processes until
    #   the primary process finishes loading the .dex / .so files.
    def create_lock_files():
      # Creates or zeros out lock files.
      cmd = ('D="%s";'
             'mkdir -p $D &&'
             'echo -n >$D/install.lock 2>$D/firstrun.lock')
      device.RunShellCommand(
          cmd % device_incremental_dir, shell=True, check_return=True)
  
    # The firstrun.lock is released by the app itself.
    def release_installer_lock():
      device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir,
                             check_return=True, shell=True)
  
    # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't
    # been designed for multi-threading. Enabling only because this is a
    # developer-only tool.
    setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache,
                           check_device_configured)
  
    _Execute(use_concurrency, do_install, do_push_files)
  
    finalize_timer = _Execute(use_concurrency, release_installer_lock, save_cache)
  
    logging.info(
        'Install of %s took %s seconds (setup=%s, install=%s, lib_push=%s, '
        'dex_merge=%s dex_push=%s, finalize=%s)', os.path.basename(apk.path),
        main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(),
        push_native_timer.GetDelta(), merge_dex_timer.GetDelta(),
        push_dex_timer.GetDelta(), finalize_timer.GetDelta())
    if show_proguard_warning:
      logging.warning('Target had proguard enabled, but incremental install uses '
                      'non-proguarded .dex files. Performance characteristics '
                      'may differ.')
  
  
  def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('json_path',
                        help='The path to the generated incremental apk .json.')
    parser.add_argument('-d', '--device', dest='device',
                        help='Target device for apk to install on.')
    parser.add_argument('--uninstall',
                        action='store_true',
                        default=False,
                        help='Remove the app and all side-loaded files.')
    parser.add_argument('--output-directory',
                        help='Path to the root build directory.')
    parser.add_argument('--no-threading',
                        action='store_false',
                        default=True,
                        dest='threading',
                        help='Do not install and push concurrently')
    parser.add_argument('--no-cache',
                        action='store_false',
                        default=True,
                        dest='cache',
                        help='Do not use cached information about what files are '
                             'currently on the target device.')
    parser.add_argument('-v',
                        '--verbose',
                        dest='verbose_count',
                        default=0,
                        action='count',
                        help='Verbose level (multiple times for more)')
  
    args = parser.parse_args()
  
    run_tests_helper.SetLogLevel(args.verbose_count)
    if args.output_directory:
      constants.SetOutputDirectory(args.output_directory)
  
    devil_chromium.Initialize(output_directory=constants.GetOutDirectory())
  
    # Retries are annoying when commands fail for legitimate reasons. Might want
    # to enable them if this is ever used on bots though.
    device = device_utils.DeviceUtils.HealthyDevices(
        device_arg=args.device,
        default_retries=0,
        enable_device_files_cache=True)[0]
  
    if args.uninstall:
      with open(args.json_path) as f:
        install_dict = json.load(f)
      apk = apk_helper.ToHelper(install_dict['apk_path'])
      Uninstall(device, apk.GetPackageName(), enable_device_cache=args.cache)
    else:
      Install(device, args.json_path, enable_device_cache=args.cache,
              use_concurrency=args.threading)
  
  
  if __name__ == '__main__':
    sys.exit(main())
  