# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""An auto-roller for GN binaries into Chromium.

This script is used to update the GN binaries that a Chromium
checkout uses. In order to update the binaries, one must follow
four steps in order:

1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
   the newly-built binaries into the right Google CloudStorage bucket.
2. Wait for the try jobs to complete.
3. Update the buildtools repo with the .sha1 hashes of the newly built
   binaries.
4. Update Chromium's DEPS file to the new version of the buildtools repo.

The script has four commands that correspond to the four steps above:
'build', 'wait', 'roll_buildtools', and 'roll_deps'.

The script has a fifth command, 'roll', that runs the four in order.

If given no arguments, the script will run the 'roll' command.

It can only be run on linux in a clean Chromium checkout; it should
error out in most cases if something bad happens, but the error checking
isn't yet foolproof.

"""

from __future__ import print_function

import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import time
import urllib2

depot_tools_path = None
for p in os.environ['PATH'].split(os.pathsep):
  if (p.rstrip(os.sep).endswith('depot_tools') and
      os.path.isfile(os.path.join(p, 'gclient.py'))):
    depot_tools_path = p

assert depot_tools_path
if not depot_tools_path in sys.path:
  sys.path.insert(0, depot_tools_path)

third_party_path = os.path.join(depot_tools_path, 'third_party')
if not third_party_path in sys.path:
  sys.path.insert(0, third_party_path)

import upload


CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'

CODE_REVIEW_SERVER = 'https://codereview.chromium.org'

COMMITISH_DIGITS = 10

class GNRoller(object):
  def __init__(self):
    self.chromium_src_dir = None
    self.buildtools_dir = None
    self.old_gn_commitish = None
    self.new_gn_commitish = None
    self.old_gn_version   = None
    self.new_gn_version = None
    self.reviewer = 'dpranke@chromium.org'
    if os.getenv('USER') == 'dpranke':
      self.reviewer = 'brettw@chromium.org'

  def Roll(self):
    parser = argparse.ArgumentParser()
    parser.usage = __doc__
    parser.add_argument('command', nargs='?', default='roll',
                        help='build|roll|roll_buildtools|roll_deps|wait'
                             ' (%(default)s is the default)')

    args = parser.parse_args()
    command = args.command
    ret = self.SetUp()
    if not ret and command in ('roll', 'build'):
      ret = self.TriggerBuild()
    if not ret and command in ('roll', 'wait'):
      ret = self.WaitForBuildToFinish()
    if not ret and command in ('roll', 'roll_buildtools'):
      ret = self.RollBuildtools()
    if not ret and command in ('roll', 'roll_deps'):
      ret = self.RollDEPS()

    return ret

  def SetUp(self):
    if sys.platform != 'linux2':
      print('roll_gn is only tested and working on Linux for now.')
      return 1

    ret, out, _ = self.Call('git config --get remote.origin.url')
    origin = out.strip()
    if ret or origin != CHROMIUM_REPO:
      print('Not in a Chromium repo? git config --get remote.origin.url '
            'returned %d: %s' % (ret, origin))
      return 1

    ret, _, _ = self.Call('git diff -q')
    if ret:
      print("Checkout is dirty, exiting")
      return 1

    _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
    self.chromium_src_dir = out.strip()
    self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')

    self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()

    _, out, _ = self.Call('gn --version')
    self.old_gn_version = out.strip()

    _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
    self.old_gn_commitish = out.strip()
    return 0

  def GetNewVersions(self):
    _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
    commit_msg = out.splitlines()
    first_line = commit_msg[0]
    new_gn_commitish = first_line.split()[1]

    last_line = commit_msg[-1]
    new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)

    return new_gn_commitish, new_gn_version

  def TriggerBuild(self):
    ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
    if ret:
      print('Failed to create a new branch for build_gn_%s' %
            self.new_gn_version)
      return 1

    self.MakeDummyDepsChange()

    ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
                         self.new_gn_version)
    if ret:
      print('git commit failed: %s' % out + err)
      return 1

    print('Uploading CL to build GN at {#%s} - %s' %
          (self.new_gn_version, self.new_gn_commitish))
    ret, out, err = self.Call('git cl upload -f')
    if ret:
      print('git-cl upload failed: %s' % out + err)
      return 1

    print('Starting try jobs')
    self.Call('git-cl try -m tryserver.chromium.linux '
              '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish)
    self.Call('git-cl try -m tryserver.chromium.mac '
              '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish)
    self.Call('git-cl try -m tryserver.chromium.win '
              '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish)

    return 0

  def MakeDummyDepsChange(self):
    with open('DEPS') as fp:
      deps_content = fp.read()
      new_deps = deps_content.replace("'buildtools_revision':",
                                      "'buildtools_revision':  ")

    with open('DEPS', 'w') as fp:
      fp.write(new_deps)

  def WaitForBuildToFinish(self):
    print('Checking build')
    results = self.CheckBuild()
    while any(r['state'] == 'pending' for r in results.values()):
      print()
      print('Sleeping for 30 seconds')
      time.sleep(30)
      print('Checking build')
      results = self.CheckBuild()
    return 0 if all(r['state'] == 'success' for r in results.values()) else 1

  def CheckBuild(self):
    _, out, _ = self.Call('git-cl issue')

    issue = int(out.split()[2])

    _, out, _ = self.Call('git config user.email')
    email = ''
    rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email)
    try:
      props = json.loads(rpc_server.Send('/api/%d' % issue))
    except Exception as _e:
      raise

    patchset = int(props['patchsets'][-1])

    try:
      patchset_data = json.loads(rpc_server.Send('/api/%d/%d' %
                                                 (issue, patchset)))
    except Exception as _e:
      raise

    TRY_JOB_RESULT_STATES = ('success', 'warnings', 'failure', 'skipped',
                             'exception', 'retry', 'pending')
    try_job_results = patchset_data['try_job_results']
    if not try_job_results:
      print('No try jobs found on most recent patchset')
      return 1

    results = {}
    for job in try_job_results:
      builder = job['builder']
      if builder == 'linux_chromium_gn_upload':
        platform = 'linux64'
      elif builder == 'mac_chromium_gn_upload':
        platform = 'mac'
      elif builder == 'win8_chromium_gn_upload':
        platform = 'win'
      else:
        print('Unexpected builder: %s')
        continue

      state = TRY_JOB_RESULT_STATES[int(job['result'])]
      url_str = ' %s' % job['url']
      build = url_str.split('/')[-1]

      sha1 = '-'
      results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str})

      if state == 'success':
        jsurl = url_str.replace('/builders/', '/json/builders/')
        fp = urllib2.urlopen(jsurl)
        js = json.loads(fp.read())
        fp.close()
        for step in js['steps']:
          if step['name'] == 'gn sha1':
            sha1 = step['text'][1]

      if results[platform]['build'] < build:
        results[platform]['build'] = build
        results[platform]['sha1'] = sha1
        results[platform]['state'] = state
        results[platform]['url'] = url_str

    for platform, r in results.items():
      print(platform)
      print('  sha1:  %s' % r['sha1'])
      print('  state: %s' % r['state'])
      print('  build: %s' % r['build'])
      print('  url:   %s' % r['url'])
      print()

    return results

  def RollBuildtools(self):
    results = self.CheckBuild()
    if not all(r['state'] == 'success' for r in results.values()):
      print("Roll isn't done or didn't succeed, exiting:")
      return 1

    desc = self.GetBuildtoolsDesc()

    self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
              cwd=self.buildtools_dir)

    for platform in results:
      fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
      path = os.path.join(self.buildtools_dir, platform, fname)
      with open(path, 'w') as fp:
        fp.write('%s\n' % results[platform]['sha1'])

    desc_file = tempfile.NamedTemporaryFile(delete=False)
    try:
      desc_file.write(desc)
      desc_file.close()
      self.Call('git commit -a -F %s' % desc_file.name,
                cwd=self.buildtools_dir)
      self.Call('git-cl upload -f --send-mail',
                cwd=self.buildtools_dir)
    finally:
      os.remove(desc_file.name)

    ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir)
    if ret:
        print("buildtools git cl land failed: %d" % ret)
        if out:
            print(out)
        if err:
            print(err)
        return ret

    # Fetch the revision we just committed so that RollDEPS will find it.
    self.Call('git fetch', cwd=self.buildtools_dir)

    return 0

  def RollDEPS(self):
    ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version)
    if ret:
      print('Failed to create a new branch for roll_gn_%s' %
            self.new_gn_version)
      return 1

    _, out, _ = self.Call('git rev-parse origin/master',
                          cwd=self.buildtools_dir)
    new_buildtools_commitish = out.strip()

    new_deps_lines = []
    old_buildtools_commitish = ''
    with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
      for l in fp.readlines():
        m = re.match(".*'buildtools_revision':.*'(.+)',", l)
        if m:
          old_buildtools_commitish = m.group(1)
          new_deps_lines.append("  'buildtools_revision': '%s'," %
                                new_buildtools_commitish)
        else:
          new_deps_lines.append(l)

    if not old_buildtools_commitish:
      print('Could not update DEPS properly, exiting')
      return 1

    with open('DEPS', 'w') as fp:
      fp.write(''.join(new_deps_lines) + '\n')

    desc = self.GetDEPSRollDesc(old_buildtools_commitish,
                                new_buildtools_commitish)
    desc_file = tempfile.NamedTemporaryFile(delete=False)
    try:
      desc_file.write(desc)
      desc_file.close()
      self.Call('git commit -a -F %s' % desc_file.name)
      self.Call('git-cl upload -f --send-mail --use-commit-queue')
    finally:
      os.remove(desc_file.name)
    return 0

  def GetBuildtoolsDesc(self):
    gn_changes = self.GetGNChanges()
    return (
      'Roll gn %s..%s (r%s:r%s)\n'
      '\n'
      '%s'
      '\n'
      'TBR=%s\n' % (
        self.old_gn_commitish[:COMMITISH_DIGITS],
        self.new_gn_commitish[:COMMITISH_DIGITS],
        self.old_gn_version,
        self.new_gn_version,
        gn_changes,
        self.reviewer,
      ))

  def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
    gn_changes = self.GetGNChanges()

    return (
      'Roll DEPS %s..%s\n'
      '\n'
      '  In order to roll GN %s..%s (r%s:r%s) and pick up\n'
      '  the following changes:\n'
      '\n'
      '%s'
      '\n'
      'TBR=%s\n'
      'CQ_EXTRA_TRYBOTS=tryserver.chromium.mac:mac_chromium_gn_rel,'
      'mac_chromium_gn_dbg;'
      'tryserver.chromium.win:win8_chromium_gn_dbg,'
      'win_chromium_gn_x64_rel\n' % (
        old_buildtools_commitish[:COMMITISH_DIGITS],
        new_buildtools_commitish[:COMMITISH_DIGITS],
        self.old_gn_commitish[:COMMITISH_DIGITS],
        self.new_gn_commitish[:COMMITISH_DIGITS],
        self.old_gn_version,
        self.new_gn_version,
        gn_changes,
        self.reviewer,
      ))

  def GetGNChanges(self):
    _, out, _ = self.Call(
        "git log --pretty='  %h %s' " +
        "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
    return out

  def Call(self, cmd, cwd=None):
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
                            cwd=(cwd or self.chromium_src_dir))
    out, err = proc.communicate()
    return proc.returncode, out, err


if __name__ == '__main__':
  roller = GNRoller()
  sys.exit(roller.Roll())
