text: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import argparse
  import datetime
  import re
  import time
  import shutil
  import os
  import sys
  import subprocess
  import textwrap
  import urllib.request, urllib.error, urllib.parse
  import xml.etree.ElementTree as ET
  
  LOG = '/tmp/release.log'
  
  def log(msg):
    f = open(LOG, mode='ab')
    f.write(msg.encode('utf-8'))
    f.close()
    
  def run(command):
    log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command))
    if os.system('%s >> %s 2>&1' % (command, LOG)):
      msg = '    FAILED: %s [see log %s]' % (command, LOG)
      print(msg)
      raise RuntimeError(msg)
  
  def runAndSendGPGPassword(command, password):
    p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
    f = open(LOG, 'ab')
    while True:
      p.stdout.flush()
      line = p.stdout.readline()
      if len(line) == 0:
        break
      f.write(line)
      if line.find(b'Enter GPG keystore password:') != -1:
        time.sleep(1.0)
        p.stdin.write((password + '\n').encode('UTF-8'))
        p.stdin.write('\n'.encode('UTF-8'))
  
    result = p.poll()
    if result != 0:
      msg = '    FAILED: %s [see log %s]' % (command, LOG)
      print(msg)
      raise RuntimeError(msg)
  
  def load(urlString):
    try:
      content = urllib.request.urlopen(urlString).read().decode('utf-8')
    except Exception as e:
      print('Retrying download of url %s after exception: %s' % (urlString, e))
      content = urllib.request.urlopen(urlString).read().decode('utf-8')
    return content
  
  def getGitRev():
    status = os.popen('git status').read().strip()
    if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
      raise RuntimeError('git clone is dirty:\n\n%s' % status)
    branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
    command = 'git log origin/%s..' % branch
    unpushedCommits = os.popen(command).read().strip()
    if len(unpushedCommits) > 0:
      raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, unpushedCommits))
  
    print('  git clone is clean')
    return os.popen('git rev-parse HEAD').read().strip()
  
  def prepare(root, version, gpgKeyID, gpgPassword):
    print()
    print('Prepare release...')
    if os.path.exists(LOG):
      os.remove(LOG)
  
    os.chdir(root)
    print('  git pull...')
    run('git pull')
  
    rev = getGitRev()
    print('  git rev: %s' % rev)
    log('\nGIT rev: %s\n' % rev)
  
    print('  Check DOAP files')
    checkDOAPfiles(version)
  
    print('  ant clean test')
    run('ant clean test')
  
    open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
    
    print('  lucene prepare-release')
    os.chdir('lucene')
    cmd = 'ant -Dversion=%s' % version
    if gpgKeyID is not None:
      cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
    else:
      cmd += ' prepare-release-no-sign'
  
    if gpgPassword is not None:
      runAndSendGPGPassword(cmd, gpgPassword)
    else:
      run(cmd)
    
    print('  solr prepare-release')
    os.chdir('../solr')
    cmd = 'ant -Dversion=%s' % version
    if gpgKeyID is not None:
      cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
    else:
      cmd += ' prepare-release-no-sign'
  
    if gpgPassword is not None:
      runAndSendGPGPassword(cmd, gpgPassword)
    else:
      run(cmd)
      
    print('  done!')
    print()
    return rev
  
  reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
  reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE)
  reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE)
  def checkDOAPfiles(version):
    # In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced.
    errorMessages = []
    for product in 'lucene', 'solr':
      url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product)
      distpage = load(url)
      releases = set()
      for regex in reVersion1, reVersion2:
        for tup in regex.findall(distpage):
          if tup[0] in ('1', '2'):                    # Ignore 1.X and 2.X releases
            continue
          releases.add(normalizeVersion(tup))
      doapNS = '{http://usefulinc.com/ns/doap#}'
      xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
      doapFile = "dev-tools/doap/%s.rdf" % product
      treeRoot = ET.parse(doapFile).getroot()
      doapRevisions = set()
      for revision in treeRoot.findall(xpathRevision):
        match = reDoapRevision.match(revision.text)
        if (match is not None):
          if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
            doapRevisions.add(normalizeVersion(match.groups()))
        else:
          errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
      missingDoapRevisions = set()
      for release in releases:
        if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
          missingDoapRevisions.add(release)
      if len(missingDoapRevisions) > 0:
        errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
    if (len(errorMessages) > 0):
      raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)'
                         % '\n'.join(errorMessages))
  
  def normalizeVersion(tup):
    suffix = ''
    if tup[-1] is not None and tup[-1].lower() == '-alpha':
      tup = tup[:(len(tup) - 1)]
      suffix = '-ALPHA'
    elif tup[-1] is not None and tup[-1].lower() == '-beta':
      tup = tup[:(len(tup) - 1)]
      suffix = '-BETA'
    while tup[-1] in ('', None):
      tup = tup[:(len(tup) - 1)]
    while len(tup) < 3:
      tup = tup + ('0',)
    return '.'.join(tup) + suffix
  
  def pushLocal(version, root, rev, rcNum, localDir):
    print('Push local [%s]...' % localDir)
    os.makedirs(localDir)
  
    dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev)
    os.makedirs('%s/%s/lucene' % (localDir, dir))
    os.makedirs('%s/%s/solr' % (localDir, dir))
    print('  Lucene')
    os.chdir('%s/lucene/dist' % root)
    print('    zip...')
    if os.path.exists('lucene.tar.bz2'):
      os.remove('lucene.tar.bz2')
    run('tar cjf lucene.tar.bz2 *')
  
    os.chdir('%s/%s/lucene' % (localDir, dir))
    print('    unzip...')
    run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root)
    os.remove('%s/lucene/dist/lucene.tar.bz2' % root)
  
    print('  Solr')
    os.chdir('%s/solr/package' % root)
    print('    zip...')
    if os.path.exists('solr.tar.bz2'):
      os.remove('solr.tar.bz2')
    run('tar cjf solr.tar.bz2 *')
    print('    unzip...')
    os.chdir('%s/%s/solr' % (localDir, dir))
    run('tar xjf "%s/solr/package/solr.tar.bz2"' % root)
    os.remove('%s/solr/package/solr.tar.bz2' % root)
  
    print('  KEYS')
    run('wget http://home.apache.org/keys/group/lucene.asc')
    os.rename('lucene.asc', 'KEYS')
    run('chmod a+r-w KEYS')
    run('cp KEYS ../lucene')
  
    print('  chmod...')
    os.chdir('..')
    run('chmod -R a+rX-w .')
  
    print('  done!')
    return 'file://%s/%s' % (os.path.abspath(localDir), dir)
  
  def read_version(path):
    version_props_file = os.path.join(path, 'lucene', 'version.properties')
    return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1)
  
  def parse_config():
    epilogue = textwrap.dedent('''
      Example usage for a Release Manager:
      python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
    ''')
    description = 'Utility to build, push, and test a release.'
    parser = argparse.ArgumentParser(description=description, epilog=epilogue,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false',
                        help='Use the already built release in the provided checkout')
    parser.add_argument('--push-local', metavar='PATH',
                        help='Push the release to the local path')
    parser.add_argument('--sign', metavar='KEYID',
                        help='Sign the release with the given gpg key')
    parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
                        help='Release Candidate number.  Default: 1')
    parser.add_argument('--root', metavar='PATH', default='.',
                        help='Root of Git working tree for lucene-solr.  Default: "." (the current directory)')
    config = parser.parse_args()
  
    if not config.prepare and config.sign:
      parser.error('Cannot sign already built release')
    if config.push_local is not None and os.path.exists(config.push_local):
      parser.error('Cannot push to local path that already exists')
    if config.rc_num <= 0:
      parser.error('Release Candidate number must be a positive integer')
    if not os.path.isdir(config.root):
      parser.error('Root path "%s" is not a directory' % config.root)
    cwd = os.getcwd()
    os.chdir(config.root)
    config.root = os.getcwd() # Absolutize root dir
    if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]):
      parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root)
    os.chdir(cwd)
  
    config.version = read_version(config.root)
    print('Building version: %s' % config.version)
  
    if config.sign:
      sys.stdout.flush()
      import getpass
      config.key_id = config.sign
      config.key_password = getpass.getpass('Enter GPG keystore password: ')
    else:
      config.gpg_password = None
  
    return config
  
  def check_cmdline_tools():  # Fail fast if there are cmdline tool problems
    if os.system('git --version >/dev/null 2>/dev/null'):
      raise RuntimeError('"git --version" returned a non-zero exit code.')
    antVersion = os.popen('ant -version').read().strip()
    if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'):
      raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion)
    
  def main():
    check_cmdline_tools()
  
    c = parse_config()
  
    if c.prepare:
      rev = prepare(c.root, c.version, c.key_id, c.key_password)
    else:
      os.chdir(root)
      rev = open('rev.txt', encoding='UTF-8').read()
  
    if c.push_local:
      url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
    else:
      url = None
  
    if url is not None:
      print('  URL: %s' % url)
      print('Next run the smoker tester:')
      p = re.compile(".*/")
      m = p.match(sys.argv[0])
      print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url))
  
  if __name__ == '__main__':
    try:
      main()
    except KeyboardInterrupt:
      print('Keyboard interrupt...exiting')
  
  