text: #!/usr/bin/env python
  
  # Copyright 2016 The Kubernetes Authors.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """Tests for bootstrap."""
  
  # pylint: disable=too-many-public-methods, too-many-branches, too-many-locals
  # pylint: disable=protected-access, attribute-defined-outside-init, too-many-statements
  
  import argparse
  import collections
  import json
  import os
  import re
  import select
  import signal
  import subprocess
  import tempfile
  import time
  import unittest
  
  import bootstrap
  
  import yaml
  
  
  BRANCH = 'random_branch'
  BUILD = 'random_build'
  FAIL = ['/bin/bash', '-c', 'exit 1']
  JOB = 'random_job'
  PASS = ['/bin/bash', '-c', 'exit 0']
  PULL = 12345
  REPO = 'github.com/random_org/random_repo'
  ROBOT = 'fake-service-account.json'
  ROOT = '/random/root'
  UPLOAD = 'fake-gs://fake-bucket'
  
  
  class Stub(object):
      """Replace thing.param with replacement until exiting with."""
      def __init__(self, thing, param, replacement):
          self.thing = thing
          self.param = param
          self.replacement = replacement
          self.old = getattr(thing, param)
          setattr(thing, param, self.replacement)
  
      def __enter__(self, *a, **kw):
          return self.replacement
  
      def __exit__(self, *a, **kw):
          setattr(self.thing, self.param, self.old)
  
  
  class FakeCall(object):
      def __init__(self):
          self.calls = []
  
      def __call__(self, *a, **kw):
          self.calls.append((a, kw))
  
  
  class FakeSubprocess(object):
      """Keep track of calls."""
      def __init__(self):
          self.calls = []
  
      def __call__(self, cmd, *a, **kw):
          self.calls.append((cmd, a, kw))
  
  
  # pylint: disable=invalid-name
  def Pass(*_a, **_kw):
      """Do nothing."""
      pass
  
  
  def Truth(*_a, **_kw):
      """Always true."""
      return True
  
  
  def Bomb(*a, **kw):
      """Always raise."""
      raise AssertionError('Should not happen', a, kw)
  # pylint: enable=invalid-name
  
  
  class ReadAllTest(unittest.TestCase):
      endless = 0
      ended = time.time() - 50
      number = 0
      end = -1
  
      def fileno(self):
          return self.end
  
      def readline(self):
          line = 'line %d\n' % self.number
          self.number += 1
          return line
  
      def test_read_more(self):
          """Read lines until we clear the buffer, noting there may be more."""
          lines = []
          total = 10
          def more_lines(*_a, **_kw):
              if len(lines) < total:
                  return [self], [], []
              return [], [], []
          with Stub(select, 'select', more_lines):
              done = bootstrap.read_all(self.endless, self, lines.append)
  
          self.assertFalse(done)
          self.assertEquals(total, len(lines))
          expected = ['line %d' % d for d in range(total)]
          self.assertEquals(expected, lines)
  
      def test_read_expired(self):
          """Read nothing as we are expired, noting there may be more."""
          lines = []
          with Stub(select, 'select', lambda *a, **kw: ([], [], [])):
              done = bootstrap.read_all(self.ended, self, lines.append)
  
          self.assertFalse(done)
          self.assertFalse(lines)
  
      def test_read_end(self):
          """Note we reached the end of the stream."""
          lines = []
          with Stub(select, 'select', lambda *a, **kw: ([self], [], [])):
              with Stub(self, 'readline', lambda: ''):
                  done = bootstrap.read_all(self.endless, self, lines.append)
  
          self.assertTrue(done)
  
  
  class TerminateTest(unittest.TestCase):
      """Tests for termiante()."""
      pid = 1234
      pgid = 5555
      terminated = False
      killed = False
  
      def terminate(self):
          self.terminated = True
  
      def kill(self):
          self.killed = True
  
      def getpgid(self, pid):
          self.got = pid
          return self.pgid
  
      def killpg(self, pgig, sig):
          self.killed_pg = (pgig, sig)
  
      def test_terminate_later(self):
          """Do nothing if end is in the future."""
          timeout = bootstrap.terminate(time.time() + 50, self, False)
          self.assertFalse(timeout)
  
      def test_terminate_never(self):
          """Do nothing if end is zero."""
          timeout = bootstrap.terminate(0, self, False)
          self.assertFalse(timeout)
  
      def test_terminate_terminate(self):
          """Terminate pid if after end and kill is false."""
          timeout = bootstrap.terminate(time.time() - 50, self, False)
          self.assertTrue(timeout)
          self.assertFalse(self.killed)
          self.assertTrue(self.terminated)
  
      def test_terminate_kill(self):
          """Kill process group if after end and kill is true."""
          with Stub(os, 'getpgid', self.getpgid), Stub(os, 'killpg', self.killpg):
              timeout = bootstrap.terminate(time.time() - 50, self, True)
          self.assertTrue(timeout)
          self.assertFalse(self.terminated)
          self.assertTrue(self.killed)
          self.assertEquals(self.pid, self.got)
          self.assertEquals(self.killed_pg, (self.pgid, signal.SIGKILL))
  
  
  class SubprocessTest(unittest.TestCase):
      """Tests for call()."""
  
      def test_stdin(self):
          """Will write to subprocess.stdin."""
          with self.assertRaises(subprocess.CalledProcessError) as cpe:
              bootstrap._call(0, ['/bin/bash'], stdin='exit 92')
          self.assertEquals(92, cpe.exception.returncode)
  
      def test_check_true(self):
          """Raise on non-zero exit codes if check is set."""
          with self.assertRaises(subprocess.CalledProcessError):
              bootstrap._call(0, FAIL, check=True)
  
          bootstrap._call(0, PASS, check=True)
  
      def test_check_default(self):
          """Default to check=True."""
          with self.assertRaises(subprocess.CalledProcessError):
              bootstrap._call(0, FAIL)
  
          bootstrap._call(0, PASS)
  
      @staticmethod
      def test_check_false():
          """Never raise when check is not set."""
          bootstrap._call(0, FAIL, check=False)
          bootstrap._call(0, PASS, check=False)
  
      def test_output(self):
          """Output is returned when requested."""
          cmd = ['/bin/bash', '-c', 'echo hello world']
          self.assertEquals(
              'hello world\n', bootstrap._call(0, cmd, output=True))
  
      def test_zombie(self):
          with self.assertRaises(subprocess.CalledProcessError):
              # make a zombie
              bootstrap._call(0, ['/bin/bash', '-c', 'A=$BASHPID && ( kill -STOP $A ) & exit 1'])
  
  
  class PullRefsTest(unittest.TestCase):
      """Tests for pull_ref, branch_ref, ref_has_shas, and pull_numbers."""
  
      def test_multiple_no_shas(self):
          """Test master,1111,2222."""
          self.assertEqual(
              bootstrap.pull_ref('master,123,456'),
              ([
                  'master',
                  '+refs/pull/123/head:refs/pr/123',
                  '+refs/pull/456/head:refs/pr/456',
              ], [
                  'FETCH_HEAD',
                  'refs/pr/123',
                  'refs/pr/456',
              ]),
          )
  
      def test_pull_has_shas(self):
          self.assertTrue(bootstrap.ref_has_shas('master:abcd'))
          self.assertFalse(bootstrap.ref_has_shas('123'))
          self.assertFalse(bootstrap.ref_has_shas(123))
          self.assertFalse(bootstrap.ref_has_shas(None))
  
      def test_pull_numbers(self):
          self.assertListEqual(bootstrap.pull_numbers(123), ['123'])
          self.assertListEqual(bootstrap.pull_numbers('master:abcd'), [])
          self.assertListEqual(
              bootstrap.pull_numbers('master:abcd,123:qwer,124:zxcv'),
              ['123', '124'])
  
      def test_pull_ref(self):
          self.assertEqual(
              bootstrap.pull_ref('master:abcd,123:effe'),
              (['master', '+refs/pull/123/head:refs/pr/123'], ['abcd', 'effe'])
          )
          self.assertEqual(
              bootstrap.pull_ref('123'),
              (['+refs/pull/123/merge'], ['FETCH_HEAD'])
          )
  
      def test_branch_ref(self):
          self.assertEqual(
              bootstrap.branch_ref('branch:abcd'),
              (['branch'], ['abcd'])
          )
          self.assertEqual(
              bootstrap.branch_ref('master'),
              (['master'], ['FETCH_HEAD'])
          )
  
  
  class ChooseSshKeyTest(unittest.TestCase):
      """Tests for choose_ssh_key()."""
      def test_empty(self):
          """Do not change environ if no ssh key."""
          fake_env = {}
          with Stub(os, 'environ', fake_env):
              with bootstrap.choose_ssh_key(''):
                  self.assertFalse(fake_env)
  
      def test_full(self):
          fake_env = {}
          with Stub(os, 'environ', fake_env):
              with bootstrap.choose_ssh_key('hello there'):
                  self.assertIn('GIT_SSH', fake_env)
                  with open(fake_env['GIT_SSH']) as fp:
                      buf = fp.read()
                  self.assertIn('hello there', buf)
                  self.assertIn('ssh ', buf)
                  self.assertIn(' -i ', buf)
              self.assertFalse(fake_env)  # Resets env
  
      def test_full_old_value(self):
          fake_env = {'GIT_SSH': 'random-value'}
          old_env = dict(fake_env)
          with Stub(os, 'environ', fake_env):
              with bootstrap.choose_ssh_key('hello there'):
                  self.assertNotEqual(old_env, fake_env)
              self.assertEquals(old_env, fake_env)
  
  
  class CheckoutTest(unittest.TestCase):
      """Tests for checkout()."""
  
      def test_clean(self):
          """checkout cleans and resets if asked to."""
          fake = FakeSubprocess()
          with Stub(os, 'chdir', Pass):
              bootstrap.checkout(fake, REPO, None, PULL, clean=True)
  
          self.assertTrue(any(
              'clean' in cmd for cmd, _, _ in fake.calls if 'git' in cmd))
          self.assertTrue(any(
              'reset' in cmd for cmd, _, _ in fake.calls if 'git' in cmd))
  
      def test_fetch_retries(self):
          self.tries = 0
          expected_attempts = 3
          def third_time_charm(cmd, *_a, **_kw):
              if 'fetch' not in cmd:  # init/checkout are unlikely to fail
                  return
              self.tries += 1
              if self.tries != expected_attempts:
                  raise subprocess.CalledProcessError(128, cmd, None)
          with Stub(os, 'chdir', Pass):
              with Stub(time, 'sleep', Pass):
                  bootstrap.checkout(third_time_charm, REPO, None, PULL)
          self.assertEquals(expected_attempts, self.tries)
  
      def test_pull(self):
          """checkout fetches the right ref for a pull."""
          fake = FakeSubprocess()
          with Stub(os, 'chdir', Pass):
              bootstrap.checkout(fake, REPO, None, PULL)
  
          expected_ref = bootstrap.pull_ref(PULL)[0][0]
          self.assertTrue(any(
              expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
  
      def test_branch(self):
          """checkout fetches the right ref for a branch."""
          fake = FakeSubprocess()
          with Stub(os, 'chdir', Pass):
              bootstrap.checkout(fake, REPO, BRANCH, None)
  
          expected_ref = BRANCH
          self.assertTrue(any(
              expected_ref in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
  
      def test_repo(self):
          """checkout initializes and fetches the right repo."""
          fake = FakeSubprocess()
          with Stub(os, 'chdir', Pass):
              bootstrap.checkout(fake, REPO, BRANCH, None)
  
          expected_uri = 'https://%s' % REPO
          self.assertTrue(any(
              expected_uri in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
  
      def test_branch_xor_pull(self):
          """Either branch or pull specified, not both."""
          with Stub(os, 'chdir', Bomb):
              with self.assertRaises(ValueError):
                  bootstrap.checkout(Bomb, REPO, None, None)
              with self.assertRaises(ValueError):
                  bootstrap.checkout(Bomb, REPO, BRANCH, PULL)
  
      def test_happy(self):
          """checkout sanity check."""
          fake = FakeSubprocess()
          with Stub(os, 'chdir', Pass):
              bootstrap.checkout(fake, REPO, BRANCH, None)
  
          self.assertTrue(any(
              '--tags' in cmd for cmd, _, _ in fake.calls if 'fetch' in cmd))
          self.assertTrue(any(
              'FETCH_HEAD' in cmd for cmd, _, _ in fake.calls
              if 'checkout' in cmd))
  
  class ParseReposTest(unittest.TestCase):
      def test_bare(self):
          """--bare works."""
          self.assertFalse(
              bootstrap.parse_repos(FakeArgs(repo=[], branch=[], pull=[], bare=True)))
  
      def test_deprecated_branch(self):
          """--repo=foo --branch=bbb works"""
          self.assertEquals(
              {'foo': ('bbb', '')},
              bootstrap.parse_repos(FakeArgs(repo=['foo'], branch='bbb', pull='')))
  
      def test_deprecated_branch_commit(self):
          """--repo=foo --branch=bbb:1234 works"""
          self.assertEquals(
              {'foo': ('bbb:1234', '')},
              bootstrap.parse_repos(FakeArgs(repo=['foo'], branch='bbb:1234', pull='')))
  
      def test_depre_branch_repo_commit(self):
          """--repo=foo=master:aaa --branch=bar is not allowed"""
          with self.assertRaises(ValueError):
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=master:aaa'], branch='master'))
          with self.assertRaises(ValueError):
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=master'], branch='bar'))
  
      def test_deprecated_pull(self):
          """--repo=foo --pull=123 works."""
          self.assertEquals(
              {'foo': ('', '123:abc,333:ddd')},
              bootstrap.parse_repos(FakeArgs(repo=['foo'], branch='', pull='123:abc,333:ddd')))
  
  
      def test_depre_pull_repo_commit(self):
          """--repo=foo=master:aaa --pull=123:abc is not allowed"""
          with self.assertRaises(ValueError):
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=master:aaa'], branch='', pull='123:abc'))
          with self.assertRaises(ValueError):
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=master'], branch='', pull='123:abc'))
  
      def test_plain(self):
          """"--repo=foo equals foo=master."""
          self.assertEquals(
              {'foo': ('master', '')},
              bootstrap.parse_repos(FakeArgs(repo=['foo'], branch='', pull='')))
  
      def test_branch(self):
          """--repo=foo=branch."""
          self.assertEquals(
              {'foo': ('this', '')},
              bootstrap.parse_repos(
                  FakeArgs(repo=['foo=this'], branch='', pull='')))
  
      def test_branch_commit(self):
          """--repo=foo=branch:commit works."""
          self.assertEquals(
              {'foo': ('this:abcd', '')},
              bootstrap.parse_repos(
                  FakeArgs(repo=['foo=this:abcd'], branch='', pull='')))
  
      def test_pull(self):
          """--repo=foo=111,222 works"""
          self.assertEquals(
              {'foo': ('', '111,222')},
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=111,222'], branch='', pull='')))
  
      def test_pull_branch(self):
          """--repo=foo=master,111,222 works"""
          self.assertEquals(
              {'foo': ('', 'master,111,222')},
              bootstrap.parse_repos(
                  FakeArgs(repo=['foo=master,111,222'], branch='', pull='')))
  
      def test_pull_release_branch(self):
          """--repo=foo=release-3.14,&a-fancy%_branch+:abcd,222 works"""
          self.assertEquals(
              {'foo': ('', 'release-3.14,&a-fancy%_branch+:abcd,222')},
              bootstrap.parse_repos(
                  FakeArgs(repo=['foo=release-3.14,&a-fancy%_branch+:abcd,222'],
                           branch='', pull='')))
  
      def test_pull_branch_commit(self):
          """--repo=foo=master,111,222 works"""
          self.assertEquals(
              {'foo': ('', 'master:aaa,111:bbb,222:ccc')},
              bootstrap.parse_repos(FakeArgs(
                  repo=['foo=master:aaa,111:bbb,222:ccc'], branch='', pull='')))
  
      def test_multi_repo(self):
          """--repo=foo=master,111,222 bar works"""
          self.assertEquals(
              {
                  'foo': ('', 'master:aaa,111:bbb,222:ccc'),
                  'bar': ('master', '')},
              bootstrap.parse_repos(
                  FakeArgs(branch='', pull='', repo=[
                      'foo=master:aaa,111:bbb,222:ccc',
                      'bar',
                  ])))
  
  
  class GSUtilTest(unittest.TestCase):
      """Tests for GSUtil."""
      def test_upload_json(self):
          fake = FakeSubprocess()
          gsutil = bootstrap.GSUtil(fake)
          gsutil.upload_json('fake_path', {'wee': 'fun'})
          self.assertTrue(any(
              'application/json' in a for a in fake.calls[0][0]))
          self.assertIn('stdin', fake.calls[0][2])  # kwargs
  
      def test_upload_text_cached(self):
          fake = FakeSubprocess()
          gsutil = bootstrap.GSUtil(fake)
          gsutil.upload_text('fake_path', 'hello world', cached=True)
          self.assertFalse(any(
              'Cache-Control' in a and 'max-age' in a
              for a in fake.calls[0][0]))
          self.assertIn('stdin', fake.calls[0][2])  # kwargs
  
      def test_upload_text_default(self):
          fake = FakeSubprocess()
          gsutil = bootstrap.GSUtil(fake)
          gsutil.upload_text('fake_path', 'hello world')
          self.assertFalse(any(
              'Cache-Control' in a and 'max-age' in a
              for a in fake.calls[0][0]))
          self.assertIn('stdin', fake.calls[0][2])  # kwargs
  
      def test_upload_text_uncached(self):
          fake = FakeSubprocess()
          gsutil = bootstrap.GSUtil(fake)
          gsutil.upload_text('fake_path', 'hello world', cached=False)
          self.assertTrue(any(
              'Cache-Control' in a and 'max-age' in a
              for a in fake.calls[0][0]))
          self.assertIn('stdin', fake.calls[0][2])  # kwargs
  
  
  class FakeGSUtil(object):
      generation = 123
  
      def __init__(self):
          self.cats = []
          self.jsons = []
          self.stats = []
          self.texts = []
  
      def cat(self, *a, **kw):
          self.cats.append((a, kw))
          return 'this is not a list'
  
      def stat(self, *a, **kw):
          self.stats.append((a, kw))
          return 'Generation: %s' % self.generation
  
      def upload_text(self, *args, **kwargs):
          self.texts.append((args, kwargs))
  
      def upload_json(self, *args, **kwargs):
          self.jsons.append((args, kwargs))
  
  class GubernatorUriTest(unittest.TestCase):
      def create_path(self, uri):
          self.fake_path = FakePath()
          self.fake_path.build_log = uri
          return self.fake_path
  
      def test_non_gs(self):
          uri = 'hello/world'
          self.assertEquals('hello', bootstrap.gubernator_uri(self.create_path(uri)))
  
      def test_multiple_gs(self):
          uri = 'gs://hello/gs://there'
          self.assertEquals(
              bootstrap.GUBERNATOR + '/hello/gs:',
              bootstrap.gubernator_uri(self.create_path(uri)))
  
      def test_gs(self):
          uri = 'gs://blah/blah/blah.txt'
          self.assertEquals(
              bootstrap.GUBERNATOR + '/blah/blah',
              bootstrap.gubernator_uri(self.create_path(uri)))
  
  
  
  class AppendResultTest(unittest.TestCase):
      """Tests for append_result()."""
  
      def test_new_job(self):
          """Stat fails when the job doesn't exist."""
          gsutil = FakeGSUtil()
          build = 123
          version = 'v.interesting'
          success = True
          def fake_stat(*_a, **_kw):
              raise subprocess.CalledProcessError(1, ['gsutil'], None)
          gsutil.stat = fake_stat
          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
          cache = gsutil.jsons[0][0][1]
          self.assertEquals(1, len(cache))
  
      def test_collision_cat(self):
          """cat fails if the cache has been updated."""
          gsutil = FakeGSUtil()
          build = 42
          version = 'v1'
          success = False
          generations = ['555', '444']
          orig_stat = gsutil.stat
          def fake_stat(*a, **kw):
              gsutil.generation = generations.pop()
              return orig_stat(*a, **kw)
          def fake_cat(_, gen):
              if gen == '555':  # Which version is requested?
                  return '[{"hello": 111}]'
              raise subprocess.CalledProcessError(1, ['gsutil'], None)
          with Stub(bootstrap, 'random_sleep', Pass):
              with Stub(gsutil, 'stat', fake_stat):
                  with Stub(gsutil, 'cat', fake_cat):
                      bootstrap.append_result(
                          gsutil, 'fake_path', build, version, success)
          self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons)
          self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons)
  
      def test_collision_upload(self):
          """Test when upload_json tries to update an old version."""
          gsutil = FakeGSUtil()
          build = 42
          version = 'v1'
          success = False
          generations = [555, 444]
          orig = gsutil.upload_json
          def fake_upload(path, cache, generation):
              if generation == '555':
                  return orig(path, cache, generation=generation)
              raise subprocess.CalledProcessError(128, ['gsutil'], None)
          orig_stat = gsutil.stat
          def fake_stat(*a, **kw):
              gsutil.generation = generations.pop()
              return orig_stat(*a, **kw)
          def fake_cat(*_a, **_kw):
              return '[{"hello": 111}]'
          gsutil.stat = fake_stat
          gsutil.upload_json = fake_upload
          gsutil.cat = fake_cat
          with Stub(bootstrap, 'random_sleep', Pass):
              bootstrap.append_result(
                  gsutil, 'fake_path', build, version, success)
          self.assertIn('generation', gsutil.jsons[-1][1], gsutil.jsons)
          self.assertEquals('555', gsutil.jsons[-1][1]['generation'], gsutil.jsons)
  
      def test_handle_junk(self):
          gsutil = FakeGSUtil()
          gsutil.cat = lambda *a, **kw: '!@!$!@$@!$'
          build = 123
          version = 'v.interesting'
          success = True
          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
          cache = gsutil.jsons[0][0][1]
          self.assertEquals(1, len(cache))
          self.assertIn(build, cache[0].values())
          self.assertIn(version, cache[0].values())
  
      def test_passed_is_bool(self):
          build = 123
          version = 'v.interesting'
          def try_run(success):
              gsutil = FakeGSUtil()
              bootstrap.append_result(gsutil, 'fake_path', build, version, success)
              cache = gsutil.jsons[0][0][1]
              self.assertTrue(isinstance(cache[0]['passed'], bool))
  
          try_run(1)
          try_run(0)
          try_run(None)
          try_run('')
          try_run('hello')
          try_run('true')
  
      def test_truncate(self):
          old = json.dumps({n: True for n in range(100000)})
          gsutil = FakeGSUtil()
          build = 123
          version = 'v.interesting'
          success = True
          bootstrap.append_result(gsutil, 'fake_path', build, version, success)
          cache = gsutil.jsons[0][0][1]
          self.assertLess(len(cache), len(old))
  
  
  class FinishTest(unittest.TestCase):
      """Tests for finish()."""
      def setUp(self):
          self.stubs = [
              Stub(bootstrap.GSUtil, 'upload_artifacts', Pass),
              Stub(bootstrap, 'append_result', Pass),
              Stub(os.path, 'isfile', Pass),
              Stub(os.path, 'isdir', Pass),
          ]
  
      def tearDown(self):
          for stub in self.stubs:
              with stub:
                  pass
  
      def test_no_version(self):
          gsutil = FakeGSUtil()
          paths = FakePath()
          success = True
          artifacts = 'not-a-dir'
          no_version = ''
          version = 'should not have found it'
          repos = repo({REPO: ('master', '')})
          with Stub(bootstrap, 'metadata', lambda *a: {'random-meta': version}):
              bootstrap.finish(gsutil, paths, success, artifacts,
                               BUILD, no_version, repos, FakeCall())
          bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, repos, FakeCall())
          calls = gsutil.jsons[-1]
          # json data is second positional argument
          self.assertNotIn('job-version', calls[0][1])
          self.assertNotIn('version', calls[0][1])
          self.assertTrue(calls[0][1].get('metadata'))
  
      def test_metadata_version(self):
          """Test that we will extract version info from metadata."""
          self.check_metadata_version('job-version')
          self.check_metadata_version('version')
  
      def check_metadata_version(self, key):
          gsutil = FakeGSUtil()
          paths = FakePath()
          success = True
          artifacts = 'not-a-dir'
          no_version = ''
          version = 'found it'
          with Stub(bootstrap, 'metadata', lambda *a: {key: version}):
              bootstrap.finish(gsutil, paths, success, artifacts, BUILD, no_version, REPO, FakeCall())
          calls = gsutil.jsons[-1]
          # Meta is second positional argument
          self.assertEquals(version, calls[0][1].get('job-version'))
          self.assertEquals(version, calls[0][1].get('version'))
  
      def test_ignore_err_up_artifacts(self):
          paths = FakePath()
          gsutil = FakeGSUtil()
          local_artifacts = None
          build = 123
          version = 'v1.terrible'
          success = True
          calls = []
          with Stub(os.path, 'isdir', lambda _: True):
              with Stub(os, 'walk', lambda d: [(True, True, True)]):
                  def fake_upload(*a, **kw):
                      calls.append((a, kw))
                      raise subprocess.CalledProcessError(1, ['fakecmd'], None)
                  gsutil.upload_artifacts = fake_upload
                  repos = repo({REPO: ('master', '')})
                  bootstrap.finish(
                      gsutil, paths, success, local_artifacts,
                      build, version, repos, FakeCall())
                  self.assertTrue(calls)
  
  
      def test_ignore_error_uploadtext(self):
          paths = FakePath()
          gsutil = FakeGSUtil()
          local_artifacts = None
          build = 123
          version = 'v1.terrible'
          success = True
          calls = []
          with Stub(os.path, 'isdir', lambda _: True):
              with Stub(os, 'walk', lambda d: [(True, True, True)]):
                  def fake_upload(*a, **kw):
                      calls.append((a, kw))
                      raise subprocess.CalledProcessError(1, ['fakecmd'], None)
                  gsutil.upload_artifacts = Pass
                  gsutil.upload_text = fake_upload
                  repos = repo({REPO: ('master', '')})
                  bootstrap.finish(
                      gsutil, paths, success, local_artifacts,
                      build, version, repos, FakeCall())
                  self.assertTrue(calls)
                  self.assertGreater(calls, 1)
  
      def test_skip_upload_artifacts(self):
          """Do not upload artifacts dir if it doesn't exist."""
          paths = FakePath()
          gsutil = FakeGSUtil()
          local_artifacts = None
          build = 123
          version = 'v1.terrible'
          success = True
          calls = []
          with Stub(os.path, 'isdir', lambda _: False):
              with Stub(bootstrap.GSUtil, 'upload_artifacts', Bomb):
                  repos = repo({REPO: ('master', '')})
                  bootstrap.finish(
                      gsutil, paths, success, local_artifacts,
                      build, version, repos, FakeCall())
                  self.assertFalse(calls)
  
  
  class MetadataTest(unittest.TestCase):
  
      def test_always_set_metadata(self):
          repos = repo({REPO: ('master', '')})
          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall())
          self.assertIn('repo', meta)
          self.assertEquals(REPO, meta['repo'])
  
      def test_multi_repo(self):
          repos = repo({REPO: ('foo', ''), 'other-repo': ('', '123,456')})
          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', FakeCall())
          self.assertIn('repo', meta)
          self.assertEquals(REPO, meta['repo'])
          self.assertIn(REPO, meta.get('repos'))
          self.assertEquals('foo', meta['repos'][REPO])
          self.assertIn('other-repo', meta.get('repos'))
          self.assertEquals('123,456', meta['repos']['other-repo'])
  
  
  SECONDS = 10
  
  
  def fake_environment(set_home=True, set_node=True, set_job=True, **kwargs):
      if set_home:
          kwargs.setdefault(bootstrap.HOME_ENV, '/fake/home-dir')
      if set_node:
          kwargs.setdefault(bootstrap.NODE_ENV, 'fake-node')
      if set_job:
          kwargs.setdefault(bootstrap.JOB_ENV, JOB)
      return kwargs
  
  
  class BuildNameTest(unittest.TestCase):
      """Tests for build_name()."""
  
      def test_auto(self):
          """Automatically select a build if not done by user."""
          with Stub(os, 'environ', fake_environment()) as fake:
              bootstrap.build_name(SECONDS)
              self.assertTrue(fake[bootstrap.BUILD_ENV])
  
      def test_manual(self):
          """Respect user-selected build."""
          with Stub(os, 'environ', fake_environment()) as fake:
              truth = 'erick is awesome'
              fake[bootstrap.BUILD_ENV] = truth
              self.assertEquals(truth, fake[bootstrap.BUILD_ENV])
  
      def test_unique(self):
          """New build every minute."""
          with Stub(os, 'environ', fake_environment()) as fake:
              bootstrap.build_name(SECONDS)
              first = fake[bootstrap.BUILD_ENV]
              del fake[bootstrap.BUILD_ENV]
              bootstrap.build_name(SECONDS + 60)
              self.assertNotEqual(first, fake[bootstrap.BUILD_ENV])
  
  
  class SetupCredentialsTest(unittest.TestCase):
      """Tests for setup_credentials()."""
  
      def setUp(self):
          keys = {
              bootstrap.GCE_KEY_ENV: 'fake-key',
              bootstrap.SERVICE_ACCOUNT_ENV: 'fake-service-account.json',
          }
          self.env = fake_environment(**keys)
  
      def test_norobot_noupload_noenv(self):
          """Can avoid setting up credentials."""
          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
          with Stub(os, 'environ', self.env):
              bootstrap.setup_credentials(Bomb, None, None)
  
      def test_upload_no_robot_raises(self):
          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
          with Stub(os, 'environ', self.env):
              with self.assertRaises(ValueError):
                  bootstrap.setup_credentials(Pass, None, 'gs://fake')
  
  
      def test_application_credentials(self):
          """Raise if GOOGLE_APPLICATION_CREDENTIALS does not exist."""
          del self.env[bootstrap.SERVICE_ACCOUNT_ENV]
          with Stub(os, 'environ', self.env) as fake:
              gac = 'FAKE_CREDS.json'
              fake['HOME'] = 'kansas'
              with Stub(os.path, 'isfile', lambda p: p != gac):
                  with self.assertRaises(IOError):
                      bootstrap.setup_credentials(Pass, gac, UPLOAD)
  
              with Stub(os.path, 'isfile', Truth):
                  call = lambda *a, **kw: 'robot'
                  bootstrap.setup_credentials(call, gac, UPLOAD)
              # setup_creds should set SERVICE_ACCOUNT_ENV
              self.assertEquals(gac, fake.get(bootstrap.SERVICE_ACCOUNT_ENV))
              # now that SERVICE_ACCOUNT_ENV is set, it should try to activate
              # this
              with Stub(os.path, 'isfile', lambda p: p != gac):
                  with self.assertRaises(IOError):
                      bootstrap.setup_credentials(Pass, None, UPLOAD)
  
  
  class SetupMagicEnvironmentTest(unittest.TestCase):
      def test_workspace(self):
          """WORKSPACE exists, equals HOME and is set to cwd."""
          env = fake_environment()
          cwd = '/fake/random-location'
          with Stub(os, 'environ', env):
              with Stub(os, 'getcwd', lambda: cwd):
                  bootstrap.setup_magic_environment(JOB)
  
          self.assertIn(bootstrap.WORKSPACE_ENV, env)
          self.assertEquals(env[bootstrap.HOME_ENV], env[bootstrap.WORKSPACE_ENV])
          self.assertEquals(cwd, env[bootstrap.WORKSPACE_ENV])
  
      def test_job_env_mismatch(self):
          env = fake_environment()
          with Stub(os, 'environ', env):
              self.assertNotEquals('this-is-a-job', env[bootstrap.JOB_ENV])
              bootstrap.setup_magic_environment('this-is-a-job')
              self.assertEquals('this-is-a-job', env[bootstrap.JOB_ENV])
  
      def test_expected(self):
          env = fake_environment()
          del env[bootstrap.JOB_ENV]
          del env[bootstrap.NODE_ENV]
          with Stub(os, 'environ', env):
              bootstrap.setup_magic_environment(JOB)
  
          def check(name):
              self.assertIn(name, env)
  
          # Some of these are probably silly to check...
          # TODO(fejta): remove as many of these from our infra as possible.
          check(bootstrap.JOB_ENV)
          check(bootstrap.CLOUDSDK_ENV)
          check(bootstrap.BOOTSTRAP_ENV)
          check(bootstrap.WORKSPACE_ENV)
          self.assertNotIn(bootstrap.SERVICE_ACCOUNT_ENV, env)
  
      def test_node_present(self):
          expected = 'whatever'
          env = {bootstrap.NODE_ENV: expected}
          with Stub(os, 'environ', env):
              self.assertEquals(expected, bootstrap.node())
          self.assertEquals(expected, env[bootstrap.NODE_ENV])
  
      def test_node_missing(self):
          env = {}
          with Stub(os, 'environ', env):
              expected = bootstrap.node()
              self.assertTrue(expected)
          self.assertEquals(expected, env[bootstrap.NODE_ENV])
  
  
  
      def test_cloud_sdk_config(self):
          cwd = 'now-here'
          env = fake_environment()
          with Stub(os, 'environ', env):
              with Stub(os, 'getcwd', lambda: cwd):
                  bootstrap.setup_magic_environment(JOB)
  
  
          self.assertTrue(env[bootstrap.CLOUDSDK_ENV].startswith(cwd))
  
  
  class FakePath(object):
      artifacts = 'fake_artifacts'
      pr_latest = 'fake_pr_latest'
      pr_build_link = 'fake_pr_link'
      build_log = 'fake_log_path'
      pr_path = 'fake_pr_path'
      pr_result_cache = 'fake_pr_result_cache'
      latest = 'fake_latest'
      result_cache = 'fake_result_cache'
      started = 'fake_started.json'
      finished = 'fake_finished.json'
      def __call__(self, *arg, **kw):
          self.arg = arg
          self.kw = kw
          return self
  
  
  class FakeLogging(object):
      close = Pass
      def __call__(self, *_a, **_kw):
          return self
  
  
  class FakeFinish(object):
      called = False
      result = None
      def __call__(self, unused_a, unused_b, success, *a, **kw):
          self.called = True
          self.result = success
  
  def repo(config):
      repos = bootstrap.Repos()
      for cur_repo, tup in config.items():
          repos[cur_repo] = tup
      return repos
  
  class PRPathsTest(unittest.TestCase):
      def test_kubernetes_kubernetes(self):
          """Test the kubernetes/kubernetes prefix."""
          path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/kubernetes': ('', PULL)}), JOB, BUILD)
          self.assertTrue(any(
              str(PULL) == p for p in path.build_log.split('/')))
  
      def test_kubernetes(self):
          """Test the kubernetes/something prefix."""
          path = bootstrap.pr_paths(UPLOAD, repo({'kubernetes/prefix': ('', PULL)}), JOB, BUILD)
          self.assertTrue(any(
              'prefix' in p for p in path.build_log.split('/')), path.build_log)
          self.assertTrue(any(
              str(PULL) in p for p in path.build_log.split('/')), path.build_log)
  
      def test_other(self):
          """Test the none kubernetes prefixes."""
          path = bootstrap.pr_paths(UPLOAD, repo({'github.com/random/repo': ('', PULL)}), JOB, BUILD)
          self.assertTrue(any(
              'random_repo' in p for p in path.build_log.split('/')), path.build_log)
          self.assertTrue(any(
              str(PULL) in p for p in path.build_log.split('/')), path.build_log)
  
  
  class FakeArgs(object):
      bare = False
      clean = False
      git_cache = ''
      job = JOB
      root = ROOT
      service_account = ROBOT
      ssh = False
      timeout = 0
      upload = UPLOAD
      json = False
  
      def __init__(self, **kw):
          self.branch = BRANCH
          self.pull = PULL
          self.repo = [REPO]
          for key, val in kw.items():
              if not hasattr(self, key):
                  raise AttributeError(self, key)
              setattr(self, key, val)
  
  
  def test_bootstrap(**kw):
      if isinstance(kw.get('repo'), basestring):
          kw['repo'] = [kw['repo']]
      return bootstrap.bootstrap(FakeArgs(**kw))
  
  class BootstrapTest(unittest.TestCase):
  
      def setUp(self):
          self.boiler = [
              Stub(bootstrap, 'checkout', Pass),
              Stub(bootstrap, 'finish', Pass),
              Stub(bootstrap.GSUtil, 'copy_file', Pass),
              Stub(bootstrap, 'node', lambda: 'fake-node'),
              Stub(bootstrap, 'setup_credentials', Pass),
              Stub(bootstrap, 'setup_logging', FakeLogging()),
              Stub(bootstrap, 'start', Pass),
              Stub(bootstrap, '_call', Pass),
              Stub(os, 'environ', fake_environment()),
              Stub(os, 'chdir', Pass),
              Stub(os, 'makedirs', Pass),
          ]
  
      def tearDown(self):
          for stub in self.boiler:
              with stub:  # Leaving with restores things
                  pass
  
      def test_setcreds_setroot_fails(self):
          """We should still call setup_credentials even if setup_root blows up."""
          called = set()
          with Stub(bootstrap, 'setup_root', Bomb):
              with Stub(bootstrap, 'setup_credentials',
                        lambda *a, **kw: called.add('setup_credentials')):
                  with Stub(bootstrap, 'finish', lambda *a, **kw: called.add('finish')):
                      with self.assertRaises(AssertionError):
                          test_bootstrap()
  
          for needed in ['setup_credentials', 'finish']:
              self.assertIn(needed, called)
  
      def test_empty_repo(self):
          repo_name = None
          with Stub(bootstrap, 'checkout', Bomb):
              test_bootstrap(repo=repo_name, branch=None, pull=None, bare=True)
          with self.assertRaises(ValueError):
              test_bootstrap(repo=repo_name, branch=None, pull=PULL)
          with self.assertRaises(ValueError):
              test_bootstrap(repo=repo_name, branch=BRANCH, pull=None)
  
      def test_root_not_exists(self):
          with Stub(os, 'chdir', FakeCall()) as fake_chdir:
              with Stub(os.path, 'exists', lambda p: False):
                  with Stub(os, 'makedirs', FakeCall()) as fake_makedirs:
                      test_bootstrap(branch=None)
          self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls), fake_chdir.calls)
          self.assertTrue(any(ROOT in c[0] for c in fake_makedirs.calls), fake_makedirs.calls)
  
      def test_root_exists(self):
          with Stub(os, 'chdir', FakeCall()) as fake_chdir:
              test_bootstrap(branch=None)
          self.assertTrue(any(ROOT in c[0] for c in fake_chdir.calls))
  
      def test_pr_paths(self):
          """Use a pr_paths when pull is set."""
  
          with Stub(bootstrap, 'ci_paths', Bomb):
              with Stub(bootstrap, 'pr_paths', FakePath()) as path:
                  test_bootstrap(branch=None, pull=PULL)
              self.assertEquals(PULL, path.arg[1][REPO][1], (PULL, path.arg))
  
      def test_ci_paths(self):
          """Use a ci_paths when branch is set."""
  
          with Stub(bootstrap, 'pr_paths', Bomb):
              with Stub(bootstrap, 'ci_paths', FakePath()) as path:
                  test_bootstrap(pull=None, branch=BRANCH)
              self.assertFalse(any(
                  PULL in o for o in (path.arg, path.kw)))
  
      def test_finish_when_start_fails(self):
          """Finish is called even if start fails."""
          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
              with Stub(bootstrap, 'start', Bomb):
                  with self.assertRaises(AssertionError):
                      test_bootstrap(pull=None)
          self.assertTrue(fake.called)
          self.assertTrue(fake.result is False)  # Distinguish from None
  
      def test_finish_when_build_fails(self):
          """Finish is called even if the build fails."""
          def call_error(*_a, **_kw):
              raise subprocess.CalledProcessError(1, [], '')
          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
              with Stub(bootstrap, '_call', call_error):
                  with self.assertRaises(SystemExit):
                      test_bootstrap(pull=None)
          self.assertTrue(fake.called)
          self.assertTrue(fake.result is False)  # Distinguish from None
  
      def test_happy(self):
          with Stub(bootstrap, 'finish', FakeFinish()) as fake:
              test_bootstrap(pull=None)
          self.assertTrue(fake.called)
          self.assertTrue(fake.result)  # Distinguish from None
  
      def test_job_env(self):
          """bootstrap sets JOB_NAME."""
          with Stub(os, 'environ', fake_environment()) as env:
              test_bootstrap(pull=None)
          self.assertIn(bootstrap.JOB_ENV, env)
  
  
  class RepositoryTest(unittest.TestCase):
      def test_kubernetes_kubernetes(self):
          expected = 'https://github.com/kubernetes/kubernetes'
          actual = bootstrap.repository('k8s.io/kubernetes', '')
          self.assertEquals(expected, actual)
  
      def test_kubernetes_testinfra(self):
          expected = 'https://github.com/kubernetes/test-infra'
          actual = bootstrap.repository('k8s.io/test-infra', '')
          self.assertEquals(expected, actual)
  
      def test_whatever(self):
          expected = 'https://foo.com/bar'
          actual = bootstrap.repository('foo.com/bar', '')
          self.assertEquals(expected, actual)
  
      def test_k8s_k8s_ssh(self):
          expected = 'git@github.com:kubernetes/kubernetes'
          actual = bootstrap.repository('k8s.io/kubernetes', 'path')
          self.assertEquals(expected, actual)
  
      def test_k8s_k8s_ssh_with_colon(self):
          expected = 'git@github.com:kubernetes/kubernetes'
          actual = bootstrap.repository('github.com:kubernetes/kubernetes', 'path')
          self.assertEquals(expected, actual)
  
      def test_whatever_ssh(self):
          expected = 'git@foo.com:bar'
          actual = bootstrap.repository('foo.com/bar', 'path')
          self.assertEquals(expected, actual)
  
  
  
  class IntegrationTest(unittest.TestCase):
      REPO = 'hello/world'
      MASTER = 'fake-master-file'
      BRANCH_FILE = 'fake-branch-file'
      PR_FILE = 'fake-pr-file'
      BRANCH = 'another-branch'
      PR = 42
      PR_TAG = bootstrap.pull_ref(PR)[0][0].strip('+')
  
      def fake_repo(self, fake, _ssh=False):
          return os.path.join(self.root_github, fake)
  
      def setUp(self):
          self.boiler = [
              Stub(bootstrap, 'finish', Pass),
              Stub(bootstrap.GSUtil, 'copy_file', Pass),
              Stub(bootstrap, 'repository', self.fake_repo),
              Stub(bootstrap, 'setup_credentials', Pass),
              Stub(bootstrap, 'setup_logging', FakeLogging()),
              Stub(bootstrap, 'start', Pass),
              Stub(os, 'environ', fake_environment(set_job=False)),
          ]
          self.root_github = tempfile.mkdtemp()
          self.root_workspace = tempfile.mkdtemp()
          self.root_git_cache = tempfile.mkdtemp()
          self.ocwd = os.getcwd()
          fakerepo = self.fake_repo(self.REPO)
          subprocess.check_call(['git', 'init', fakerepo])
          os.chdir(fakerepo)
          subprocess.check_call(['git', 'config', 'user.name', 'foo'])
          subprocess.check_call(['git', 'config', 'user.email', 'foo@bar.baz'])
          subprocess.check_call(['touch', self.MASTER])
          subprocess.check_call(['git', 'add', self.MASTER])
          subprocess.check_call(['git', 'commit', '-m', 'Initial commit'])
          subprocess.check_call(['git', 'checkout', 'master'])
  
      def tearDown(self):
          for stub in self.boiler:
              with stub:  # Leaving with restores things
                  pass
          os.chdir(self.ocwd)
          subprocess.check_call(['rm', '-rf', self.root_github])
          subprocess.check_call(['rm', '-rf', self.root_workspace])
          subprocess.check_call(['rm', '-rf', self.root_git_cache])
  
      def test_git_cache(self):
          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
          subprocess.check_call(['git', 'rm', self.MASTER])
          subprocess.check_call(['touch', self.BRANCH_FILE])
          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
          test_bootstrap(
              job='fake-branch',
              repo=self.REPO,
              branch=self.BRANCH,
              pull=None,
              root=self.root_workspace,
              git_cache=self.root_git_cache)
          # Verify that the cache was populated by running a simple git command
          # in the git cache directory.
          subprocess.check_call(
              ['git', '--git-dir=%s/%s' % (self.root_git_cache, self.REPO), 'log'])
  
      def test_pr(self):
          subprocess.check_call(['git', 'checkout', 'master'])
          subprocess.check_call(['git', 'checkout', '-b', 'unknown-pr-branch'])
          subprocess.check_call(['git', 'rm', self.MASTER])
          subprocess.check_call(['touch', self.PR_FILE])
          subprocess.check_call(['git', 'add', self.PR_FILE])
          subprocess.check_call(['git', 'commit', '-m', 'Create branch for PR %d' % self.PR])
          subprocess.check_call(['git', 'tag', self.PR_TAG])
          os.chdir('/tmp')
          test_bootstrap(
              job='fake-pr',
              repo=self.REPO,
              branch=None,
              pull=self.PR,
              root=self.root_workspace)
  
      def test_branch(self):
          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
          subprocess.check_call(['git', 'rm', self.MASTER])
          subprocess.check_call(['touch', self.BRANCH_FILE])
          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
  
          os.chdir('/tmp')
          test_bootstrap(
              job='fake-branch',
              repo=self.REPO,
              branch=self.BRANCH,
              pull=None,
              root=self.root_workspace)
  
      def test_branch_ref(self):
          """Make sure we check out a specific commit."""
          subprocess.check_call(['git', 'checkout', '-b', self.BRANCH])
          subprocess.check_call(['git', 'rm', self.MASTER])
          subprocess.check_call(['touch', self.BRANCH_FILE])
          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
          subprocess.check_call(['git', 'commit', '-m', 'Create %s' % self.BRANCH])
          sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
          subprocess.check_call(['rm', self.BRANCH_FILE])
          subprocess.check_call(['git', 'add', self.BRANCH_FILE])
          subprocess.check_call(['git', 'commit', '-m', 'Delete %s' % self.BRANCH])
  
          os.chdir('/tmp')
          # Supplying the commit exactly works.
          test_bootstrap(
              job='fake-branch',
              repo=self.REPO,
              branch='%s:%s' % (self.BRANCH, sha),
              pull=None,
              root=self.root_workspace)
          # Supplying the commit through repo works.
          test_bootstrap(
              job='fake-branch',
              repo="%s=%s:%s" % (self.REPO, self.BRANCH, sha),
              branch=None,
              pull=None,
              root=self.root_workspace)
          # Using branch head fails.
          with self.assertRaises(SystemExit):
              test_bootstrap(
                  job='fake-branch',
                  repo=self.REPO,
                  branch=self.BRANCH,
                  pull=None,
                  root=self.root_workspace)
          with self.assertRaises(SystemExit):
              test_bootstrap(
                  job='fake-branch',
                  repo="%s=%s" % (self.REPO, self.BRANCH),
                  branch=None,
                  pull=None,
                  root=self.root_workspace)
  
      def test_batch(self):
          def head_sha():
              # We can't hardcode the SHAs for the test, so we need to determine
              # them after each commit.
              return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
          refs = ['master:%s' % head_sha()]
          for pr in (123, 456):
              subprocess.check_call(['git', 'checkout', '-b', 'refs/pull/%d/head' % pr, 'master'])
              subprocess.check_call(['git', 'rm', self.MASTER])
              subprocess.check_call(['touch', self.PR_FILE])
              subprocess.check_call(['git', 'add', self.PR_FILE])
              open('pr_%d.txt' % pr, 'w').write('some text')
              subprocess.check_call(['git', 'add', 'pr_%d.txt' % pr])
              subprocess.check_call(['git', 'commit', '-m', 'add some stuff (#%d)' % pr])
              refs.append('%d:%s' % (pr, head_sha()))
          os.chdir('/tmp')
          pull = ','.join(refs)
          print '--pull', pull
          test_bootstrap(
              job='fake-pr',
              repo=self.REPO,
              branch=None,
              pull=pull,
              root=self.root_workspace)
  
      def test_pr_bad(self):
          random_pr = 111
          with Stub(bootstrap, 'start', Bomb):
              with Stub(time, 'sleep', Pass):
                  with self.assertRaises(subprocess.CalledProcessError):
                      test_bootstrap(
                          job='fake-pr',
                          repo=self.REPO,
                          branch=None,
                          pull=random_pr,
                          root=self.root_workspace)
  
      def test_branch_bad(self):
          random_branch = 'something'
          with Stub(bootstrap, 'start', Bomb):
              with Stub(time, 'sleep', Pass):
                  with self.assertRaises(subprocess.CalledProcessError):
                      test_bootstrap(
                          job='fake-branch',
                          repo=self.REPO,
                          branch=random_branch,
                          pull=None,
                          root=self.root_workspace)
  
      def test_job_missing(self):
          with self.assertRaises(OSError):
              test_bootstrap(
                  job='this-job-no-exists',
                  repo=self.REPO,
                  branch='master',
                  pull=None,
                  root=self.root_workspace)
  
      def test_job_fails(self):
          with self.assertRaises(SystemExit):
              test_bootstrap(
                  job='fake-failure',
                  repo=self.REPO,
                  branch='master',
                  pull=None,
                  root=self.root_workspace)
  
      def test_commit_in_meta(self):
          sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
  
          # Commit SHA should in meta
          call = lambda *a, **kw: bootstrap._call(5, *a, **kw)
          repos = repo({REPO: ('master', ''), 'other-repo': ('other-branch', '')})
          meta = bootstrap.metadata(repos, 'missing-artifacts-dir', call)
          self.assertIn('repo-commit', meta)
          self.assertEquals(sha, meta['repo-commit'])
          self.assertEquals(40, len(meta['repo-commit']))
          self.assertIn(REPO, meta.get('repos'))
          self.assertIn('other-repo', meta.get('repos'))
          self.assertEquals(REPO, meta.get('repo'))
  
  
  class ParseArgsTest(unittest.TestCase):
      def test_json_missing(self):
          args = bootstrap.parse_args(['--bare', '--job=j'])
          self.assertFalse(args.json, args)
  
      def test_json_onlyflag(self):
          args = bootstrap.parse_args(['--json', '--bare', '--job=j'])
          self.assertTrue(args.json, args)
  
      def test_json_nonzero(self):
          args = bootstrap.parse_args(['--json=1', '--bare', '--job=j'])
          self.assertTrue(args.json, args)
  
      def test_json_zero(self):
          args = bootstrap.parse_args(['--json=0', '--bare', '--job=j'])
          self.assertFalse(args.json, args)
  
      def test_barerepo_both(self):
          with self.assertRaises(argparse.ArgumentTypeError):
              bootstrap.parse_args(['--bare', '--repo=hello', '--job=j'])
  
      def test_barerepo_neither(self):
          with self.assertRaises(argparse.ArgumentTypeError):
              bootstrap.parse_args(['--job=j'])
  
      def test_barerepo_bareonly(self):
          args = bootstrap.parse_args(['--bare', '--job=j'])
          self.assertFalse(args.repo, args)
          self.assertTrue(args.bare, args)
  
      def test_barerepo_repoonly(self):
          args = bootstrap.parse_args(['--repo=R', '--job=j'])
          self.assertFalse(args.bare, args)
          self.assertTrue(args.repo, args)
  
  
  class JobTest(unittest.TestCase):
  
      excludes = [
          'BUILD',  # For bazel
          'config.json',  # For --json mode
          'validOwners.json', # Contains a list of current sigs; sigs are allowed to own jobs
          'config_sort.py', # Tool script to sort config.json
          'move_timeout.py', # Tool to migrate timeouts to config.json
          'move_extract.py',
      ]
  
      yaml_suffix = {
          'job-configs/bootstrap-maintenance.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins/bootstrap-maintenance-ci.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins-pull/bootstrap-maintenance-pull.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins-pull/bootstrap-pull.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins-pull/bootstrap-pull-json.yaml' : 'jsonsuffix',
          'job-configs/kubernetes-jenkins-pull/bootstrap-security-pull.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins/bootstrap-ci.yaml' : 'suffix',
          'job-configs/kubernetes-jenkins/bootstrap-ci-commit.yaml' : 'commit-suffix',
          'job-configs/kubernetes-jenkins/bootstrap-ci-repo.yaml' : 'repo-suffix',
          'job-configs/kubernetes-jenkins/bootstrap-ci-soak.yaml' : 'soak-suffix',
          'job-configs/kubernetes-jenkins/bootstrap-ci-dockerpush.yaml' : 'dockerpush-suffix'
      }
  
      prow_config = '../prow/config.yaml'
  
      realjobs = {}
      prowjobs = []
  
      def test_job_script_expands_vars(self):
          fake = {
              'HELLO': 'awesome',
              'WORLD': 'sauce',
          }
          with Stub(os, 'environ', fake):
              actual = bootstrap.job_args(
                  ['$HELLO ${WORLD}', 'happy', '${MISSING}'])
          self.assertEquals(['awesome sauce', 'happy', '${MISSING}'], actual)
  
  
      @property
      def jobs(self):
          """[(job, job_path)] sequence"""
          for path, _, filenames in os.walk(
                  os.path.dirname(bootstrap.job_script(JOB, False)[0])):
              for job in [f for f in filenames if f not in self.excludes]:
                  job_path = os.path.join(path, job)
                  yield job, job_path
  
      def test_bootstrap_maintenance_yaml(self):
          def check(job, name):
              job_name = 'maintenance-%s' % name
              self.assertIn('frequency', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              return job_name
  
          self.check_bootstrap_yaml('job-configs/bootstrap-maintenance.yaml', check)
  
      def test_bootstrap_maintenance_ci(self):
          is_modern = lambda n: 'janitor' in n and 'aws' not in n
          def check(job, name):
              job_name = 'maintenance-ci-%s' % name
              self.assertIn('frequency', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              self.assertIn('timeout', job)
              self.assertIn('json', job)
              modern = is_modern(name)  # TODO(krzyzacy): all modern
              self.assertEquals(modern, job['json'])
              if is_modern(name):
                  self.assertGreater(job['timeout'], 0)
              return job_name
  
          self.check_bootstrap_yaml('job-configs/kubernetes-jenkins/bootstrap-maintenance-ci.yaml',
                                    check, use_json=is_modern)
  
      def test_bootstrap_maintenance_pull(self):
          is_modern = lambda n: 'janitor' in n
          def check(job, name):
              job_name = 'maintenance-pull-%s' % name
              self.assertIn('frequency', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              self.assertIn('timeout', job)
              self.assertIn('json', job)
              modern = is_modern(name)  # TODO(krzyzacy): all modern
              self.assertEquals(modern, job['json'])
              if is_modern(name):
                  self.assertGreater(job['timeout'], 0)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-maintenance-pull.yaml',
              check, use_json=is_modern)
  
      def test_bootstrap_pull_json_yaml(self):
          def check(job, name):
              job_name = 'pull-%s' % name
              self.assertIn('max-total', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              self.assertIn('timeout', job)
              self.assertNotIn('json', job)
              self.assertGreater(job['timeout'], 0)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-pull-json.yaml',
              check, use_json=True)
  
      def test_bootstrap_pull_yaml(self):
          bads = ['kops', 'federation-e2e']
          is_modern = lambda n: all(b not in n for b in bads)
          def check(job, name):
              job_name = 'pull-%s' % name
              self.assertIn('max-total', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              self.assertIn('timeout', job)
              self.assertIn('json', job)
              modern = is_modern(name)  # TODO(fejta): all modern
              self.assertEquals(modern, job['json'])
              if is_modern(name):
                  self.assertGreater(job['timeout'], 0)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-pull.yaml',
              check, use_json=is_modern)
  
      def test_bootstrap_security_pull(self):
          bads = ['kops', 'federation-e2e']
          is_modern = lambda n: all(b not in n for b in bads)
          def check(job, name):
              job_name = 'pull-%s' % name
              self.assertIn('max-total', job)
              self.assertIn('repo-name', job)
              self.assertIn('.', job['repo-name'])  # Has domain
              self.assertIn('timeout', job)
              self.assertIn('json', job)
              modern = is_modern(name)
              self.assertEquals(modern, job['json'])
              if is_modern(name):
                  self.assertGreater(job['timeout'], 0)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-security-pull.yaml',
              check, use_json=is_modern)
  
      def test_bootstrap_security_match(self):
          jobs1 = self.load_bootstrap_yaml('job-configs/kubernetes-jenkins-pull/bootstrap-pull.yaml')
          json_jobs = self.load_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-pull-json.yaml')
  
          jobs2 = self.load_bootstrap_yaml(
              'job-configs/kubernetes-jenkins-pull/bootstrap-security-pull.yaml')
          for name, job in jobs1.iteritems():
              if job['repo-name'] == 'k8s.io/kubernetes':
                  job2 = jobs2[name]
                  for attr in job:
                      if attr != 'repo-name':
                          self.assertEquals(job[attr], job2[attr], job)
          for name, job in jobs2.iteritems():
              if job.get('json') and any(n in name for n in ['kubernetes-e2e', 'kubemark-e2e']):
                  jobs = json_jobs
                  skip_json = True
              else:
                  jobs = jobs1
                  skip_json = False
              self.assertIn(name, jobs)
              job2 = jobs[name]
              for attr in job:
                  if attr == 'repo-name':
                      continue
                  if attr == 'json' and skip_json:
                      continue
                  self.assertEquals(job[attr], job2[attr])
  
  
      def test_bootstrap_ci_yaml(self):
          def check(job, name):
              job_name = 'ci-%s' % name
              self.assertIn('frequency', job)
              self.assertIn('trigger-job', job)
              self.assertNotIn('branch', job)
              self.assertNotIn('json', job)
              self.assertGreater(job['timeout'], 0, job_name)
              self.assertGreaterEqual(job['jenkins-timeout'], job['timeout']+100, job_name)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins/bootstrap-ci.yaml',
              check, use_json=True)
  
      def test_bootstrap_ci_commit_yaml(self):
          def check(job, name):
              job_name = 'ci-%s' % name
              self.assertIn('branch', job)
              self.assertIn('commit-frequency', job)
              self.assertIn('giturl', job)
              self.assertIn('repo-name', job)
              self.assertIn('timeout', job)
              self.assertGreater(job['timeout'], 0, job)
  
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins/bootstrap-ci-commit.yaml',
              check, use_json=True)
  
      def test_bootstrap_ci_repo_yaml(self):
          def check(job, name):
              job_name = 'ci-%s' % name
              self.assertIn('branch', job)
              self.assertIn('frequency', job)
              self.assertIn('repo-name', job)
              self.assertIn('timeout', job)
              self.assertNotIn('json', job)
              self.assertGreater(job['timeout'], 0, name)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins/bootstrap-ci-repo.yaml',
              check, use_json=True)
  
      def test_bootstrap_ci_soak_yaml(self):
          def check(job, name):
              job_name = 'ci-%s' % name
              self.assertIn('blocker', job)
              self.assertIn('frequency', job)
              self.assertIn('scan', job)
              self.assertNotIn('repo-name', job)
              self.assertNotIn('branch', job)
              self.assertIn('timeout', job)
              self.assertIn('soak-repos', job)
              self.assertGreater(job['timeout'], 0, name)
  
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins/bootstrap-ci-soak.yaml',
              check, use_json=True)
  
      def test_bootstrap_ci_dockerpush(self):
          def check(job, name):
              job_name = 'ci-%s' % name
              self.assertIn('branch', job)
              self.assertIn('frequency', job)
              self.assertIn('repo-name', job)
              self.assertIn('timeout', job)
              self.assertGreater(job['timeout'], 0, name)
              return job_name
  
          self.check_bootstrap_yaml(
              'job-configs/kubernetes-jenkins/bootstrap-ci-dockerpush.yaml',
              check, use_json=True)
  
      def check_job_template(self, tmpl):
          builders = tmpl.get('builders')
          if not isinstance(builders, list):
              self.fail(tmpl)
          self.assertEquals(1, len(builders), builders)
          shell = builders[0]
          if not isinstance(shell, dict):
              self.fail(tmpl)
          self.assertEquals(1, len(shell), tmpl)
          if 'raw' in shell:
              self.assertEquals('maintenance-all-{suffix}', tmpl['name'])
              return
          cmd = shell.get('shell')
          if not isinstance(cmd, basestring):
              self.fail(tmpl)
          self.assertIn('--service-account=', cmd)
          self.assertIn('--upload=', cmd)
          if 'kubernetes-security' in cmd:
              self.assertIn('--upload=\'gs://kubernetes-security-jenkins/pr-logs\'', cmd)
          elif '${{PULL_REFS}}' in cmd:
              self.assertIn('--upload=\'gs://kubernetes-jenkins/pr-logs\'', cmd)
          else:
              self.assertIn('--upload=\'gs://kubernetes-jenkins/logs\'', cmd)
  
      def add_prow_job(self, job):
          name = job.get('name')
          real_job = {}
          real_job['name'] = name
          if 'spec' in job:
              spec = job.get('spec')
              for container in spec.get('containers'):
                  if 'args' in container:
                      for arg in container.get('args'):
                          match = re.match(r'--timeout=(\d+)', arg)
                          if match:
                              real_job['timeout'] = match.group(1)
          if name not in self.realjobs:
              self.realjobs[name] = real_job
              self.prowjobs.append(name)
          if 'run_after_success' in job:
              for sub in job.get('run_after_success'):
                  self.add_prow_job(sub)
  
      def load_prow_yaml(self, path):
          with open(os.path.join(
              os.path.dirname(__file__), path)) as fp:
              doc = yaml.safe_load(fp)
  
          if 'periodics' not in doc:
              self.fail('No periodics in prow config!')
  
          if 'presubmits' not in doc:
              self.fail('No presubmits in prow config!')
  
          for item in doc.get('periodics'):
              self.add_prow_job(item)
  
          if 'postsubmits' not in doc:
              self.fail('No postsubmits in prow config!')
  
          presubmits = doc.get('presubmits')
          postsubmits = doc.get('postsubmits')
  
          for _repo, joblist in presubmits.items() + postsubmits.items():
              for job in joblist:
                  self.add_prow_job(job)
  
      def load_bootstrap_yaml(self, path):
          with open(os.path.join(
              os.path.dirname(__file__), path)) as fp:
              doc = yaml.safe_load(fp)
  
          project = None
          defined_templates = set()
          for item in doc:
              if not isinstance(item, dict):
                  continue
              if isinstance(item.get('job-template'), dict):
                  defined_templates.add(item['job-template']['name'])
                  self.check_job_template(item['job-template'])
              if not isinstance(item.get('project'), dict):
                  continue
              project = item['project']
              self.assertIn('bootstrap-', project.get('name'))
              break
          else:
              self.fail('Could not find bootstrap-pull-jobs project')
  
          self.assertIn('jobs', project)
          used_templates = {j for j in project['jobs']}
          msg = '\nMissing templates: %s\nUnused templates: %s' % (
              ','.join(used_templates - defined_templates),
              ','.join(defined_templates - used_templates))
          self.assertEquals(defined_templates, used_templates, msg)
  
          self.assertIn(path, self.yaml_suffix)
          jobs = project.get(self.yaml_suffix[path])
          if not jobs or not isinstance(jobs, list):
              self.fail('Could not find suffix list in %s' % (project))
  
          real_jobs = {}
          for job in jobs:
              # Things to check on all bootstrap jobs
              if not isinstance(job, dict):
                  self.fail('suffix items should be dicts: %s' % jobs)
              self.assertEquals(1, len(job), job)
              name = job.keys()[0]
              real_job = job[name]
              self.assertNotIn(name, real_jobs, 'duplicate job: %s' % name)
              real_jobs[name] = real_job
              real_name = real_job.get('job-name', 'unset-%s' % name)
              if real_name not in self.realjobs:
                  self.realjobs[real_name] = real_job
          return real_jobs
  
      def check_bootstrap_yaml(self, path, check, use_json=False):
          for name, real_job in self.load_bootstrap_yaml(path).iteritems():
              # Things to check on all bootstrap jobs
              if callable(use_json):  # TODO(fejta): gross, but temporary?
                  modern = use_json(name)
              else:
                  modern = use_json
              cmd = bootstrap.job_script(real_job.get('job-name'), modern)
              path = cmd[0]
              args = cmd[1:]
              self.assertTrue(os.path.isfile(path), (name, path))
              if modern:
                  self.assertTrue(all(isinstance(a, basestring) for a in args), args)
                  # Ensure the .sh script isn't there
                  other = bootstrap.job_script(real_job.get('job-name'), False)
                  self.assertFalse(os.path.isfile(other[0]), name)
              else:
                  self.assertEquals(1, len(cmd))
                  # Ensure the job isn't in the json
                  with self.assertRaises(KeyError):
                      bootstrap.job_script(real_job.get('job-name'), True)
                      self.fail(name)
              for key, value in real_job.items():
                  if not isinstance(value, (basestring, int)):
                      self.fail('Jobs may not contain child objects %s: %s' % (
                          key, value))
                  if '{' in str(value):
                      self.fail('Jobs may not contain {expansions} - %s: %s' % (
                          key, value))  # Use simple strings
              # Things to check on specific flavors.
              job_name = check(real_job, name)
              self.assertTrue(job_name)
              self.assertEquals(job_name, real_job.get('job-name'))
  
      def get_real_bootstrap_job(self, job):
          key = os.path.splitext(job.strip())[0]
          if not key in self.realjobs:
              for yamlf in self.yaml_suffix:
                  self.load_bootstrap_yaml(yamlf)
              self.load_prow_yaml(self.prow_config)
          self.assertIn(key, sorted(self.realjobs))  # sorted for clearer error message
          return self.realjobs.get(key)
  
      def test_valid_env(self):
          for job, job_path in self.jobs:
              with open(job_path) as fp:
                  data = fp.read()
              if 'kops' in job:  # TODO(fejta): update this one too
                  continue
              self.assertNotIn(
                  'JENKINS_USE_LOCAL_BINARIES=',
                  data,
                  'Send --extract=local to config.json, not JENKINS_USE_LOCAL_BINARIES in %s' % job)
              self.assertNotIn(
                  'JENKINS_USE_EXISTING_BINARIES=',
                  data,
                  'Send --extract=local to config.json, not JENKINS_USE_EXISTING_BINARIES in %s' % job)  # pylint: disable=line-too-long
  
      def test_valid_timeout(self):
          """All jobs set a timeout less than 120m or set DOCKER_TIMEOUT."""
          default_timeout = int(re.search(
              r'\$\{DOCKER_TIMEOUT:-(\d+)m',
              open('%s/dockerized-e2e-runner.sh' % os.path.dirname(__file__)).read()
          ).group(1))
          bad_jobs = set()
          with open(bootstrap.test_infra('jobs/config.json')) as fp:
              config = json.loads(fp.read())
  
          for job, job_path in self.jobs:
              job_name = job.rsplit('.', 1)[0]
              modern = config.get(job_name, {}).get('scenario') in [
                  'kubernetes_e2e',
                  'kubernetes_kops_aws',
              ]
              valids = [
                  'kubernetes-e2e-',
                  'kubernetes-kubemark-',
                  'kubernetes-soak-',
                  'kubernetes-federation-e2e-',
                  'kops-e2e-',
              ]
  
              if not re.search('|'.join(valids), job):
                  continue
              with open(job_path) as fp:
                  lines = list(l for l in fp if not l.startswith('#'))
              container_timeout = default_timeout
              kubetest_timeout = None
              for line in lines:  # Validate old pattern no longer used
                  if line.startswith('### Reporting'):
                      bad_jobs.add(job)
                  if '{rc}' in line:
                      bad_jobs.add(job)
              if job.endswith('.sh'):  # --json=False type jobs, TODO(fejta): deprecate
                  for line in lines:
                      if line.startswith('export DOCKER_TIMEOUT='):
                          container_timeout = int(re.match(
                              r'export DOCKER_TIMEOUT="(\d+)m".*', line).group(1))
                      if 'KUBEKINS_TIMEOUT=' not in line:
                          continue
                      mat = re.match(r'export KUBEKINS_TIMEOUT="(\d+)m".*', line)
                      self.assertTrue(mat, 'Bad KUBEKINS_TIMEOUT in %s' % job)
                      kubetest_timeout = int(mat.group(1))
              elif not modern:
                  realjob = self.get_real_bootstrap_job(job)
                  self.assertTrue(realjob)
                  self.assertIn('timeout', realjob, job)
                  container_timeout = int(realjob['timeout'])
                  for line in lines:
                      if 'DOCKER_TIMEOUT=' in line:
                          self.fail('Set docker timeout in bootstrap yaml: %s' % job)
                      if 'KUBEKINS_TIMEOUT=' not in line:
                          continue
                      mat = re.match(r'KUBEKINS_TIMEOUT=(\d+)m.*', line)
                      self.assertTrue(mat, line)
                      kubetest_timeout = int(mat.group(1))
              else:
                  realjob = self.get_real_bootstrap_job(job)
                  self.assertTrue(realjob)
                  self.assertIn('timeout', realjob, job)
                  container_timeout = int(realjob['timeout'])
                  for line in lines:
                      if 'DOCKER_TIMEOUT=' in line:
                          self.fail('Set container timeout in prow and/or bootstrap yaml: %s' % job)
                      if 'KUBEKINS_TIMEOUT=' in line:
                          self.fail(
                              'Set kubetest --timeout in config.json, not KUBEKINS_TIMEOUT: %s'
                              % job
                          )
                  for arg in config[job_name]['args']:
                      if arg == '--timeout=None':
                          bad_jobs.add(('Must specify a timeout', job, arg))
                      mat = re.match(r'--timeout=(\d+)m', arg)
                      if not mat:
                          continue
                      kubetest_timeout = int(mat.group(1))
              if kubetest_timeout is None:
                  self.fail('Missing timeout: %s' % job)
              if kubetest_timeout > container_timeout:
                  bad_jobs.add((job, kubetest_timeout, container_timeout))
              elif kubetest_timeout + 20 > container_timeout:
                  bad_jobs.add((
                      'insufficient kubetest leeway',
                      job, kubetest_timeout, container_timeout
                      ))
  
  
          if bad_jobs:
              self.fail('\n'.join(str(s) for s in bad_jobs))
  
      def test_only_jobs(self):
          """Ensure that everything in jobs/ is a valid job name and script."""
          for job, job_path in self.jobs:
              # Jobs should have simple names: letters, numbers, -, .
              self.assertTrue(re.match(r'[.0-9a-z-_]+.(sh|env)', job), job)
              # Jobs should point to a real, executable file
              # Note: it is easy to forget to chmod +x
              self.assertTrue(os.path.isfile(job_path), job_path)
              self.assertFalse(os.path.islink(job_path), job_path)
              if job.endswith('.sh'):
                  self.assertTrue(os.access(job_path, os.X_OK|os.R_OK), job_path)
              else:
                  self.assertTrue(os.access(job_path, os.R_OK), job_path)
  
      def test_all_project_are_unique(self):
          # pylint: disable=line-too-long
          allowed_list = {
              # The 1.5 and 1.6 scalability jobs intentionally share projects.
              'ci-kubernetes-e2e-gce-scalability-release-1.5.env': 'ci-kubernetes-e2e-gce-scalability-release-*',
              'ci-kubernetes-e2e-gce-scalability-release-1.6.env': 'ci-kubernetes-e2e-gce-scalability-release-*',
              'ci-kubernetes-e2e-gci-gce-scalability-release-1.5.env': 'ci-kubernetes-e2e-gci-gce-scalability-release-*',
              'ci-kubernetes-e2e-gci-gce-scalability-release-1.6.env': 'ci-kubernetes-e2e-gci-gce-scalability-release-*',
              # TODO(fejta): remove these (found while migrating jobs)
              'ci-kubernetes-kubemark-100-gce.env': 'ci-kubernetes-kubemark-*',
              'ci-kubernetes-kubemark-5-gce.env': 'ci-kubernetes-kubemark-*',
              'ci-kubernetes-kubemark-high-density-100-gce.env': 'ci-kubernetes-kubemark-*',
              'ci-kubernetes-kubemark-gce-scale.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gce-enormous-cluster.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gce-enormous-deploy.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gce-enormous-teardown.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gke-large-cluster.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gke-large-deploy.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-e2e-gke-large-teardown.env': 'ci-kubernetes-scale-*',
              'ci-kubernetes-federation-build.sh': 'ci-kubernetes-federation-*',
              'ci-kubernetes-e2e-gce-federation.env': 'ci-kubernetes-federation-*',
              'pull-kubernetes-federation-e2e-gce.env': 'pull-kubernetes-federation-e2e-gce-*',
              'ci-kubernetes-pull-gce-federation-deploy.env': 'pull-kubernetes-federation-e2e-gce-*',
              'pull-kubernetes-federation-e2e-gce-canary.env': 'pull-kubernetes-federation-e2e-gce-*',
              'ci-kubernetes-pull-gce-federation-deploy-canary.env': 'pull-kubernetes-federation-e2e-gce-*',
              'pull-kubernetes-e2e-gce.env': 'pull-kubernetes-e2e-gce-*',
              'pull-kubernetes-e2e-gce-bazel.env': 'pull-kubernetes-e2e-gce-*',
              'pull-kubernetes-e2e-gce-canary.env': 'pull-kubernetes-e2e-gce-*',
          }
          # pylint: enable=line-too-long
          projects = collections.defaultdict(set)
          for job, job_path in self.jobs:
              with open(job_path) as fp:
                  lines = list(fp)
              for line in lines:
                  line = line.strip()
                  if not line.startswith('PROJECT='):
                      continue
                  if '-soak-' in job:  # Soak jobs have deploy/test pairs
                      job = job.replace('-test', '-*').replace('-deploy', '-*')
                  if job.startswith('ci-kubernetes-node-'):
                      job = 'ci-kubernetes-node-*'
                  if not line.startswith('#') and job.endswith('.sh'):
                      self.assertIn('export', line, line)
                  if job.endswith('.sh'):
                      project = re.search(r'PROJECT="([^"]+)"', line).group(1)
                  else:
                      project = re.search(r'PROJECT=([^"]+)', line).group(1)
                  projects[project].add(allowed_list.get(job, job))
          duplicates = [(p, j) for p, j in projects.items() if len(j) > 1]
          if duplicates:
              self.fail('Jobs duplicate projects:\n  %s' % (
                  '\n  '.join('%s: %s' % t for t in duplicates)))
  
      def test_jobs_do_not_source_shell(self):
          for job, job_path in self.jobs:
              if job.startswith('pull-'):
                  continue  # No clean way to determine version
              with open(job_path) as fp:
                  script = fp.read()
              self.assertFalse(re.search(r'\Wsource ', script), job)
              self.assertNotIn('\n. ', script, job)
  
      def test_all_bash_jobs_have_errexit(self):
          options = {
              'errexit',
              'nounset',
              'pipefail',
          }
          for job, job_path in self.jobs:
              if not job.endswith('.sh'):
                  continue
              with open(job_path) as fp:
                  lines = list(fp)
              for option in options:
                  expected = 'set -o %s\n' % option
                  self.assertIn(
                      expected, lines,
                      '%s not found in %s' % (expected, job_path))
  
      def test_envs_no_export(self):
          for job, job_path in self.jobs:
              if not job.endswith('.env'):
                  continue
              with open(job_path) as fp:
                  lines = list(fp)
              for line in lines:
                  line = line.strip()
                  self.assertFalse(line.endswith('\\'))
                  if not line:
                      continue
                  if line.startswith('#'):
                      continue
                  if not re.match(r'[0-9A-Z_]+=[^\n]*', line):
                      self.fail('[%r]: Env %r: need to follow FOO=BAR pattern' % (job, line))
                  if '#' in line:
                      self.fail('[%r]: Env %r: No inline comments' % (job, line))
                  if '"' in line or '\'' in line:
                      self.fail('[%r]: Env %r: No quote in env' % (job, line))
                  if '$' in line:
                      self.fail('[%r]: Env %r: Please resolve variables in env file' % (job, line))
  
                  # also test for https://github.com/kubernetes/test-infra/issues/2829
                  black = [
                      ('CHARTS_TEST=', '--charts-tests'),
                      ('E2E_DOWN=', '--down=true|false'),
                      ('E2E_NAME=', '--cluster=whatever'),
                      ('E2E_PUBLISH_PATH=', '--publish=gs://FOO'),
                      ('E2E_TEST=', '--test=true|false'),
                      ('E2E_UP=', '--up=true|false'),
                      ('FEDERATION_DOWN=', '--down=true|false'),
                      ('FEDERATION_UP=', '--up=true|false'),
                      ('JENKINS_FEDERATION_PREFIX=', '--stage=gs://FOO'),
                      ('JENKINS_PUBLISHED_VERSION=', '--extract=V'),
                      ('JENKINS_PUBLISHED_SKEW_VERSION=', '--extract=V'),
                      ('JENKINS_USE_SKEW_KUBECTL=', 'SKEW_KUBECTL=y'),
                      ('JENKINS_USE_SKEW_TESTS=', '--skew'),
                      ('JENKINS_SOAK_MODE', '--soak'),
                      ('JENKINS_SOAK_PREFIX', '--stage=gs://FOO'),
                      ('JENKINS_USE_EXISTING_BINARIES=', '--extract=local'),
                      ('JENKINS_USE_LOCAL_BINARIES=', '--extract=none'),
                      ('JENKINS_USE_SERVER_VERSION=', '--extract=gke'),
                      ('JENKINS_USE_GCI_VERSION=', '--extract=gci/FAMILY'),
                      ('JENKINS_USE_GCI_HEAD_IMAGE_FAMILY=', '--extract=gci/FAMILY'),
                      ('KUBEKINS_TIMEOUT=', '--timeout=XXm'),
                      ('PERF_TESTS=', '--perf'),
                      ('USE_KUBEMARK=', '--kubemark'),
                  ]
                  for env, fix in black:
                      if 'JENKINS_PUBLISHED_VERSION' in env and 'kops' in job:
                          continue  # TOOD(fejta): migrate kops jobs
                      if env in line:
                          self.fail('[%s]: Env %s: Convert %s to use %s in jobs/config.json' % (
                              job, line, env, fix))
  
  
      def test_no_bad_vars_in_jobs(self):
          """Searches for jobs that contain ${{VAR}}"""
          for job, job_path in self.jobs:
              with open(job_path) as fp:
                  script = fp.read()
              bad_vars = re.findall(r'(\${{.+}})', script)
              if bad_vars:
                  self.fail('Job %s contains bad bash variables: %s' % (job, ' '.join(bad_vars)))
  
      def test_valid_job_envs(self):
          """Validate jobs/config.json."""
          self.load_prow_yaml(self.prow_config)
          config = bootstrap.test_infra('jobs/config.json')
          owners = bootstrap.test_infra('jobs/validOwners.json')
          with open(config) as fp, open(owners) as ownfp:
              config = json.loads(fp.read())
              valid_owners = json.loads(ownfp.read())
              for job in config:
                  # onwership assertions
                  self.assertIn('sigOwners', config[job], job)
                  self.assertIsInstance(config[job]['sigOwners'], list, job)
                  self.assertTrue(config[job]['sigOwners'], job) # non-empty
                  owners = config[job]['sigOwners']
                  for owner in owners:
                      self.assertIsInstance(owner, basestring, job)
                      self.assertIn(owner, valid_owners, job)
  
                  # env assertions
                  self.assertTrue('scenario' in config[job], job)
                  scenario = bootstrap.test_infra('scenarios/%s.py' % config[job]['scenario'])
                  self.assertTrue(os.path.isfile(scenario), job)
                  self.assertTrue(os.access(scenario, os.X_OK|os.R_OK), job)
                  has_matching_env = False
                  right_mode = True
                  if job in self.prowjobs:
                      right_mode = False
                  for arg in config[job].get('args', []):
                      if arg == '--mode=local':
                          right_mode = True
                      match = re.match(r'--env-file=([^\"]+)\.env', arg)
                      if match:
                          env = match.group(1)
                          if env == 'jobs/%s' % job:
                              has_matching_env = True
                          path = bootstrap.test_infra('%s.env' % env)
                          self.assertTrue(
                              os.path.isfile(path),
                              '%s does not exist for %s' % (path, job))
                      elif 'kops' not in job:
                          match = re.match(r'--cluster=([^\"]+)', arg)
                          if match:
                              cluster = match.group(1)
                              self.assertLessEqual(
                                  len(cluster), 20,
                                  'Job %r, --cluster should be 20 chars or fewer' % job
                                  )
                  if config[job]['scenario'] == 'kubernetes_e2e':
                      args = config[job]['args']
                      if not any('--extract=' in a for a in args):
                          self.fail('e2e job needs --extract flag: %s %s' % (job, args))
                      self.assertTrue(has_matching_env, job)
                      self.assertTrue(right_mode, job)
                      if job.startswith('pull-kubernetes-'):
                          self.assertIn('--cluster=', args)
                          if 'gke' in job:
                              stage = 'gs://kubernetes-release-dev/ci'
                              suffix = True
                          elif 'kubeadm' in job:
                              # kubeadm-based jobs use out-of-band .deb artifacts,
                              # not the --stage flag.
                              continue
                          else:
                              stage = 'gs://kubernetes-release-pull/ci/%s' % job
                              suffix = False
                          self.assertIn('--stage=%s' % stage, args)
                          self.assertEquals(
                              suffix,
                              any('--stage-suffix=' in a for a in args),
                              ('--stage-suffix=', suffix, job, args))
  
      def test_config_is_sorted(self):
          """Test jobs/config.json is sorted."""
          with open(bootstrap.test_infra('jobs/config.json')) as fp:
              original = fp.read()
              expect = json.dumps(json.loads(original), sort_keys=True, indent=2) + '\n'
              if original != expect:
                  self.fail('config.json is not sorted, please run jobs/config_sort.py')
  
  
  if __name__ == '__main__':
      unittest.main()
  