sregistry/main/google_storage/build.py
def setup_build(self, name, repo, config, tag=None, commit=None,
                      recipe="Singularity", startup_script=None):

    '''setup the build based on the selected configuration file, meaning
       producing the configuration file filled in based on the user's input
 
       Parameters
       ==========
       config: the complete configuration file provided by the client
       template: an optional custom start script to use
       tag: a user specified tag for the build, derived from uri or manual
       recipe: a recipe, if defined, overrides recipe set in config.
       commit: a commit to check out, if needed
       start_script: the start script to use, if not defined 
                     defaults to apt (or manager) base in main/templates/build

    '''

    manager = self._get_and_update_setting('SREGISTRY_BUILDER_MANAGER', 'apt')
    startup_script = get_build_template(startup_script, manager)

    # Read in the config to know what we can edit
    config = self._load_build_config(config)
    if not config:
        bot.error('Cannot find config, check path or URI.')
        sys.exit(1)

    # Ensure that the builder config is intended for the client
    self._client_tagged(config['data']['tags'])

    # Compute settings that are parsed into runscript via metadata
    defaults = config['data']['metadata']
    selfLink = config['links']['self']

    # Make sure the builder repository and folder is passed forward
    builder_repo = config['data']['repo']
    builder_bundle = config['data']['path']
    builder_id = config['data']['id']
    config = config['data']['config']

    # Config settings from the environment, fall back to defaults
    image_project = defaults.get('GOOGLE_COMPUTE_PROJECT', 'debian-cloud')
    image_family = defaults.get('GOOGLE_COMPUTE_IMAGE_FAMILY', 'debian-8')

    # Generate names, description is for repo, name is random
    instance_name = "%s-builder %s" %(name.replace('/','-'), selfLink)
    robot_name = RobotNamer().generate()

    project = self._get_project()
    zone = self._get_zone()

    # Machine Type
    machine_type = defaults.get('SREGISTRY_BUILDER_machine_type', 'n1-standard-1')
    machine_type = "zones/%s/machineTypes/%s" %(zone, machine_type)

    # Disk Size
    disk_size = defaults.get('SREGISTRY_BUILDER_disk_size', '100')

    # Get the image type
    image_response = self._compute_service.images().getFromFamily(
                              project=image_project, 
                              family=image_family).execute()
    source_disk_image = image_response['selfLink']
    storage_bucket = self._bucket_name

    # Add the machine parameters to the config
    config['name'] = robot_name
    config['description'] = instance_name
    config['machineType'] = machine_type
    config['disks'].append({
                    "autoDelete": True,
                    "boot": True,
                    "initializeParams": { 'sourceImage': source_disk_image,
                                          'diskSizeGb': disk_size }
                   })

    # Metadata base
    metadata = {'items': 

                    [{ 'key': 'startup-script',
                       'value': startup_script },

                    # Storage Settings from Host

                     { 'key':'SREGISTRY_BUILDER_STORAGE_BUCKET',
                       'value':self._bucket_name }]}


    # Runtime variables take priority over defaults from config
    # and so here we update the defaults with runtime
    # ([defaults], [config-key], [runtime-setting])

    # User Repository
    defaults = setconfig(defaults, 'SREGISTRY_USER_REPO', repo)
    
    # Container Namespace (without tag/version)
    defaults = setconfig(defaults, 'SREGISTRY_CONTAINER_NAME', name)

    # User Repository Commit
    defaults = setconfig(defaults, 'SREGISTRY_USER_COMMIT', commit)

    # User Repository Branch
    defaults = setconfig(defaults, 'SREGISTRY_USER_BRANCH', "master")

    # User Repository Tag
    defaults = setconfig(defaults, 'SREGISTRY_USER_TAG', tag)

    # Builder repository url
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_REPO', builder_repo)

    # Builder commit
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_COMMIT')

    # Builder default runscript
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_RUNSCRIPT', "run.sh")

    # Builder repository url
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BRANCH', "master")

    # Builder id in repository
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_ID', builder_id)

    # Builder repository relative folder path
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BUNDLE', builder_bundle)

    # Number of extra hours to debug
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_DEBUGHOURS', "4")

    # Hours to kill running job
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_KILLHOURS', "10")

    # Recipe set at runtime
    defaults = setconfig(defaults, 'SINGULARITY_RECIPE', recipe)

    # Branch of Singularity to install
    defaults = setconfig(defaults, 'SINGULARITY_BRANCH')

    # Singularity commit to use (if needed)
    defaults = setconfig(defaults, 'SINGULARITY_COMMIT')

    # Singularity Repo to Use
    defaults = setconfig(defaults,'SINGULARITY_REPO', 
                                  'https://github.com/cclerget/singularity.git')

    # Update metadata config object

    seen = ['SREGISTRY_BUILDER_STORAGE_BUCKET', 'startup-script']

    for key, value in defaults.items():

        # This also appends empty values, they are meaningful
        if value not in seen:
            entry = { "key": key, 'value': value }
            metadata['items'].append(entry)
            seen.append(key)

    config['metadata'] = metadata
    return config
def setup_build(self, name, repo, config, tag=None, commit=None,
                      recipe="Singularity", startup_script=None):

    '''setup the build based on the selected configuration file, meaning
       producing the configuration file filled in based on the user's input
 
       Parameters
       ==========
       config: the complete configuration file provided by the client
       template: an optional custom start script to use
       tag: a user specified tag for the build, derived from uri or manual
       recipe: a recipe, if defined, overrides recipe set in config.
       commit: a commit to check out, if needed
       start_script: the start script to use, if not defined 
                     defaults to apt (or manager) base in main/templates/build

    '''

    manager = self._get_and_update_setting('SREGISTRY_BUILDER_MANAGER', 'apt')
    startup_script = get_build_template(startup_script, manager)

    # Read in the config to know what we can edit
    config = self._load_build_config(config)
    if not config:
        bot.error('Cannot find config, check path or URI.')
        sys.exit(1)

    # Ensure that the builder config is intended for the client
    self._client_tagged(config['data']['tags'])

    # Compute settings that are parsed into runscript via metadata
    defaults = config['data']['metadata']
    selfLink = config['links']['self']

    # Make sure the builder repository and folder is passed forward
    builder_repo = config['data']['repo']
    builder_bundle = config['data']['path']
    builder_id = config['data']['id']
    config = config['data']['config']

    # Config settings from the environment, fall back to defaults
    image_project = defaults.get('GOOGLE_COMPUTE_PROJECT', 'debian-cloud')
    image_family = defaults.get('GOOGLE_COMPUTE_IMAGE_FAMILY', 'debian-8')

    # Generate names, description is for repo, name is random
    instance_name = "%s-builder %s" %(name.replace('/','-'), selfLink)
    robot_name = RobotNamer().generate()

    project = self._get_project()
    zone = self._get_zone()

    # Machine Type
    machine_type = defaults.get('SREGISTRY_BUILDER_machine_type', 'n1-standard-1')
    machine_type = "zones/%s/machineTypes/%s" %(zone, machine_type)

    # Disk Size
    disk_size = defaults.get('SREGISTRY_BUILDER_disk_size', '100')

    # Get the image type
    image_response = self._compute_service.images().getFromFamily(
                              project=image_project, 
                              family=image_family).execute()
    source_disk_image = image_response['selfLink']
    storage_bucket = self._bucket_name

    # Add the machine parameters to the config
    config['name'] = robot_name
    config['description'] = instance_name
    config['machineType'] = machine_type
    config['disks'].append({
                    "autoDelete": True,
                    "boot": True,
                    "initializeParams": { 'sourceImage': source_disk_image,
                                          'diskSizeGb': disk_size }
                   })

    # Metadata base
    metadata = {'items': 

                    [{ 'key': 'startup-script',
                       'value': startup_script },

                    # Storage Settings from Host

                     { 'key':'SREGISTRY_BUILDER_STORAGE_BUCKET',
                       'value':self._bucket_name }]}


    # Runtime variables take priority over defaults from config
    # and so here we update the defaults with runtime
    # ([defaults], [config-key], [runtime-setting])

    # User Repository
    defaults = setconfig(defaults, 'SREGISTRY_USER_REPO', repo)
    
    # Container Namespace (without tag/version)
    defaults = setconfig(defaults, 'SREGISTRY_CONTAINER_NAME', name)

    # User Repository Commit
    defaults = setconfig(defaults, 'SREGISTRY_USER_COMMIT', commit)

    # User Repository Branch
    defaults = setconfig(defaults, 'SREGISTRY_USER_BRANCH', "master")

    # User Repository Tag
    defaults = setconfig(defaults, 'SREGISTRY_USER_TAG', tag)

    # Builder repository url
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_REPO', builder_repo)

    # Builder commit
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_COMMIT')

    # Builder default runscript
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_RUNSCRIPT', "run.sh")

    # Builder repository url
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BRANCH', "master")

    # Builder id in repository
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_ID', builder_id)

    # Builder repository relative folder path
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BUNDLE', builder_bundle)

    # Number of extra hours to debug
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_DEBUGHOURS', "4")

    # Hours to kill running job
    defaults = setconfig(defaults, 'SREGISTRY_BUILDER_KILLHOURS', "10")

    # Recipe set at runtime
    defaults = setconfig(defaults, 'SINGULARITY_RECIPE', recipe)

    # Branch of Singularity to install
    defaults = setconfig(defaults, 'SINGULARITY_BRANCH')

    # Singularity commit to use (if needed)
    defaults = setconfig(defaults, 'SINGULARITY_COMMIT')

    # Singularity Repo to Use
    defaults = setconfig(defaults,'SINGULARITY_REPO', 
                                  'https://github.com/cclerget/singularity.git')

    # Update metadata config object

    seen = ['SREGISTRY_BUILDER_STORAGE_BUCKET', 'startup-script']

    for key, value in defaults.items():

        # This also appends empty values, they are meaningful
        if value not in seen:
            entry = { "key": key, 'value': value }
            metadata['items'].append(entry)
            seen.append(key)

    config['metadata'] = metadata
    return config
def setup_build ( self , name , repo , config , tag = None , commit = None , recipe = "Singularity" , startup_script = None ) : manager = self . _get_and_update_setting ( 'SREGISTRY_BUILDER_MANAGER' , 'apt' ) startup_script = get_build_template ( startup_script , manager ) # Read in the config to know what we can edit config = self . _load_build_config ( config ) if not config : bot . error ( 'Cannot find config, check path or URI.' ) sys . exit ( 1 ) # Ensure that the builder config is intended for the client self . _client_tagged ( config [ 'data' ] [ 'tags' ] ) # Compute settings that are parsed into runscript via metadata defaults = config [ 'data' ] [ 'metadata' ] selfLink = config [ 'links' ] [ 'self' ] # Make sure the builder repository and folder is passed forward builder_repo = config [ 'data' ] [ 'repo' ] builder_bundle = config [ 'data' ] [ 'path' ] builder_id = config [ 'data' ] [ 'id' ] config = config [ 'data' ] [ 'config' ] # Config settings from the environment, fall back to defaults image_project = defaults . get ( 'GOOGLE_COMPUTE_PROJECT' , 'debian-cloud' ) image_family = defaults . get ( 'GOOGLE_COMPUTE_IMAGE_FAMILY' , 'debian-8' ) # Generate names, description is for repo, name is random instance_name = "%s-builder %s" % ( name . replace ( '/' , '-' ) , selfLink ) robot_name = RobotNamer ( ) . generate ( ) project = self . _get_project ( ) zone = self . _get_zone ( ) # Machine Type machine_type = defaults . get ( 'SREGISTRY_BUILDER_machine_type' , 'n1-standard-1' ) machine_type = "zones/%s/machineTypes/%s" % ( zone , machine_type ) # Disk Size disk_size = defaults . get ( 'SREGISTRY_BUILDER_disk_size' , '100' ) # Get the image type image_response = self . _compute_service . images ( ) . getFromFamily ( project = image_project , family = image_family ) . execute ( ) source_disk_image = image_response [ 'selfLink' ] storage_bucket = self . _bucket_name # Add the machine parameters to the config config [ 'name' ] = robot_name config [ 'description' ] = instance_name config [ 'machineType' ] = machine_type config [ 'disks' ] . append ( { "autoDelete" : True , "boot" : True , "initializeParams" : { 'sourceImage' : source_disk_image , 'diskSizeGb' : disk_size } } ) # Metadata base metadata = { 'items' : [ { 'key' : 'startup-script' , 'value' : startup_script } , # Storage Settings from Host { 'key' : 'SREGISTRY_BUILDER_STORAGE_BUCKET' , 'value' : self . _bucket_name } ] } # Runtime variables take priority over defaults from config # and so here we update the defaults with runtime # ([defaults], [config-key], [runtime-setting]) # User Repository defaults = setconfig ( defaults , 'SREGISTRY_USER_REPO' , repo ) # Container Namespace (without tag/version) defaults = setconfig ( defaults , 'SREGISTRY_CONTAINER_NAME' , name ) # User Repository Commit defaults = setconfig ( defaults , 'SREGISTRY_USER_COMMIT' , commit ) # User Repository Branch defaults = setconfig ( defaults , 'SREGISTRY_USER_BRANCH' , "master" ) # User Repository Tag defaults = setconfig ( defaults , 'SREGISTRY_USER_TAG' , tag ) # Builder repository url defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_REPO' , builder_repo ) # Builder commit defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_COMMIT' ) # Builder default runscript defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_RUNSCRIPT' , "run.sh" ) # Builder repository url defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_BRANCH' , "master" ) # Builder id in repository defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_ID' , builder_id ) # Builder repository relative folder path defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_BUNDLE' , builder_bundle ) # Number of extra hours to debug defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_DEBUGHOURS' , "4" ) # Hours to kill running job defaults = setconfig ( defaults , 'SREGISTRY_BUILDER_KILLHOURS' , "10" ) # Recipe set at runtime defaults = setconfig ( defaults , 'SINGULARITY_RECIPE' , recipe ) # Branch of Singularity to install defaults = setconfig ( defaults , 'SINGULARITY_BRANCH' ) # Singularity commit to use (if needed) defaults = setconfig ( defaults , 'SINGULARITY_COMMIT' ) # Singularity Repo to Use defaults = setconfig ( defaults , 'SINGULARITY_REPO' , 'https://github.com/cclerget/singularity.git' ) # Update metadata config object seen = [ 'SREGISTRY_BUILDER_STORAGE_BUCKET' , 'startup-script' ] for key , value in defaults . items ( ) : # This also appends empty values, they are meaningful if value not in seen : entry = { "key" : key , 'value' : value } metadata [ 'items' ] . append ( entry ) seen . append ( key ) config [ 'metadata' ] = metadata return config
setup the build based on the selected configuration file, meaning
       producing the configuration file filled in based on the user's input
 
       Parameters
       ==========
       config: the complete configuration file provided by the client
       template: an optional custom start script to use
       tag: a user specified tag for the build, derived from uri or manual
       recipe: a recipe, if defined, overrides recipe set in config.
       commit: a commit to check out, if needed
       start_script: the start script to use, if not defined 
                     defaults to apt (or manager) base in main/templates/build
setup the build based on the selected configuration file meaning producing the configuration file filled in based on the user s input Parameters ========== config : the complete configuration file provided by the client template : an optional custom start script to use tag : a user specified tag for the build derived from uri or manual recipe : a recipe if defined overrides recipe set in config . commit : a commit to check out if needed start_script : the start script to use if not defined defaults to apt ( or manager ) base in main / templates / build