process jobs in alphabetical order in generated groovy script

import difflib.DiffUtils
import groovy.io.FileType
import hudson.AbortException
import java.io.StringBufferInputStream
import java.io.StringWriter
import javax.xml.parsers.DocumentBuilderFactory
import javax.xml.transform.stream.StreamSource
import jenkins.model.Jenkins
import org.apache.xml.serialize.OutputFormat
import org.apache.xml.serialize.XMLSerializer

// job types with their prefix and a list of job names
// used to remove obsolete jobs with the same prefix which are not part of the list
job_prefixes_and_names = [:]
@[for job_type in sorted(job_prefixes_and_names.keys())]@
job_prefixes_and_names['@job_type'] = [
    'job_prefix': '@(job_prefixes_and_names[job_type][0])',
    'job_names': [],
]
@{
job_names = sorted(job_prefixes_and_names[job_type][1])
group_size = 1000
}@
// group job names in chunks of @group_size to not exceed groovy limits
@[for i in range(0, len(job_names), group_size)]@
def add_job_prefixes_and_names_@(job_type)_@(int(i / group_size) + 1)(job_names) {
@[for j in range(i, min(i + group_size, len(job_names)))]@
job_names << '@(job_names[j])'
@[end for]@
}
add_job_prefixes_and_names_@(job_type)_@(int(i / group_size) + 1)(job_prefixes_and_names['@job_type'].job_names)
@[end for]@
@[end for]@

def diff_configs(current_config_file, new_config) {

    factory = DocumentBuilderFactory.newInstance()
    builder = factory.newDocumentBuilder()

    current_doc = builder.parse(current_config_file.getFile())

    new_config_stream = new StringBufferInputStream(new_config)
    new_doc = builder.parse(new_config_stream)

    // ignore description which contains a timestamp
    if (current_doc.getElementsByTagName('description')) {
        current_doc.getElementsByTagName('description').item(0).setTextContent('')
    }
    if (new_doc.getElementsByTagName('description')) {
        new_doc.getElementsByTagName('description').item(0).setTextContent('')
    }

    current_string = format_xml(current_doc)
    new_string = format_xml(new_doc)

    if (current_string == new_string) {
        return []
    }

    current_lines = []
    current_string.eachLine { line, count ->
        current_lines << line
    }
    new_lines = []
    new_string.eachLine { line, count ->
        new_lines << line
    }

    patch = DiffUtils.diff(current_lines, new_lines)
    return DiffUtils.generateUnifiedDiff('current config', 'new config', current_lines, patch, 0)
}

def format_xml(doc) {
    format = new OutputFormat(doc)
    format.setIndenting(true)
    format.setIndent(2)
    writer = new StringWriter()
    serializer = new XMLSerializer(writer, format)
    serializer.serialize(doc)
    return writer.toString()
}

// reconfigure jobs
println '# BEGIN SECTION: Groovy script - reconfigure jobs'
println 'Reconfiguring @(expected_num_jobs) jobs...'
created = 0
updated = 0
skipped = 0

config_dir = build.getWorkspace().toString() + '/reconfigure_jobs/configs'

def dir = new File(config_dir)
def jobs = dir.listFiles()
jobs.sort()

if (jobs.size() != @(expected_num_jobs)) {
    println "ERROR: Found different number of job configs than expected!! " + jobs.size() + " is not @(expected_num_jobs) as expected."
    // fail this build
    throw new AbortException("Wrong number of job configs")
}

jobs.each{
    found_project = false
    job_name = it.getName()
    job_config = new File(it.path).getText('UTF-8')
    for (p in Jenkins.instance.allItems) {
        if (p.name != job_name) continue
        found_project = true
        job_config_file = p.getConfigFile()

        diff = diff_configs(job_config_file, job_config)
        if (!diff) {
            println "Skipped job '" + job_name + "' because the config is the same"
            skipped += 1
        } else {
            println "Updating job '" + job_name + "'"
            println '    <<<'
            for (line in diff) {
                println '    ' + line
            }
            println '    <<<'

            reader = new StringReader(job_config)
            source = new StreamSource(reader)
            p.updateByXml(source)
            updated += 1
        }
        break
    }
    if (!found_project) {
        println "Creating job '" + job_name + "'"
        stream = new StringBufferInputStream(job_config)
        Jenkins.instance.createProjectFromXML(job_name, stream)
        created += 1
    }
}
println 'Created ' + created + ' jobs, updated ' + updated + ' jobs, skipped ' + skipped + ' jobs.'
println '# END SECTION'

// delete obsolete jobs
for (item in job_prefixes_and_names) {
    job_type = item.key
    job_prefix = item.value.job_prefix
    job_names = item.value.job_names
    deleted = 0
    println "# BEGIN SECTION: Groovy script - delete obsolete '" + job_type + "' jobs"
    println "Searching for obsolete jobs starting with '" + job_prefix + "'"
    for (p in Jenkins.instance.allItems) {
        if (!p.name.startsWith(job_prefix)) continue
        if (p.name in job_names) continue
        println "Deleting job '" + p.name + "'"
        p.delete()
        deleted += 1
    }
    println 'Deleted ' + deleted + ' jobs.'
    println '# END SECTION'
}

import difflib.DiffUtils
import groovy.io.FileType
import hudson.AbortException
import java.io.StringBufferInputStream
import java.io.StringWriter
import javax.xml.parsers.DocumentBuilderFactory
import javax.xml.transform.stream.StreamSource
import jenkins.model.Jenkins
import org.apache.xml.serialize.OutputFormat
import org.apache.xml.serialize.XMLSerializer

// job types with their prefix and a list of job names
// used to remove obsolete jobs with the same prefix which are not part of the list
job_prefixes_and_names = [:]
@[for job_type in sorted(job_prefixes_and_names.keys())]@
job_prefixes_and_names['@job_type'] = [
    'job_prefix': '@(job_prefixes_and_names[job_type][0])',
    'job_names': [],
]
@{
job_names = sorted(job_prefixes_and_names[job_type][1])
group_size = 1000
}@
// group job names in chunks of @group_size to not exceed groovy limits
@[for i in range(0, len(job_names), group_size)]@
def add_job_prefixes_and_names_@(job_type)_@(int(i / group_size) + 1)(job_names) {
@[for j in range(i, min(i + group_size, len(job_names)))]@
job_names << '@(job_names[j])'
@[end for]@
}
add_job_prefixes_and_names_@(job_type)_@(int(i / group_size) + 1)(job_prefixes_and_names['@job_type'].job_names)
@[end for]@
@[end for]@

def diff_configs(current_config_file, new_config) {

    factory = DocumentBuilderFactory.newInstance()
    builder = factory.newDocumentBuilder()

    current_doc = builder.parse(current_config_file.getFile())

    new_config_stream = new StringBufferInputStream(new_config)
    new_doc = builder.parse(new_config_stream)

    // ignore description which contains a timestamp
    if (current_doc.getElementsByTagName('description')) {
        current_doc.getElementsByTagName('description').item(0).setTextContent('')
    }
    if (new_doc.getElementsByTagName('description')) {
        new_doc.getElementsByTagName('description').item(0).setTextContent('')
    }

    current_string = format_xml(current_doc)
    new_string = format_xml(new_doc)

    if (current_string == new_string) {
        return []
    }

    current_lines = []
    current_string.eachLine { line, count ->
        current_lines << line
    }
    new_lines = []
    new_string.eachLine { line, count ->
        new_lines << line
    }

    patch = DiffUtils.diff(current_lines, new_lines)
    return DiffUtils.generateUnifiedDiff('current config', 'new config', current_lines, patch, 0)
}

def format_xml(doc) {
    format = new OutputFormat(doc)
    format.setIndenting(true)
    format.setIndent(2)
    writer = new StringWriter()
    serializer = new XMLSerializer(writer, format)
    serializer.serialize(doc)
    return writer.toString()
}

// reconfigure jobs
println '# BEGIN SECTION: Groovy script - reconfigure jobs'
println 'Reconfiguring @(expected_num_jobs) jobs...'
created = 0
updated = 0
skipped = 0

config_dir = build.getWorkspace().toString() + '/reconfigure_jobs/configs'

def jobs = []
def dir = new File(config_dir)
dir.eachFileRecurse (FileType.FILES) { file ->
    jobs << file
}

if (jobs.size() != @(expected_num_jobs)) {
    println "ERROR: Found different number of job configs than expected!! " + jobs.size() + " is not @(expected_num_jobs) as expected."
    // fail this build
    throw new AbortException("Wrong number of job configs")
}

jobs.each{
    found_project = false
    job_name = it.getName()
    job_config = new File(it.path).getText('UTF-8')
    for (p in Jenkins.instance.allItems) {
        if (p.name != job_name) continue
        found_project = true
        job_config_file = p.getConfigFile()

        diff = diff_configs(job_config_file, job_config)
        if (!diff) {
            println "Skipped job '" + job_name + "' because the config is the same"
            skipped += 1
        } else {
            println "Updating job '" + job_name + "'"
            println '    <<<'
            for (line in diff) {
                println '    ' + line
            }
            println '    <<<'

            reader = new StringReader(job_config)
            source = new StreamSource(reader)
            p.updateByXml(source)
            updated += 1
        }
        break
    }
    if (!found_project) {
        println "Creating job '" + job_name + "'"
        stream = new StringBufferInputStream(job_config)
        Jenkins.instance.createProjectFromXML(job_name, stream)
        created += 1
    }
}
println 'Created ' + created + ' jobs, updated ' + updated + ' jobs, skipped ' + skipped + ' jobs.'
println '# END SECTION'

// delete obsolete jobs
for (item in job_prefixes_and_names) {
    job_type = item.key
    job_prefix = item.value.job_prefix
    job_names = item.value.job_names
    deleted = 0
    println "# BEGIN SECTION: Groovy script - delete obsolete '" + job_type + "' jobs"
    println "Searching for obsolete jobs starting with '" + job_prefix + "'"
    for (p in Jenkins.instance.allItems) {
        if (!p.name.startsWith(job_prefix)) continue
        if (p.name in job_names) continue
        println "Deleting job '" + p.name + "'"
        p.delete()
        deleted += 1
    }
    println 'Deleted ' + deleted + ' jobs.'
    println '# END SECTION'
}


update default ros2.repos URL (#62)

update default ros2.repos URL (#62)

* update default ros2.repos URL

Signed-off-by: Mikael Arguedas <a16f295d1139a1392bb19262902c32cf3e999ce4@gmail.com>

* reverse order

Signed-off-by: Mikael Arguedas <a16f295d1139a1392bb19262902c32cf3e999ce4@gmail.com>

@[if 'vcs' in locals()]@
@[  if vcs]@
@[    for i, (imports_name, imports) in enumerate(vcs.items())]@
@{
if imports['repos'] is None:
    imports['repos'] = "https://raw.githubusercontent.com/ros2/ros2/$ROS_DISTRO-release/ros2.repos"
}@
RUN wget @(imports['repos']) \
    && vcs import @(ws) < @(imports['repos'].split('/')[-1])
@[    end for]@
@[  end if]@
@[end if]@

@[if 'vcs' in locals()]@
@[  if vcs]@
@[    for i, (imports_name, imports) in enumerate(vcs.items())]@
@{
if imports['repos'] is None:
    imports['repos'] = "https://raw.githubusercontent.com/ros2/ros2/$ROS_DISTRO/ros2.repos"
}@
RUN wget @(imports['repos']) \
    && vcs import @(ws) < @(imports['repos'].split('/')[-1])
@[    end for]@
@[  end if]@
@[end if]@


Add checks for expected functionality in PresenceEventsTest.

The previous fix was necessary because we didn't properly setup
ProxManager to ensure callbacks will be able to use the functionality
we expect in them. This adds a couple of simple tests to the
Emerson PresenceEventsTests to check that this functionality is
available. Currently the tests are whether system.self is set and
whether we can call getProxSet.

/**
 *  presenceEventsTest -- tests onPresenceConnected,
 *  onPresenceDisconnected, and createPresence callback
 *
 *  Scene.db:
 *   * Ent 1 : Anything
 */

system.require('emUtil/util.em');

mTest = new UnitTest('presenceEventsTest');

// We just want to make sure we can trigger each of the 3 events. A
// timeout will indicate if we failed.
var expectedEvents = 3;
var testTimeout = system.timeout(
    5,
    function() {
        mTest.fail('Timed out before all events were triggered.');
        system.killEntity();
    }
);

// Verify that functionality that should always be available in a presence
// callback is there
var verifyAvailableFunctionality = function() {
    // system.self should be available when we get connected
    if (system.self === undefined || system.self === null) {
        mTest.fail("system.self isn't available in presence connection event callback");
        return;
    }

    // getProxSet should work; even if we didn't register a query, it should
    // just be empty.
    try {
        var ps = system.getProxSet(system.self);
    }
    catch(e) {
        mTest.fail("Exception when trying to get prox set");
    }
};

var logEvent = function(name) {
    system.print('Callback ' + name + ' fired for ' + system.self + '\n');
    expectedEvents--;
    if (expectedEvents == 0) {
        testTimeout.clear();
        system.killEntity();
    }
};

system.onPresenceConnected(
    function(newPres, clearable) {
        clearable.clear();

        // 3 handlers
        var presenceConnectedCallback = function() {
            verifyAvailableFunctionality();
            logEvent('createPresenceCallback');
            // Delayed event, to ensure all presence connected callbacks finish
            system.event(
                function() {
                    system.self.disconnect();
                }
            );
        };
        system.onPresenceConnected(
            function() {
                verifyAvailableFunctionality();
                logEvent('onPresenceConnected');
            }
        );
        system.onPresenceDisconnected(
            function() {
                verifyAvailableFunctionality();
                logEvent('onPresenceDisconnected');
            }
        );

        // And this actually starts the process
        system.createPresence({}, presenceConnectedCallback);
    }
);

/**
 *  presenceEventsTest -- tests onPresenceConnected,
 *  onPresenceDisconnected, and createPresence callback
 *
 *  Scene.db:
 *   * Ent 1 : Anything
 */

system.require('emUtil/util.em');

mTest = new UnitTest('presenceEventsTest');

// We just want to make sure we can trigger each of the 3 events. A
// timeout will indicate if we failed.
var expectedEvents = 3;
var testTimeout = system.timeout(
    5,
    function() {
        mTest.fail('Timed out before all events were triggered.');
        system.killEntity();
    }
);
var logEvent = function(name) {
    system.print('Callback ' + name + ' fired for ' + system.self + '\n');
    expectedEvents--;
    if (expectedEvents == 0) {
        testTimeout.clear();
        system.killEntity();
    }
};

system.onPresenceConnected(
    function(newPres, clearable) {
        clearable.clear();

        // 3 handlers
        var presenceConnectedCallback = function() {
            logEvent('createPresenceCallback');
            // Delayed event, to ensure all presence connected callbacks finish
            system.event(
                function() {
                    system.self.disconnect();
                }
            );
        };
        system.onPresenceConnected(
            function() {
                logEvent('onPresenceConnected');
            }
        );
        system.onPresenceDisconnected(
            function() {
                logEvent('onPresenceDisconnected');
            }
        );

        // And this actually starts the process
        system.createPresence({}, presenceConnectedCallback);
    }
);


read stdout_stderr.log from build steps to extract CMake / compiler warnings (#477)

* read stdout_stderr.log from latest build to extract CMake / compiler warnings

Signed-off-by: Dirk Thomas <19795c8185177e940eedd30ceb2b716a54a3d5dc@users.noreply.github.com>

* use all build log directories, even if there is only one atm

Signed-off-by: Dirk Thomas <19795c8185177e940eedd30ceb2b716a54a3d5dc@users.noreply.github.com>

* also handle 'work space'

Signed-off-by: Dirk Thomas <19795c8185177e940eedd30ceb2b716a54a3d5dc@users.noreply.github.com>

<io.jenkins.plugins.analysis.core.steps.IssuesRecorder plugin="warnings-ng@@7.3.0">
  <analysisTools>
@[for workspace in ['ws', 'work space']]@
    <io.jenkins.plugins.analysis.warnings.Cmake>
      <id></id>
      <name></name>
      <pattern>@(workspace)/log/build_*/*/stdout_stderr.log</pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Cmake>
@[  if os_name in ['linux', 'linux-armhf', 'linux-aarch64', 'linux-centos']]@
   <io.jenkins.plugins.analysis.warnings.Gcc4>
      <id></id>
      <name></name>
      <pattern>@(workspace)/log/build_*/*/stdout_stderr.log</pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Gcc4>
@[  end if]
@[  if os_name in ['osx', 'linux', 'linux-armhf', 'linux-aarch64', 'linux-centos']]@
    <io.jenkins.plugins.analysis.warnings.Clang>
      <id></id>
      <name></name>
      <pattern>@(workspace)/log/build_*/*/stdout_stderr.log</pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Clang>
    <io.jenkins.plugins.analysis.warnings.ClangTidy>
      <id></id>
      <name></name>
      <pattern>@(workspace)/log/build_*/*/stdout_stderr.log</pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.ClangTidy>
@[  elif os_name in ['windows', 'windows-metal']]@
    <io.jenkins.plugins.analysis.warnings.MsBuild>
      <id></id>
      <name></name>
      <pattern>@(workspace)/log/build_*/*/stdout_stderr.log</pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.MsBuild>
@[  else]@
@{assert False, 'Unknown os_name: ' + os_name}@
@[  end if]@
@[end for]@
  </analysisTools>
  <sourceCodeEncoding></sourceCodeEncoding>
  <sourceDirectory></sourceDirectory>
  <ignoreQualityGate>false</ignoreQualityGate>
  <ignoreFailedBuilds>true</ignoreFailedBuilds>
  <referenceJobName>-</referenceJobName>
  <failOnError>false</failOnError>
  <healthy>0</healthy>
  <unhealthy>0</unhealthy>
  <minimumSeverity plugin="analysis-model-api@@7.0.2">
    <name>LOW</name>
  </minimumSeverity>
  <filters/>
  <isEnabledForFailure>false</isEnabledForFailure>
  <isAggregatingResults>false</isAggregatingResults>
  <isBlameDisabled>false</isBlameDisabled>
  <isForensicsDisabled>false</isForensicsDisabled>
  <qualityGates>
    <io.jenkins.plugins.analysis.core.util.QualityGate>
      <threshold>1</threshold>
      <type>TOTAL</type>
      <status>WARNING</status>
    </io.jenkins.plugins.analysis.core.util.QualityGate>
  </qualityGates>
  <trendChartType>AGGREGATION_TOOLS</trendChartType>
</io.jenkins.plugins.analysis.core.steps.IssuesRecorder>

<io.jenkins.plugins.analysis.core.steps.IssuesRecorder plugin="warnings-ng@@7.3.0">
  <analysisTools>
    <io.jenkins.plugins.analysis.warnings.Cmake>
      <id></id>
      <name></name>
      <pattern></pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Cmake>
@[if os_name in ['linux', 'linux-armhf', 'linux-aarch64', 'linux-centos']]@
   <io.jenkins.plugins.analysis.warnings.Gcc4>
      <id></id>
      <name></name>
      <pattern></pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Gcc4>
@[end if]
@[if os_name in ['osx', 'linux', 'linux-armhf', 'linux-aarch64', 'linux-centos']]@
    <io.jenkins.plugins.analysis.warnings.Clang>
      <id></id>
      <name></name>
      <pattern></pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.Clang>
    <io.jenkins.plugins.analysis.warnings.ClangTidy>
      <id></id>
      <name></name>
      <pattern></pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.ClangTidy>
@[elif os_name in ['windows', 'windows-metal']]@
    <io.jenkins.plugins.analysis.warnings.MsBuild>
      <id></id>
      <name></name>
      <pattern></pattern>
      <reportEncoding></reportEncoding>
      <skipSymbolicLinks>false</skipSymbolicLinks>
    </io.jenkins.plugins.analysis.warnings.MsBuild>
@[else]@
@{assert False, 'Unknown os_name: ' + os_name}@
@[end if]@
  </analysisTools>
  <sourceCodeEncoding></sourceCodeEncoding>
  <sourceDirectory></sourceDirectory>
  <ignoreQualityGate>false</ignoreQualityGate>
  <ignoreFailedBuilds>true</ignoreFailedBuilds>
  <referenceJobName>-</referenceJobName>
  <failOnError>false</failOnError>
  <healthy>0</healthy>
  <unhealthy>0</unhealthy>
  <minimumSeverity plugin="analysis-model-api@@7.0.2">
    <name>LOW</name>
  </minimumSeverity>
  <filters/>
  <isEnabledForFailure>false</isEnabledForFailure>
  <isAggregatingResults>false</isAggregatingResults>
  <isBlameDisabled>false</isBlameDisabled>
  <isForensicsDisabled>false</isForensicsDisabled>
  <qualityGates>
    <io.jenkins.plugins.analysis.core.util.QualityGate>
      <threshold>1</threshold>
      <type>TOTAL</type>
      <status>WARNING</status>
    </io.jenkins.plugins.analysis.core.util.QualityGate>
  </qualityGates>
  <trendChartType>AGGREGATION_TOOLS</trendChartType>
</io.jenkins.plugins.analysis.core.steps.IssuesRecorder>


Add missing build dependencies to fix race conditions

find_package(opensplice REQUIRED)
include_directories(${OPENSPLICE_INCLUDE_DIRS})

set(GENIDLCPP_PP_BIN "idlpp")
@[if DEVELSPACE]@
# bin and template dir variables in develspace
set(GENIDLCPP_BIN "@(CMAKE_CURRENT_SOURCE_DIR)/scripts/gen_idl_cpp_convert.py")
set(GENIDLCPP_TEMPLATE_DIR "@(CMAKE_CURRENT_SOURCE_DIR)/scripts")
@[else]@
# bin and template dir variables in installspace
set(GENIDLCPP_BIN "${genidlcpp_DIR}/../../../@(CATKIN_PACKAGE_BIN_DESTINATION)/gen_idl_cpp_convert.py")
set(GENIDLCPP_TEMPLATE_DIR "${genidlcpp_DIR}/..")
@[end if]@

# Generate .idl -> dds code generation and _convert.h
# The generated files should be added to ALL_GEN_OUTPUT_FILES_idlcpp
macro(_generate_msg_idlcpp ARG_PKG ARG_MSG ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg ${ARG_PKG} ${ARG_MSG} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  # generate dds code for message
  get_filename_component(MSG_SHORT_NAME ${ARG_MSG} NAME_WE)
  _generate_msg_idlcpp_dds(${ARG_PKG} ${MSG_SHORT_NAME} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  # generate _convert.h for message
  _generate_msg_idlcpp_convert(${ARG_PKG} ${ARG_MSG} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
endmacro()

# Generate .idl -> dds code generation and _convert.h
# The generated files should be added to ALL_GEN_OUTPUT_FILES_idlcpp
macro(_generate_srv_idlcpp ARG_PKG ARG_SRV ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("srv ${ARG_PKG} ${ARG_SRV} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  # generate dds code for service as well as request and response messages
  get_filename_component(SRV_SHORT_NAME ${ARG_SRV} NAME_WE)
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}Request" "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}Response" "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  # Note that we add FooRequest.msg and FooResponse.msg to ARG_MSG_DEPS
  # for Foo.srv, even though those .msg files never actually exist.  We do
  # this to ensure that the corresponding FooRequest_.h and FooResponse_.h
  # files will have been generated by the time we try to process Foo_.idl.
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}" "${ARG_IFLAGS}" "${ARG_MSG_DEPS};${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${ARG_PKG}/dds_impl/${SRV_SHORT_NAME}Request.msg;${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${ARG_PKG}/dds_impl/${SRV_SHORT_NAME}Response.msg" ${ARG_GEN_OUTPUT_DIR})
  # generate _convert.h for request and response
  _generate_msg_idlcpp_convert(${ARG_PKG} ${ARG_SRV} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
endmacro()

macro(_generate_msg_idlcpp_dds ARG_PKG MSG_SHORT_NAME ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg_dds ${ARG_PKG} ${MSG_SHORT_NAME} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  set(IDL_INPUT_FILE "${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${ARG_PKG}/dds_idl/${MSG_SHORT_NAME}_.idl")
  set(GEN_OUTPUT_FILES
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps_impl.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps_impl.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_SplDcps.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_SplDcps.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/ccpp_${MSG_SHORT_NAME}_.h"
  )

  set(IDL_DEPS "")
  foreach(dep ${ARG_MSG_DEPS})
    get_filename_component(dep_name ${dep} NAME_WE)
    get_filename_component(dep_pkg ${dep} PATH)
    get_filename_component(dep_pkg ${dep_pkg} PATH)
    get_filename_component(dep_pkg ${dep_pkg} NAME)
    list(APPEND IDL_DEPS "${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${dep_pkg}/dds_idl/${dep_name}_.idl")
    list(APPEND ${ARG_PKG}_MSG_DEPENDENCIES "${dep_pkg}")
  endforeach()

  set(COMMAND "${GENIDLCPP_PP_BIN}" -I "@(CATKIN_DEVEL_PREFIX)/@(CATKIN_GLOBAL_SHARE_DESTINATION)" -S -l cpp -o dds-types -d "${ARG_GEN_OUTPUT_DIR}/dds_impl" "${IDL_INPUT_FILE}")

  string(REPLACE ";" " " COMMAND_STR "${COMMAND}")
  #message("${COMMAND_STR}")

  file(MAKE_DIRECTORY ${ARG_GEN_OUTPUT_DIR}/dds_impl)

  add_custom_command(OUTPUT ${GEN_OUTPUT_FILES}
    DEPENDS ${IDL_DEPS} ${IDL_INPUT_FILE}
    COMMAND ${COMMAND}
    COMMENT "Generating DDS C++ code from ${ARG_PKG}/${MSG_SHORT_NAME}_.idl"
    )
  list(APPEND ALL_GEN_OUTPUT_FILES_idlcpp ${GEN_OUTPUT_FILES})

  genidlcpp_append_include_dirs()
endmacro()

macro(_generate_msg_idlcpp_convert ARG_PKG ARG_MSG ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg_convert ${ARG_PKG} ${ARG_MSG} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  get_filename_component(MSG_SHORT_NAME ${ARG_MSG} NAME_WE)
  set(GEN_OUTPUT_FILES
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_convert.h"
  )

  set(COMMAND ${GENIDLCPP_BIN} ${ARG_MSG} ${ARG_PKG} -o "${ARG_GEN_OUTPUT_DIR}/dds_impl" -e ${GENIDLCPP_TEMPLATE_DIR})

  string(REPLACE ";" " " COMMAND_STR "${COMMAND}")
  #message("${COMMAND_STR}")

  get_filename_component(MSG_NAME ${ARG_MSG} NAME)

  assert(CATKIN_ENV)
  add_custom_command(OUTPUT ${GEN_OUTPUT_FILES}
    DEPENDS ${GENIDLCPP_BIN} ${ARG_MSG} "${GENIDLCPP_TEMPLATE_DIR}/msg_convert.h.template"
    COMMAND ${CATKIN_ENV} ${PYTHON_EXECUTABLE} ${COMMAND}
    COMMENT "Generating conversion code between ROS and DDS message instances for ${ARG_PKG}/${MSG_NAME}"
    )
  list(APPEND ALL_GEN_OUTPUT_FILES_idlcpp ${GEN_OUTPUT_FILES})

  genidlcpp_append_include_dirs()
endmacro()

macro(_generate_module_idlcpp ARG_PKG ARG_GEN_OUTPUT_DIR)
  include_directories(BEFORE ${genidlcpp_INCLUDE_DIRS})
  if(${ARG_PKG}_MSG_DEPENDENCIES)
    list(REMOVE_DUPLICATES ${ARG_PKG}_MSG_DEPENDENCIES)
  endif()
  set(dds_impl_libs "")
  foreach(dep ${${ARG_PKG}_MSG_DEPENDENCIES})
    include_directories(BEFORE ${CATKIN_DEVEL_PREFIX}/include/${dep}/dds_impl)
    list(APPEND dds_impl_libs ${dep}_dds_msgs)
  endforeach()
  if(NOT "${ARGN}" STREQUAL "")
    add_library(${PROJECT_NAME}_dds_msgs ${ARGN})
    # This dependency is here to prevent simultaneous invocation of idlpp
    # on the same .idl file.  In that situation, the two invocations fight
    # over access to a temporary intermediate file and cause intermittent
    # crashes.  I'm not sure exactly why we were getting simultanous
    # invocations in the first place, but this dependency seems to prevent
    # it.
    add_dependencies(${PROJECT_NAME}_dds_msgs ${PROJECT_NAME}_genidlcpp)
    target_link_libraries(${PROJECT_NAME}_dds_msgs ${dds_impl_libs} ${OPENSPLICE_LIBRARIES})
    list(APPEND ${PROJECT_NAME}_LIBRARIES ${PROJECT_NAME}_dds_msgs)
  endif()
endmacro()

set(genidlcpp_INSTALL_DIR include)

macro(genidlcpp_append_include_dirs)
  if(NOT genidlcpp_APPENDED_INCLUDE_DIRS)
    # make sure we can find generated messages and that they overlay all other includes
    include_directories(BEFORE ${CATKIN_DEVEL_PREFIX}/${genidlcpp_INSTALL_DIR})
    # pass the include directory to catkin_package()
    list(APPEND ${PROJECT_NAME}_INCLUDE_DIRS ${CATKIN_DEVEL_PREFIX}/${genidlcpp_INSTALL_DIR})
    set(genidlcpp_APPENDED_INCLUDE_DIRS TRUE)
  endif()
endmacro()

find_package(opensplice REQUIRED)
include_directories(${OPENSPLICE_INCLUDE_DIRS})

set(GENIDLCPP_PP_BIN "idlpp")
@[if DEVELSPACE]@
# bin and template dir variables in develspace
set(GENIDLCPP_BIN "@(CMAKE_CURRENT_SOURCE_DIR)/scripts/gen_idl_cpp_convert.py")
set(GENIDLCPP_TEMPLATE_DIR "@(CMAKE_CURRENT_SOURCE_DIR)/scripts")
@[else]@
# bin and template dir variables in installspace
set(GENIDLCPP_BIN "${genidlcpp_DIR}/../../../@(CATKIN_PACKAGE_BIN_DESTINATION)/gen_idl_cpp_convert.py")
set(GENIDLCPP_TEMPLATE_DIR "${genidlcpp_DIR}/..")
@[end if]@

# Generate .idl -> dds code generation and _convert.h
# The generated files should be added to ALL_GEN_OUTPUT_FILES_idlcpp
macro(_generate_msg_idlcpp ARG_PKG ARG_MSG ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg ${ARG_PKG} ${ARG_MSG} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  # generate dds code for message
  get_filename_component(MSG_SHORT_NAME ${ARG_MSG} NAME_WE)
  _generate_msg_idlcpp_dds(${ARG_PKG} ${MSG_SHORT_NAME} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  # generate _convert.h for message
  _generate_msg_idlcpp_convert(${ARG_PKG} ${ARG_MSG} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
endmacro()

# Generate .idl -> dds code generation and _convert.h
# The generated files should be added to ALL_GEN_OUTPUT_FILES_idlcpp
macro(_generate_srv_idlcpp ARG_PKG ARG_SRV ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("srv ${ARG_PKG} ${ARG_SRV} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  # generate dds code for service as well as request and response messages
  get_filename_component(SRV_SHORT_NAME ${ARG_SRV} NAME_WE)
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}" "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}Request" "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  _generate_msg_idlcpp_dds(${ARG_PKG} "${SRV_SHORT_NAME}Response" "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
  # generate _convert.h for request and response
  _generate_msg_idlcpp_convert(${ARG_PKG} ${ARG_SRV} "${ARG_IFLAGS}" "${ARG_MSG_DEPS}" ${ARG_GEN_OUTPUT_DIR})
endmacro()

macro(_generate_msg_idlcpp_dds ARG_PKG MSG_SHORT_NAME ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg_dds ${ARG_PKG} ${MSG_SHORT_NAME} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  set(IDL_INPUT_FILE "${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${ARG_PKG}/dds_idl/${MSG_SHORT_NAME}_.idl")
  set(GEN_OUTPUT_FILES
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps_impl.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_Dcps_impl.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_SplDcps.h"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_SplDcps.cpp"
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/ccpp_${MSG_SHORT_NAME}_.h"
  )

  set(IDL_DEPS "")
  foreach(dep ${ARG_MSG_DEPS})
    get_filename_component(dep_name ${dep} NAME_WE)
    get_filename_component(dep_pkg ${dep} PATH)
    get_filename_component(dep_pkg ${dep_pkg} PATH)
    get_filename_component(dep_pkg ${dep_pkg} NAME)
    list(APPEND IDL_DEPS "${CATKIN_DEVEL_PREFIX}/${CATKIN_GLOBAL_SHARE_DESTINATION}/${dep_pkg}/dds_idl/${dep_name}_.idl")
    list(APPEND ${ARG_PKG}_MSG_DEPENDENCIES "${dep_pkg}")
  endforeach()

  set(COMMAND "${GENIDLCPP_PP_BIN}" -I "@(CATKIN_DEVEL_PREFIX)/@(CATKIN_GLOBAL_SHARE_DESTINATION)" -S -l cpp -o dds-types -d "${ARG_GEN_OUTPUT_DIR}/dds_impl" "${IDL_INPUT_FILE}")

  string(REPLACE ";" " " COMMAND_STR "${COMMAND}")
  #message("${COMMAND_STR}")

  file(MAKE_DIRECTORY ${ARG_GEN_OUTPUT_DIR}/dds_impl)

  add_custom_command(OUTPUT ${GEN_OUTPUT_FILES}
    DEPENDS ${IDL_DEPS} ${IDL_INPUT_FILE}
    COMMAND ${COMMAND}
    COMMENT "Generating DDS C++ code from ${ARG_PKG}/${MSG_SHORT_NAME}_.idl"
    )
  list(APPEND ALL_GEN_OUTPUT_FILES_idlcpp ${GEN_OUTPUT_FILES})

  genidlcpp_append_include_dirs()
endmacro()

macro(_generate_msg_idlcpp_convert ARG_PKG ARG_MSG ARG_IFLAGS ARG_MSG_DEPS ARG_GEN_OUTPUT_DIR)
  #message("msg_convert ${ARG_PKG} ${ARG_MSG} ${ARG_IFLAGS} ${ARG_MSG_DEPS}")

  get_filename_component(MSG_SHORT_NAME ${ARG_MSG} NAME_WE)
  set(GEN_OUTPUT_FILES
    "${ARG_GEN_OUTPUT_DIR}/dds_impl/${MSG_SHORT_NAME}_convert.h"
  )

  set(COMMAND ${GENIDLCPP_BIN} ${ARG_MSG} ${ARG_PKG} -o "${ARG_GEN_OUTPUT_DIR}/dds_impl" -e ${GENIDLCPP_TEMPLATE_DIR})

  string(REPLACE ";" " " COMMAND_STR "${COMMAND}")
  #message("${COMMAND_STR}")

  get_filename_component(MSG_NAME ${ARG_MSG} NAME)

  assert(CATKIN_ENV)
  add_custom_command(OUTPUT ${GEN_OUTPUT_FILES}
    DEPENDS ${GENIDLCPP_BIN} ${ARG_MSG} "${GENIDLCPP_TEMPLATE_DIR}/msg_convert.h.template"
    COMMAND ${CATKIN_ENV} ${PYTHON_EXECUTABLE} ${COMMAND}
    COMMENT "Generating conversion code between ROS and DDS message instances for ${ARG_PKG}/${MSG_NAME}"
    )
  list(APPEND ALL_GEN_OUTPUT_FILES_idlcpp ${GEN_OUTPUT_FILES})

  genidlcpp_append_include_dirs()
endmacro()

macro(_generate_module_idlcpp ARG_PKG ARG_GEN_OUTPUT_DIR)
  include_directories(BEFORE ${genidlcpp_INCLUDE_DIRS})
  if(${ARG_PKG}_MSG_DEPENDENCIES)
    list(REMOVE_DUPLICATES ${ARG_PKG}_MSG_DEPENDENCIES)
  endif()
  set(dds_impl_libs "")
  foreach(dep ${${ARG_PKG}_MSG_DEPENDENCIES})
    include_directories(BEFORE ${CATKIN_DEVEL_PREFIX}/include/${dep}/dds_impl)
    list(APPEND dds_impl_libs ${dep}_dds_msgs)
  endforeach()
  if(NOT "${ARGN}" STREQUAL "")
    add_library(${PROJECT_NAME}_dds_msgs ${ARGN})
    target_link_libraries(${PROJECT_NAME}_dds_msgs ${dds_impl_libs} ${OPENSPLICE_LIBRARIES})
    list(APPEND ${PROJECT_NAME}_LIBRARIES ${PROJECT_NAME}_dds_msgs)
  endif()
endmacro()

set(genidlcpp_INSTALL_DIR include)

macro(genidlcpp_append_include_dirs)
  if(NOT genidlcpp_APPENDED_INCLUDE_DIRS)
    # make sure we can find generated messages and that they overlay all other includes
    include_directories(BEFORE ${CATKIN_DEVEL_PREFIX}/${genidlcpp_INSTALL_DIR})
    # pass the include directory to catkin_package()
    list(APPEND ${PROJECT_NAME}_INCLUDE_DIRS ${CATKIN_DEVEL_PREFIX}/${genidlcpp_INSTALL_DIR})
    set(genidlcpp_APPENDED_INCLUDE_DIRS TRUE)
  endif()
endmacro()


Install apt transport https (#551)

Install apt transport https (#551)

* dont' overwrite default sources

* Revert "dont' overwrite default sources"

This reverts commit 0d2c3a4fec23bb1ddc28509f0d4ae37437f084ec.

* add template to install apt-transport-https on debian

* install apt-transport-https in image templates

* change branch name accordingly

* Install apt-transport-https for the cloudfront servers.

Signed-off-by: Chris Lalancette <281cd07d7578d97c83271fbbf2faddb83ab3791c@openrobotics.org>

* update comment

* More installation of apt-transport-https before changing mirrors.

Signed-off-by: Chris Lalancette <281cd07d7578d97c83271fbbf2faddb83ab3791c@openrobotics.org>

* Switch back to master.

Signed-off-by: Chris Lalancette <281cd07d7578d97c83271fbbf2faddb83ab3791c@openrobotics.org>

@[if os_name == 'ubuntu']@
@{
from itertools import product
}@
@[  if arch in ['amd64', 'i386']]@
@{
archive_commands = []
old_releases_commands = []
for distribution, archive_type in product((os_code_name, os_code_name + '-updates', os_code_name + '-security'), ('deb', 'deb-src')):
    archive_entry = '%s http://archive.ubuntu.com/ubuntu/ %s multiverse' % (archive_type, distribution)
    archive_pattern = '%s http://archive\.ubuntu\.com/ubuntu/? %s ([-a-z]+ )*multiverse( [-a-z])*' % (archive_type, distribution)
    old_releases_entry = '%s http://old-releases.ubuntu.com/ubuntu/ %s multiverse' % (archive_type, distribution)
    old_releases_pattern = '%s http://old-releases\.ubuntu\.com/ubuntu/? %s ([-a-z]+ )*multiverse( [-a-z]+)*' % (archive_type, distribution)
    archive_commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (archive_pattern, archive_entry))
    old_releases_commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (old_releases_pattern, old_releases_entry))
}@
RUN grep -q -F -e "deb http://old-releases.ubuntu.com" /etc/apt/sources.list && (@(' && '.join(old_releases_commands))) || (@(' && '.join(archive_commands)))
@[  elif arch in ['armhf', 'armv8']]@
@{
commands = []
for distribution, archive_type in product((os_code_name, os_code_name + '-updates', os_code_name + '-security'), ('deb', 'deb-src')):
    entry = '%s http://ports.ubuntu.com/ %s multiverse' % (archive_type, distribution)
    pattern = '%s http://ports\.ubuntu\.com/? %s ([-a-z]+ )*multiverse( [-a-z])*' % (archive_type, distribution)
    commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (pattern, entry))
}@
RUN @(' && '.join(commands))
@[  end if]@
@[else if os_name == 'debian']@
# Add contrib and non-free to debian images
# Using httpredir here to match mirror used in osrf image
# (https://github.com/osrf/multiarch-docker-image-generation/blob/d251b9a/build-image.sh#L46)
@{
commands = []
for component in ('contrib', 'non-free'):
    entry = 'deb http://httpredir.debian.org/debian %s %s' % (os_code_name, component)
    pattern = 'deb http://httpredir\.debian\.org/debian/? %s ([-a-z] )*%s( [-a-z])*' % (os_code_name, component)
    commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (pattern, entry))
}@
RUN @(' && '.join(commands))
# Make sure to install apt-transport-https since some CloudFront mirrors are currently being redirected to https
RUN for i in 1 2 3; do apt-get update && apt-get install -q -y apt-transport-https && apt-get clean && break || if [[ $i < 3 ]]; then sleep 5; else false; fi; done
# Hit cloudfront mirror because of corrupted packages on fastly mirrors (https://github.com/ros-infrastructure/ros_buildfarm/issues/455)
# You can remove this line to target the default mirror or replace this to use the mirror of your preference
RUN sed -i 's/httpredir\.debian\.org/cloudfront.debian.net/' /etc/apt/sources.list
@[end if]@

@[if os_name == 'ubuntu']@
@{
from itertools import product
}@
@[  if arch in ['amd64', 'i386']]@
@{
archive_commands = []
old_releases_commands = []
for distribution, archive_type in product((os_code_name, os_code_name + '-updates', os_code_name + '-security'), ('deb', 'deb-src')):
    archive_entry = '%s http://archive.ubuntu.com/ubuntu/ %s multiverse' % (archive_type, distribution)
    archive_pattern = '%s http://archive\.ubuntu\.com/ubuntu/? %s ([-a-z]+ )*multiverse( [-a-z])*' % (archive_type, distribution)
    old_releases_entry = '%s http://old-releases.ubuntu.com/ubuntu/ %s multiverse' % (archive_type, distribution)
    old_releases_pattern = '%s http://old-releases\.ubuntu\.com/ubuntu/? %s ([-a-z]+ )*multiverse( [-a-z]+)*' % (archive_type, distribution)
    archive_commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (archive_pattern, archive_entry))
    old_releases_commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (old_releases_pattern, old_releases_entry))
}@
RUN grep -q -F -e "deb http://old-releases.ubuntu.com" /etc/apt/sources.list && (@(' && '.join(old_releases_commands))) || (@(' && '.join(archive_commands)))
@[  elif arch in ['armhf', 'armv8']]@
@{
commands = []
for distribution, archive_type in product((os_code_name, os_code_name + '-updates', os_code_name + '-security'), ('deb', 'deb-src')):
    entry = '%s http://ports.ubuntu.com/ %s multiverse' % (archive_type, distribution)
    pattern = '%s http://ports\.ubuntu\.com/? %s ([-a-z]+ )*multiverse( [-a-z])*' % (archive_type, distribution)
    commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (pattern, entry))
}@
RUN @(' && '.join(commands))
@[  end if]@
@[else if os_name == 'debian']@
# Add contrib and non-free to debian images
# Using httpredir here to match mirror used in osrf image
# (https://github.com/osrf/multiarch-docker-image-generation/blob/d251b9a/build-image.sh#L46)
@{
commands = []
for component in ('contrib', 'non-free'):
    entry = 'deb http://httpredir.debian.org/debian %s %s' % (os_code_name, component)
    pattern = 'deb http://httpredir\.debian\.org/debian/? %s ([-a-z] )*%s( [-a-z])*' % (os_code_name, component)
    commands.append('(grep -q -E -x -e "%s" /etc/apt/sources.list || echo "%s" >> /etc/apt/sources.list)' % (pattern, entry))
}@
RUN @(' && '.join(commands))
# Hit cloudfront mirror because of corrupted packages on fastly mirrors (https://github.com/ros-infrastructure/ros_buildfarm/issues/455)
# You can remove this line to target the default mirror or replace this to use the mirror of your preference
RUN sed -i 's/httpredir\.debian\.org/cloudfront.debian.net/' /etc/apt/sources.list
@[end if]@


use job specific ghprb commit status context

    <org.jenkinsci.plugins.ghprb.GhprbTrigger plugin="ghprb@@1.29.8">
      <spec/>
      <triggerPhrase/>
      <adminlist/>
      <allowMembersOfWhitelistedOrgsAsAdmin>true</allowMembersOfWhitelistedOrgsAsAdmin>
      <orgslist>@ESCAPE(github_orgunit)</orgslist>
      <cron/>
      <buildDescTemplate/>
      <onlyTriggerPhrase>false</onlyTriggerPhrase>
      <useGitHubHooks>true</useGitHubHooks>
      <permitAll>true</permitAll>
      <whitelist/>
      <autoCloseFailedPullRequests>false</autoCloseFailedPullRequests>
      <displayBuildErrorsOnDownstreamBuilds>false</displayBuildErrorsOnDownstreamBuilds>
      <whiteListTargetBranches>
        <org.jenkinsci.plugins.ghprb.GhprbBranch>
          <branch>@ESCAPE(branch_name)</branch>
        </org.jenkinsci.plugins.ghprb.GhprbBranch>
      </whiteListTargetBranches>
      <project>@ESCAPE(job_name)</project>
      <extensions>
        <org.jenkinsci.plugins.ghprb.extensions.status.GhprbSimpleStatus>
          <commitStatusContext>@ESCAPE(job_name)</commitStatusContext>
          <triggeredStatus/>
          <startedStatus/>
          <statusUrl/>
        </org.jenkinsci.plugins.ghprb.extensions.status.GhprbSimpleStatus>
      </extensions>
    </org.jenkinsci.plugins.ghprb.GhprbTrigger>

    <org.jenkinsci.plugins.ghprb.GhprbTrigger plugin="ghprb@@1.29.8">
      <spec/>
      <adminlist/>
      <allowMembersOfWhitelistedOrgsAsAdmin>true</allowMembersOfWhitelistedOrgsAsAdmin>
      <orgslist>@ESCAPE(github_orgunit)</orgslist>
      <cron/>
      <triggerPhrase/>
      <onlyTriggerPhrase>false</onlyTriggerPhrase>
      <useGitHubHooks>true</useGitHubHooks>
      <permitAll>true</permitAll>
      <commentFilePath/>
      <whitelist/>
      <autoCloseFailedPullRequests>false</autoCloseFailedPullRequests>
      <displayBuildErrorsOnDownstreamBuilds>false</displayBuildErrorsOnDownstreamBuilds>
      <whiteListTargetBranches>
        <org.jenkinsci.plugins.ghprb.GhprbBranch>
          <branch>@ESCAPE(branch_name)</branch>
        </org.jenkinsci.plugins.ghprb.GhprbBranch>
      </whiteListTargetBranches>
      <msgSuccess/>
      <msgFailure/>
      <project>@ESCAPE(job_name)</project>
    </org.jenkinsci.plugins.ghprb.GhprbTrigger>


fix logic of publisher warnings argument

    <hudson.plugins.warnings.WarningsPublisher plugin="warnings@@4.52">
      <healthy/>
      <unHealthy/>
      <thresholdLimit>low</thresholdLimit>
      <pluginName>[WARNINGS]</pluginName>
      <defaultEncoding/>
      <canRunOnFailed>false</canRunOnFailed>
      <usePreviousBuildAsReference>false</usePreviousBuildAsReference>
      <useStableBuildAsReference>false</useStableBuildAsReference>
      <useDeltaValues>false</useDeltaValues>
      <thresholds plugin="analysis-core@@1.76">
@[if unstable_threshold != '']@
        <unstableTotalAll>@unstable_threshold</unstableTotalAll>
@[else]@
        <unstableTotalAll/>
@[end if]@
        <unstableTotalHigh/>
        <unstableTotalNormal/>
        <unstableTotalLow/>
        <unstableNewAll/>
        <unstableNewHigh/>
        <unstableNewNormal/>
        <unstableNewLow/>
        <failedTotalAll/>
        <failedTotalHigh/>
        <failedTotalNormal/>
        <failedTotalLow/>
        <failedNewAll/>
        <failedNewHigh/>
        <failedNewNormal/>
        <failedNewLow/>
      </thresholds>
      <shouldDetectModules>false</shouldDetectModules>
      <dontComputeNew>true</dontComputeNew>
      <doNotResolveRelativePaths>true</doNotResolveRelativePaths>
      <includePattern/>
      <excludePattern/>
      <messagesPattern/>
      <parserConfigurations/>
      <consoleParsers>
      <hudson.plugins.warnings.ConsoleParser>
      <parserName>GNU C Compiler 4 (gcc)</parserName>
      </hudson.plugins.warnings.ConsoleParser>
      </consoleParsers>
    </hudson.plugins.warnings.WarningsPublisher>

    <hudson.plugins.warnings.WarningsPublisher plugin="warnings@@4.52">
      <healthy/>
      <unHealthy/>
      <thresholdLimit>low</thresholdLimit>
      <pluginName>[WARNINGS]</pluginName>
      <defaultEncoding/>
      <canRunOnFailed>false</canRunOnFailed>
      <usePreviousBuildAsReference>false</usePreviousBuildAsReference>
      <useStableBuildAsReference>false</useStableBuildAsReference>
      <useDeltaValues>false</useDeltaValues>
      <thresholds plugin="analysis-core@@1.76">
@[if unstable_threshold]@
        <unstableTotalAll>@unstable_threshold</unstableTotalAll>
@[else]@
        <unstableTotalAll/>
@[end if]@
        <unstableTotalHigh/>
        <unstableTotalNormal/>
        <unstableTotalLow/>
        <unstableNewAll/>
        <unstableNewHigh/>
        <unstableNewNormal/>
        <unstableNewLow/>
        <failedTotalAll/>
        <failedTotalHigh/>
        <failedTotalNormal/>
        <failedTotalLow/>
        <failedNewAll/>
        <failedNewHigh/>
        <failedNewNormal/>
        <failedNewLow/>
      </thresholds>
      <shouldDetectModules>false</shouldDetectModules>
      <dontComputeNew>true</dontComputeNew>
      <doNotResolveRelativePaths>true</doNotResolveRelativePaths>
      <includePattern/>
      <excludePattern/>
      <messagesPattern/>
      <parserConfigurations/>
      <consoleParsers>
      <hudson.plugins.warnings.ConsoleParser>
      <parserName>GNU C Compiler 4 (gcc)</parserName>
      </hudson.plugins.warnings.ConsoleParser>
      </consoleParsers>
    </hudson.plugins.warnings.WarningsPublisher>


Tweeking gazeboX template formatting

Tweeking gazeboX template
formatting

@(TEMPLATE(
    'snippet/add_generated_comment.Dockerfile.em',
    user_name=user_name,
    tag_name=tag_name,
    source_template_name=template_name,
    now_str=now_str,
))@
@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    template_packages=template_packages,
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
    base_image=base_image,
))@
MAINTAINER Nate Koenig nkoenig@@osrfoundation.org

RUN apt-get update && apt-get install -y \
    software-properties-common  \
    && rm -rf /var/lib/apt/lists/*

RUN apt-add-repository ppa:libccd-debs \
    && apt-add-repository ppa:fcl-debs \
    && apt-add-repository ppa:dartsim

@[if 'ros_packages' in locals()]@
@[if ros_packages]@
# ROS Setup ####################################################################
# setup environment
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8

# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116

# setup sources.list
RUN echo "deb http://packages.ros.org/ros/@os_name @os_code_name main" > /etc/apt/sources.list.d/ros-latest.list

# install bootstrap tools
ENV ROS_DISTRO @rosdistro_name
RUN apt-get update && apt-get install --no-install-recommends -y \
    python-rosdep \
    python-rosinstall \
    python-vcstools \
    && rm -rf /var/lib/apt/lists/*

# bootstrap rosdep
RUN rosdep init \
    && rosdep update

# install ros packages
RUN apt-get update && apt-get install -y \
    @(' \\\n    '.join(ros_packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'gazebo_packages' in locals()]@
@[if gazebo_packages]@
# Gazebo Setup #################################################################
# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys D2486D2DD83DB69272AFE98867170598AF249743

# setup sources.list
RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-latest.list

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(gazebo_packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'packages' in locals()]@
@[if packages]@
# install packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'pip_install' in locals()]@
@[if pip_install]@
# install python packages
RUN pip install \
    @(' \\\n    '.join(pip_install))@

@[end if]@
@[end if]@

@[if 'sources' in locals()]@
@[if sources]@
# clone source
RUN mkdir -p @(ws)/src
@(TEMPLATE(
    'snippet/clone_sources.Dockerfile.em',
    sources=sources,
    ws=ws,
))@

# build source
WORKDIR @(ws)
RUN catkin init \
    && catkin build \
    -vi \
    --cmake-args \
    @(' \\\n    '.join(cmake_args))@

@[end if]@
@[end if]@

# setup environment
EXPOSE 11345

@[if 'entrypoint_name' in locals()]@
@[if entrypoint_name]@
@{
entrypoint_file = entrypoint_name.split('/')[-1]
}@
# setup entrypoint
COPY ./@entrypoint_file /

ENTRYPOINT ["/@entrypoint_file"]
@[end if]@
@[end if]@
@{
cmds = [
'gzserver',
]
}@
CMD ["@(' && '.join(cmds))"]

@(TEMPLATE(
    'snippet/add_generated_comment.Dockerfile.em',
    user_name=user_name,
    tag_name=tag_name,
    source_template_name=template_name,
    now_str=now_str,
))@
@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    template_packages=template_packages,
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
    base_image=base_image,
))@
MAINTAINER Nate Koenig nkoenig@@osrfoundation.org

RUN apt-get update && apt-get install -y \
    software-properties-common  \
    && rm -rf /var/lib/apt/lists/*

RUN apt-add-repository ppa:libccd-debs \
    && apt-add-repository ppa:fcl-debs \
    && apt-add-repository ppa:dartsim

@[if 'ros_packages' in locals()]@
@[if ros_packages]@
# ROS Setup ####################################################################
# setup environment
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8

# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116

# setup sources.list
RUN echo "deb http://packages.ros.org/ros/@os_name @os_code_name main" > /etc/apt/sources.list.d/ros-latest.list

# install bootstrap tools
ENV ROS_DISTRO @rosdistro_name
RUN apt-get update && apt-get install --no-install-recommends -y \
    python-rosdep \
    python-rosinstall \
    python-vcstools \
    && rm -rf /var/lib/apt/lists/*

# bootstrap rosdep
RUN rosdep init \
    && rosdep update

# install ros packages
RUN apt-get update && apt-get install -y \
    @(' \\\n    '.join(ros_packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'gazebo_packages' in locals()]@
@[if gazebo_packages]@
# Gazebo Setup #################################################################
# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys D2486D2DD83DB69272AFE98867170598AF249743

# setup sources.list
RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-latest.list

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(gazebo_packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'packages' in locals()]@
@[if packages]@
# install packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'pip_install' in locals()]@
@[if pip_install]@
# install python packages
RUN pip install \
    @(' \\\n    '.join(pip_install))@
@[end if]@
@[end if]@

@[if 'sources' in locals()]@
@[if sources]@
# clone source
ENV WS @(ws)
RUN mkdir -p @(ws)/src
@(TEMPLATE(
    'snippet/clone_sources.Dockerfile.em',
    sources=sources,
    ws=ws,
))@

# build source
RUN cd @(ws) \
    && catkin init \
    && catkin build \
    -vi \
    --cmake-args \
    @(' \\\n    '.join(cmake_args))@
@[end if]@
@[end if]@

# setup environment
EXPOSE 11345

@[if 'entrypoint_name' in locals()]@
@[if entrypoint_name]@
@{
entrypoint_file = entrypoint_name.split('/')[-1]
}@
# setup entrypoint
COPY ./@entrypoint_file /

ENTRYPOINT ["/@entrypoint_file"]
@[end if]@
@[end if]@
@{
cmds = [
'gzserver',
]
}@
CMD ["@(' && '.join(cmds))"]


add an argument to the import_upstream job to support test runs Fixes #128

<project>
  <actions/>
  <description>Generated at @ESCAPE(now_str) from template '@ESCAPE(template_name)'</description>
@(SNIPPET(
    'log-rotator',
    days_to_keep=100,
    num_to_keep=100,
))@
  <keepDependencies>false</keepDependencies>
  <properties>
@(SNIPPET(
    'property_parameters-definition',
    parameters=[
        {
            'type': 'string',
            'name': 'config_file',
            'description': 'A specific yaml file or files. If unset it defaults to aggregating all yaml files in the directory defined by upstream_config in reprepro-updater.ini',
        },
    ],
))@
@(SNIPPET(
    'property_parameters-definition',
    parameters=[
        {
            'type': 'boolean',
            'name': 'EXECUTE_IMPORT',
            'description': 'If this is not true, it will only do a dry run and print the expect import but not execute.',
            'default_value': 'false',
        },
    ],
))@
@(SNIPPET(
    'property_requeue-job',
))@
  </properties>
@(SNIPPET(
    'scm_git',
    url='https://github.com/ros-infrastructure/reprepro-updater.git',
    branch_name='refactor',
    relative_target_dir='reprepro-updater',
    refspec=None,
))@
  <assignedNode>building_repository</assignedNode>
  <canRoam>false</canRoam>
  <disabled>false</disabled>
  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
  <triggers/>
  <concurrentBuild>false</concurrentBuild>
  <builders>
@(SNIPPET(
    'builder_shell',
    script='\n'.join([
        'echo "# BEGIN SECTION: import debian packages"',
        'if [ "$EXECUTE_IMPORT" = "true" ]; then',
        '  export COMMIT_ARG="--commit"',
        'fi',
        'export PYTHONPATH=$WORKSPACE/reprepro-updater/src:$PYTHONPATH',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_building"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_building $config_file $COMMIT_ARG',
        'echo "# END SUBSECTION"',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_testing"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_testing $config_file $COMMIT_ARG',
        'echo "# END SUBSECTION"',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_main"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_main $config_file $COMMIT_ARG',
        'echo "# END SUBSECTION"',
        'echo "# END SECTION"',
    ]),
))@
  </builders>
  <publishers>
@(SNIPPET(
    'publisher_mailer',
    recipients=recipients,
    dynamic_recipients=[],
    send_to_individuals=False,
))@
  </publishers>
  <buildWrappers>
@(SNIPPET(
    'build-wrapper_timestamper',
))@
  </buildWrappers>
</project>

<project>
  <actions/>
  <description>Generated at @ESCAPE(now_str) from template '@ESCAPE(template_name)'</description>
@(SNIPPET(
    'log-rotator',
    days_to_keep=100,
    num_to_keep=100,
))@
  <keepDependencies>false</keepDependencies>
  <properties>
@(SNIPPET(
    'property_parameters-definition',
    parameters=[
        {
            'type': 'string',
            'name': 'config_file',
            'description': 'A specific yaml file or files. If unset it defaults to aggregating all yaml files in the directory defined by upstream_config in reprepro-updater.ini',
        },
    ],
))@
@(SNIPPET(
    'property_requeue-job',
))@
  </properties>
@(SNIPPET(
    'scm_git',
    url='https://github.com/ros-infrastructure/reprepro-updater.git',
    branch_name='refactor',
    relative_target_dir='reprepro-updater',
    refspec=None,
))@
  <assignedNode>building_repository</assignedNode>
  <canRoam>false</canRoam>
  <disabled>false</disabled>
  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
  <triggers/>
  <concurrentBuild>false</concurrentBuild>
  <builders>
@(SNIPPET(
    'builder_shell',
    script='\n'.join([
        'echo "# BEGIN SECTION: import debian packages"',
        'export PYTHONPATH=$WORKSPACE/reprepro-updater/src:$PYTHONPATH',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_building"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_building $config_file --commit',
        'echo "# END SUBSECTION"',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_testing"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_testing $config_file --commit',
        'echo "# END SUBSECTION"',
        'echo "# BEGIN SUBSECTION: import debian packages for ubuntu_main"',
        'python -u $WORKSPACE/reprepro-updater/scripts/import_upstream.py ubuntu_main $config_file --commit',
        'echo "# END SUBSECTION"',
        'echo "# END SECTION"',
    ]),
))@
  </builders>
  <publishers>
@(SNIPPET(
    'publisher_mailer',
    recipients=recipients,
    dynamic_recipients=[],
    send_to_individuals=False,
))@
  </publishers>
  <buildWrappers>
@(SNIPPET(
    'build-wrapper_timestamper',
))@
  </buildWrappers>
</project>


Shifting set of locales to after package install

and adding LC_ALL evn as well

@(TEMPLATE(
    'snippet/add_generated_comment.Dockerfile.em',
    user_name=user_name,
    tag_name=tag_name,
    source_template_name=template_name,
    now_str=now_str,
))@
@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    template_packages=template_packages,
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
    base_image=base_image,
    maintainer_name=maintainer_name,
))@

# ROS1 Repo Setup ##############################################################
# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116

# setup sources.list
RUN echo "deb http://packages.ros.org/ros/@os_name @os_code_name main" > /etc/apt/sources.list.d/ros-latest.list

# OSRF Repo Setup ##############################################################
# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys D2486D2DD83DB69272AFE98867170598AF249743

# setup sources.list
RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable @os_code_name main" > /etc/apt/sources.list.d/gazebo-latest.list

# ROS2 Setup ###################################################################
@[if 'packages' in locals()]@
@[if packages]@
# install packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'pip3_install' in locals()]@
@[if pip3_install]@
# install python packages
RUN pip3 install -U \
    @(' \\\n    '.join(pip3_install))@

@[end if]@
@[end if]@

@[if 'vcs' in locals()]@
@[if vcs]@
# setup environment
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8

# clone source
ENV WS @(ws)
RUN mkdir -p @(ws)
@(TEMPLATE(
    'snippet/vcs_import.Dockerfile.em',
    vcs=vcs,
    ws=ws,
))@
@[end if]@
@[end if]@

@[if 'ament_args' in locals()]@
@[if ament_args]@
# build source
WORKDIR @(ws)/..
RUN src/ament/ament_tools/scripts/ament.py \
    @(' \\\n    '.join(ament_args))@

@[end if]@
@[end if]@

@[if 'entrypoint_name' in locals()]@
@[if entrypoint_name]@
@{
entrypoint_file = entrypoint_name.split('/')[-1]
}@
# setup entrypoint
COPY ./@entrypoint_file /

ENTRYPOINT ["/@entrypoint_file"]
@[end if]@
@[end if]@
@{
cmds = [
'bash',
]
}@
CMD ["@(' && '.join(cmds))"]

@(TEMPLATE(
    'snippet/add_generated_comment.Dockerfile.em',
    user_name=user_name,
    tag_name=tag_name,
    source_template_name=template_name,
    now_str=now_str,
))@
@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    template_packages=template_packages,
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
    base_image=base_image,
    maintainer_name=maintainer_name,
))@

# ROS1 Repo Setup ##############################################################
# setup environment
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8

# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116

# setup sources.list
RUN echo "deb http://packages.ros.org/ros/@os_name @os_code_name main" > /etc/apt/sources.list.d/ros-latest.list

# OSRF Repo Setup ##############################################################
# setup keys
RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys D2486D2DD83DB69272AFE98867170598AF249743

# setup sources.list
RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable @os_code_name main" > /etc/apt/sources.list.d/gazebo-latest.list

# ROS2 Setup ###################################################################
@[if 'packages' in locals()]@
@[if packages]@
# install packages
RUN apt-get update && apt-get install -q -y \
    @(' \\\n    '.join(packages))@  \
    && rm -rf /var/lib/apt/lists/*
@[end if]@
@[end if]@

@[if 'pip3_install' in locals()]@
@[if pip3_install]@
# install python packages
RUN pip3 install -U \
    @(' \\\n    '.join(pip3_install))@

@[end if]@
@[end if]@

@[if 'vcs' in locals()]@
@[if vcs]@
# clone source
ENV WS @(ws)
RUN mkdir -p @(ws)
@(TEMPLATE(
    'snippet/vcs_import.Dockerfile.em',
    vcs=vcs,
    ws=ws,
))@
@[end if]@
@[end if]@

@[if 'ament_args' in locals()]@
@[if ament_args]@
# build source
WORKDIR @(ws)/..
RUN src/ament/ament_tools/scripts/ament.py \
    @(' \\\n    '.join(ament_args))@

@[end if]@
@[end if]@

@[if 'entrypoint_name' in locals()]@
@[if entrypoint_name]@
@{
entrypoint_file = entrypoint_name.split('/')[-1]
}@
# setup entrypoint
COPY ./@entrypoint_file /

ENTRYPOINT ["/@entrypoint_file"]
@[end if]@
@[end if]@
@{
cmds = [
'bash',
]
}@
CMD ["@(' && '.join(cmds))"]


Enable RPM debug packages (#829)

Enable RPM debug packages (#829)

Now that the pulp infrastructure can handle the separate debug packages,
enable RPM debuginfo extraction. This should reduce the install size
substantially.

include('/etc/mock/default.cfg')

# Change the root name since we're modifying the chroot
config_opts['root'] += '-ros-buildfarm'

# Disable mock bootstrapping
config_opts['use_bootstrap'] = False

# Add python3-rpm-macros to resolve %{python3_pkgversion} in rosdep rules
config_opts['chroot_setup_cmd'] += ' python3-rpm-macros'

# Install weak dependencies to get group members
config_opts[f'{config_opts.package_manager}_builddep_opts'] = config_opts.get(f'{config_opts.package_manager}_builddep_opts', []) + ['--setopt=install_weak_deps=True']

@[if env_vars]@
# Set environment vars from the build config
@[for env_key, env_val in env_vars.items()]@
config_opts['environment']['@env_key'] = '@env_val'
@[end for]
@[end if]@
# Make debuginfo/debugsource packages best-effort
config_opts['macros']['%_empty_manifest_terminate_build'] = '%{nil}'
config_opts['macros']['%_missing_build_ids_terminate_build'] = '%{nil}'

# Hack the %{dist} macro to allow release suffixing
config_opts['macros']['%dist'] = '.' + config_opts['dist'] + '%{?dist_suffix}'

# Required for running mock in Docker
config_opts['use_nspawn'] = False

@[if os_name in ['centos', 'rhel'] and os_code_name == '7']@
# Inject g++ 8 into RHEL 7 builds
config_opts['chroot_setup_cmd'] += ' devtoolset-8-gcc-c++ devtoolset-8-make-nonblocking'
config_opts['macros']['%_buildshell'] = '/usr/bin/scl enable devtoolset-8 -- /bin/sh'

# Disable weak dependencies on RHEL 7 builds
config_opts['macros']['%_without_weak_deps'] = '1'
@[else]@
# Add g++, which is an assumed dependency in ROS
config_opts['chroot_setup_cmd'] += ' gcc-c++ make'
@[end if]@

config_opts[f'{config_opts.package_manager}.conf'] += """
@[for i, url in enumerate(distribution_repository_urls)]@
[ros-buildfarm-@(i)]
name=ROS Buildfarm Repository @(i) - $basearch
baseurl=@(url)
gpgkey=file:///etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i)
repo_gpgcheck=@(1 if i < len(distribution_repository_keys) and distribution_repository_keys[i] else 0)
gpgcheck=0
enabled=1
skip_if_unavailable=False

@[end for]@
@[if target_repository]@
[ros-buildfarm-target-source]
name=ROS Buildfarm Repository Target - SRPMS
baseurl=@(target_repository)
repo_gpgcheck=0
gpgcheck=0
enabled=0
skip_if_unavailable=False

@[end if]@
"""

include('/etc/mock/default.cfg')

# Change the root name since we're modifying the chroot
config_opts['root'] += '-ros-buildfarm'

# Disable mock bootstrapping
config_opts['use_bootstrap'] = False

# Add python3-rpm-macros to resolve %{python3_pkgversion} in rosdep rules
config_opts['chroot_setup_cmd'] += ' python3-rpm-macros'

# Install weak dependencies to get group members
config_opts[f'{config_opts.package_manager}_builddep_opts'] = config_opts.get(f'{config_opts.package_manager}_builddep_opts', []) + ['--setopt=install_weak_deps=True']

@[if env_vars]@
# Set environment vars from the build config
@[for env_key, env_val in env_vars.items()]@
config_opts['environment']['@env_key'] = '@env_val'
@[end for]
@[end if]@
# Disable debug packages until infrastructure can handle it
config_opts['macros']['%debug_package'] = '%{nil}'

# Hack the %{dist} macro to allow release suffixing
config_opts['macros']['%dist'] = '.' + config_opts['dist'] + '%{?dist_suffix}'

# Required for running mock in Docker
config_opts['use_nspawn'] = False

@[if os_name in ['centos', 'rhel'] and os_code_name == '7']@
# Inject g++ 8 into RHEL 7 builds
config_opts['chroot_setup_cmd'] += ' devtoolset-8-gcc-c++ devtoolset-8-make-nonblocking'
config_opts['macros']['%_buildshell'] = '/usr/bin/scl enable devtoolset-8 -- /bin/sh'

# Disable weak dependencies on RHEL 7 builds
config_opts['macros']['%_without_weak_deps'] = '1'
@[else]@
# Add g++, which is an assumed dependency in ROS
config_opts['chroot_setup_cmd'] += ' gcc-c++ make'
@[end if]@

config_opts[f'{config_opts.package_manager}.conf'] += """
@[for i, url in enumerate(distribution_repository_urls)]@
[ros-buildfarm-@(i)]
name=ROS Buildfarm Repository @(i) - $basearch
baseurl=@(url)
gpgkey=file:///etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i)
repo_gpgcheck=@(1 if i < len(distribution_repository_keys) and distribution_repository_keys[i] else 0)
gpgcheck=0
enabled=1
skip_if_unavailable=False

@[end for]@
@[if target_repository]@
[ros-buildfarm-target-source]
name=ROS Buildfarm Repository Target - SRPMS
baseurl=@(target_repository)
repo_gpgcheck=0
gpgcheck=0
enabled=0
skip_if_unavailable=False

@[end if]@
"""


fixing ordering lost in review

fixing ordering lost in review

# generated from @template_name

@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
))@
MAINTAINER Dirk Thomas dthomas+buildfarm@@osrfoundation.org

VOLUME ["/var/cache/apt/archives"]

ENV DEBIAN_FRONTEND noninteractive
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8

RUN useradd -u @uid -m buildfarm

@(TEMPLATE(
    'snippet/old_release_set.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
))@

@(TEMPLATE(
    'snippet/add_distribution_repositories.Dockerfile.em',
    distribution_repository_keys=distribution_repository_keys,
    distribution_repository_urls=distribution_repository_urls,
    os_code_name=os_code_name,
    add_source=False,
))@

@(TEMPLATE(
    'snippet/add_wrapper_scripts.Dockerfile.em',
    wrapper_scripts=wrapper_scripts,
))@

# automatic invalidation once every day
RUN echo "@today_str"

@(TEMPLATE(
    'snippet/install_python3.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
))@

RUN python3 -u /tmp/wrapper_scripts/apt-get.py update-and-install -q -y ccache

@(TEMPLATE(
    'snippet/install_dependencies.Dockerfile.em',
    dependencies=dependencies,
    dependency_versions=dependency_versions,
))@

USER buildfarm
ENTRYPOINT ["sh", "-c"]
@{
cmd = \
    'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH' + \
    ' PATH=/usr/lib/ccache:$PATH' + \
    ' python3 -u' + \
    ' /tmp/ros_buildfarm/scripts/release/build_binarydeb.py' + \
    ' ' + rosdistro_name + \
    ' ' + package_name + \
    ' --sourcedeb-dir ' + binarydeb_dir
}@
CMD ["@cmd"]

# generated from @template_name

@(TEMPLATE(
    'snippet/from_base_image.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
    arch=arch,
))@
MAINTAINER Dirk Thomas dthomas+buildfarm@@osrfoundation.org

VOLUME ["/var/cache/apt/archives"]

ENV DEBIAN_FRONTEND noninteractive
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8

RUN useradd -u @uid -m buildfarm

@(TEMPLATE(
    'snippet/old_release_set.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
))@

@(TEMPLATE(
    'snippet/add_distribution_repositories.Dockerfile.em',
    distribution_repository_keys=distribution_repository_keys,
    distribution_repository_urls=distribution_repository_urls,
    os_code_name=os_code_name,
    add_source=False,
))@

@(TEMPLATE(
    'snippet/add_wrapper_scripts.Dockerfile.em',
    wrapper_scripts=wrapper_scripts,
))@

# automatic invalidation once every day
RUN echo "@today_str"

@(TEMPLATE(
    'snippet/install_python3.Dockerfile.em',
    os_name=os_name,
    os_code_name=os_code_name,
))@

RUN python3 -u /tmp/wrapper_scripts/apt-get.py update-and-install -q -y ccache

@(TEMPLATE(
    'snippet/install_dependencies.Dockerfile.em',
    dependencies=dependencies,
    dependency_versions=dependency_versions,
))@

USER buildfarm
ENTRYPOINT ["sh", "-c"]
@{
cmd = \
    'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH python3 -u' + \
    ' PATH=/usr/lib/ccache:$PATH' + \
    ' /tmp/ros_buildfarm/scripts/release/build_binarydeb.py' + \
    ' ' + rosdistro_name + \
    ' ' + package_name + \
    ' --sourcedeb-dir ' + binarydeb_dir
}@
CMD ["@cmd"]