text: /*
   * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   *                         University Research and Technology
   *                         Corporation.  All rights reserved.
   * Copyright (c) 2004-2011 The University of Tennessee and The University
   *                         of Tennessee Research Foundation.  All rights
   *                         reserved.
   * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   *                         University of Stuttgart.  All rights reserved.
   * Copyright (c) 2004-2005 The Regents of the University of California.
   *                         All rights reserved.
   * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
   * Copyright (c) 2013-2014 Intel, Inc. All rights reserved.
   * $COPYRIGHT$
   *
   * Additional copyrights may follow
   *
   * $HEADER$
   */
  
  #include "orte_config.h"
  #include "orte/constants.h"
  
  #include <string.h>
  
  #include "opal/util/output.h"
  #include "opal/mca/hwloc/base/base.h"
  
  #include "orte/util/show_help.h"
  #include "orte/util/name_fns.h"
  #include "orte/runtime/orte_globals.h"
  #include "orte/mca/errmgr/errmgr.h"
  
  #include "orte/mca/rmaps/base/rmaps_private.h"
  #include "orte/mca/rmaps/base/base.h"
  #include "rmaps_rr.h"
  
  int orte_rmaps_rr_byslot(orte_job_t *jdata,
                           orte_app_context_t *app,
                           opal_list_t *node_list,
                           orte_std_cntr_t num_slots,
                           orte_vpid_t num_procs)
  {
      int rc, i, nprocs_mapped;
      orte_node_t *node;
      orte_proc_t *proc;
      int num_procs_to_assign, extra_procs_to_assign=0, nxtra_nodes=0;
  #if OPAL_HAVE_HWLOC
      hwloc_obj_t obj=NULL;
  #endif
      float balance;
      bool add_one=false;
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr: mapping by slot for job %s slots %d num_procs %lu",
                          ORTE_JOBID_PRINT(jdata->jobid), (int)num_slots, (unsigned long)num_procs);
  
      /* check to see if we can map all the procs */
      if (num_slots < ((int)app->num_procs * orte_rmaps_base.cpus_per_rank)) {
          if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
              orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
                             true, app->num_procs, app->app);
              return ORTE_ERR_SILENT;
          }
      }
  
      /* first pass: map the number of procs to each node until we
       * map all specified procs or use all allocated slots
       */
      nprocs_mapped = 0;
      OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr:slot working node %s",
                              node->name);
  #if OPAL_HAVE_HWLOC
          /* get the root object as we are not assigning
           * locale here except at the node level
           */
          if (NULL != node->topology) {
              obj = hwloc_get_root_obj(node->topology);
          }
  #endif
          if (node->slots <= node->slots_inuse) {
              opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                                  "mca:rmaps:rr:slot node %s is full - skipping",
                                  node->name);
              continue;
          }
          /* assign a number of procs equal to the number of available
           * slots divided by the number of cpus/rank the user
           * requested
           */
          num_procs_to_assign = (node->slots - node->slots_inuse) / orte_rmaps_base.cpus_per_rank;
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr:slot assigning %d procs to node %s",
                              (int)num_procs_to_assign, node->name);
  
          for (i=0; i < num_procs_to_assign && nprocs_mapped < app->num_procs; i++) {
              /* add this node to the map - do it only once */
              if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
                  if (ORTE_SUCCESS > (rc = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
                      ORTE_ERROR_LOG(rc);
                      return rc;
                  }
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
                  OBJ_RETAIN(node);  /* maintain accounting on object */
                  ++(jdata->map->num_nodes);
              }
              if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                  return ORTE_ERR_OUT_OF_RESOURCE;
              }
              nprocs_mapped++;
  #if OPAL_HAVE_HWLOC
              orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
  #endif
          }
      }
  
      if (nprocs_mapped == app->num_procs) {
          /* we are done */
          return ORTE_SUCCESS;
      }
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr:slot job %s is oversubscribed - performing second pass",
                          ORTE_JOBID_PRINT(jdata->jobid));
  
      /* second pass: if we haven't mapped everyone yet, it is
       * because we are oversubscribed. Figure out how many procs
       * to add
       */
      balance = (float)((int)app->num_procs - nprocs_mapped) / (float)opal_list_get_size(node_list);
      extra_procs_to_assign = (int)balance;
      if (0 < (balance - (float)extra_procs_to_assign)) {
          /* compute how many nodes need an extra proc */
          nxtra_nodes = app->num_procs - nprocs_mapped - (extra_procs_to_assign * opal_list_get_size(node_list));
          /* add one so that we add an extra proc to the first nodes
           * until all procs are mapped
           */
          extra_procs_to_assign++;
          /* flag that we added one */
          add_one = true;
      }
  
      OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr:slot working node %s",
                              node->name);
  #if OPAL_HAVE_HWLOC
          /* get the root object as we are not assigning
           * locale except at the node level
           */
          if (NULL != node->topology) {
              obj = hwloc_get_root_obj(node->topology);
          }
  #endif
          /* add this node to the map - do it only once */
          if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
              if (ORTE_SUCCESS > (rc = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
                  ORTE_ERROR_LOG(rc);
                  return rc;
              }
              ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
              OBJ_RETAIN(node);  /* maintain accounting on object */
              ++(jdata->map->num_nodes);
          }
          if (add_one) {
              if (0 == nxtra_nodes) {
                  --extra_procs_to_assign;
                  add_one = false;
              } else {
                  --nxtra_nodes;
              }
          }
          num_procs_to_assign = ((node->slots - node->slots_inuse)/orte_rmaps_base.cpus_per_rank) + extra_procs_to_assign;
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr:slot adding up to %d procs to node %s",
                              num_procs_to_assign, node->name);
          for (i=0; i < num_procs_to_assign && nprocs_mapped < app->num_procs; i++) {
              if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                  return ORTE_ERR_OUT_OF_RESOURCE;
              }
              nprocs_mapped++;
  #if OPAL_HAVE_HWLOC
              orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
  #endif
          }
          /* not all nodes are equal, so only set oversubscribed for
           * this node if it is in that state
           */
          if (node->slots < ((int)node->num_procs * orte_rmaps_base.cpus_per_rank)) {
              /* flag the node as oversubscribed so that sched-yield gets
               * properly set
               */
              ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
          }
          /* if we have mapped everything, then we are done */
          if (nprocs_mapped == app->num_procs) {
              break;
          }
      }
      return ORTE_SUCCESS;
  }
  
  int orte_rmaps_rr_bynode(orte_job_t *jdata,
                           orte_app_context_t *app,
                           opal_list_t *node_list,
                           orte_std_cntr_t num_slots,
                           orte_vpid_t num_procs)
  {
      int j, nprocs_mapped, nnodes;
      orte_node_t *node;
      orte_proc_t *proc;
      int num_procs_to_assign, navg, idx;
      int extra_procs_to_assign=0, nxtra_nodes=0;
  #if OPAL_HAVE_HWLOC
      hwloc_obj_t obj=NULL;
  #endif
      float balance;
      bool add_one=false;
      bool oversubscribed=false;
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr: mapping by node for job %s app %d slots %d num_procs %lu",
                          ORTE_JOBID_PRINT(jdata->jobid), (int)app->idx,
                          (int)num_slots, (unsigned long)num_procs);
  
      /* quick check to see if we can map all the procs */
      if (num_slots < ((int)app->num_procs * orte_rmaps_base.cpus_per_rank)) {
          if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
              orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
                             true, app->num_procs, app->app);
              return ORTE_ERR_SILENT;
          }
          oversubscribed = true;
      }
  
      nnodes = opal_list_get_size(node_list);
      nprocs_mapped = 0;
  
      do {
          /* divide the procs evenly across all nodes - this is the
           * average we have to maintain as we go, but we adjust
           * the number on each node to reflect its available slots.
           * Obviously, if all nodes have the same number of slots,
           * then the avg is what we get on each node - this is
           * the most common situation.
           */
          navg = ((int)app->num_procs - nprocs_mapped) / nnodes;
          if (0 == navg) {
              /* if there are less procs than nodes, we have to
               * place at least one/node
               */
              navg = 1;
          }
  
          /* compute how many extra procs to put on each node */
          balance = (float)(((int)app->num_procs - nprocs_mapped) - (navg * nnodes)) / (float)nnodes;
          extra_procs_to_assign = (int)balance;
          nxtra_nodes = 0;
          add_one = false;
          if (0 < (balance - (float)extra_procs_to_assign)) {
              /* compute how many nodes need an extra proc */
              nxtra_nodes = ((int)app->num_procs - nprocs_mapped) - ((navg + extra_procs_to_assign) * nnodes);
              /* add one so that we add an extra proc to the first nodes
               * until all procs are mapped
               */
              extra_procs_to_assign++;
              /* flag that we added one */
              add_one = true;
          }
  
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr: mapping by node navg %d extra_procs %d extra_nodes %d",
                              navg, extra_procs_to_assign, nxtra_nodes);
  
          nnodes = 0;
          OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
  #if OPAL_HAVE_HWLOC
              /* get the root object as we are not assigning
               * locale except at the node level
               */
              if (NULL != node->topology) {
                  obj = hwloc_get_root_obj(node->topology);
              }
  #endif
              /* add this node to the map, but only do so once */
              if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
                  if (ORTE_SUCCESS > (idx = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
                      ORTE_ERROR_LOG(idx);
                      return idx;
                  }
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
                  OBJ_RETAIN(node);  /* maintain accounting on object */
                  ++(jdata->map->num_nodes);
              }
              if (oversubscribed) {
                  /* compute the number of procs to go on this node */
                  if (add_one) {
                      if (0 == nxtra_nodes) {
                          --extra_procs_to_assign;
                          add_one = false;
                      } else {
                          --nxtra_nodes;
                      }
                  }
                  /* everybody just takes their share */
                  num_procs_to_assign = navg + extra_procs_to_assign;
              } else if (node->slots <= node->slots_inuse) {
                  /* since we are not oversubcribed, ignore this node */
                  continue;
              } else {
                  /* if we are not oversubscribed, then there are enough
                   * slots to handle all the procs. However, not every
                   * node will have the same number of slots, so we
                   * have to track how many procs to "shift" elsewhere
                   * to make up the difference
                   */
                  
                  /* compute the number of procs to go on this node */
                  if (add_one) {
                      if (0 == nxtra_nodes) {
                          --extra_procs_to_assign;
                          add_one = false;
                      } else {
                          --nxtra_nodes;
                      }
                  }
                  /* if slots < avg + extra (adjusted for cpus/proc), then try to take all */
                  if ((node->slots - node->slots_inuse) < ((navg + extra_procs_to_assign) * orte_rmaps_base.cpus_per_rank)) {
                      num_procs_to_assign = (node->slots - node->slots_inuse)/orte_rmaps_base.cpus_per_rank;
                      /* if we can't take any proc, skip following steps */
                      if (num_procs_to_assign == 0) {
                          continue;
                      }
                  } else {
                  /* take the avg + extra */
                      num_procs_to_assign = navg + extra_procs_to_assign;
                  }
                  OPAL_OUTPUT_VERBOSE((20, orte_rmaps_base_framework.framework_output,
                                       "%s NODE %s AVG %d ASSIGN %d EXTRA %d",
                                       ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name,
                                       navg, num_procs_to_assign, extra_procs_to_assign));
              }
              nnodes++; // track how many nodes remain available
              OPAL_OUTPUT_VERBOSE((20, orte_rmaps_base_framework.framework_output,
                                   "%s NODE %s ASSIGNING %d",
                                   ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name,
                                   num_procs_to_assign));
              for (j=0; j < num_procs_to_assign && nprocs_mapped < app->num_procs; j++) {
                  if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                      return ORTE_ERR_OUT_OF_RESOURCE;
                  }
                  nprocs_mapped++;
  #if OPAL_HAVE_HWLOC
              orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
  #endif
              }
              /* not all nodes are equal, so only set oversubscribed for
               * this node if it is in that state
               */
              if (node->slots < ((int)node->num_procs * orte_rmaps_base.cpus_per_rank)) {
                  /* flag the node as oversubscribed so that sched-yield gets
                   * properly set
                   */
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
              }
              if (nprocs_mapped == app->num_procs) {
                  /* we are done */
                  break;
              }
          }
      } while (nprocs_mapped < app->num_procs && 0 < nnodes);
  
      /* now fillin as required until fully mapped */
      while (nprocs_mapped < app->num_procs) {
          OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
  #if OPAL_HAVE_HWLOC
              /* get the root object as we are not assigning
               * locale except at the node level
               */
              if (NULL != node->topology) {
                  obj = hwloc_get_root_obj(node->topology);
              }
  #endif
             OPAL_OUTPUT_VERBOSE((20, orte_rmaps_base_framework.framework_output,
                                   "%s ADDING PROC TO NODE %s",
                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name));
              if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                  return ORTE_ERR_OUT_OF_RESOURCE;
              }
              nprocs_mapped++;
  #if OPAL_HAVE_HWLOC
              orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
  #endif
              /* not all nodes are equal, so only set oversubscribed for
               * this node if it is in that state
               */
              if (node->slots < ((int)node->num_procs * orte_rmaps_base.cpus_per_rank)) {
                  /* flag the node as oversubscribed so that sched-yield gets
                   * properly set
                   */
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
              }
              if (nprocs_mapped == app->num_procs) {
                  /* we are done */
                  break;
              }
          }
      }
  
      return ORTE_SUCCESS;
  }
  
  #if OPAL_HAVE_HWLOC
  static  int byobj_span(orte_job_t *jdata,
                         orte_app_context_t *app,
                         opal_list_t *node_list,
                         orte_std_cntr_t num_slots,
                         orte_vpid_t num_procs,
                         hwloc_obj_type_t target, unsigned cache_level);
  
  /* mapping by hwloc object looks a lot like mapping by node,
   * but has the added complication of possibly having different
   * numbers of objects on each node
   */
  int orte_rmaps_rr_byobj(orte_job_t *jdata,
                          orte_app_context_t *app,
                          opal_list_t *node_list,
                          orte_std_cntr_t num_slots,
                          orte_vpid_t num_procs,
                          hwloc_obj_type_t target, unsigned cache_level)
  {
      int i, nmapped, nprocs_mapped;
      orte_node_t *node;
      orte_proc_t *proc;
      int nprocs, start;
      int idx;
      hwloc_obj_t obj=NULL;
      unsigned int nobjs;
      bool add_one;
      bool second_pass;
  
      /* there are two modes for mapping by object: span and not-span. The
       * span mode essentially operates as if there was just a single
       * "super-node" in the system - i.e., it balances the load across
       * all objects of the indicated type regardless of their location.
       * In essence, it acts as if we placed one proc on each object, cycling
       * across all objects on all nodes, and then wrapped around to place
       * another proc on each object, doing so until all procs were placed.
       *
       * In contrast, the non-span mode operates similar to byslot mapping.
       * All slots on each node are filled, assigning each proc to an object
       * on that node in a balanced fashion, and then the mapper moves on
       * to the next node. Thus, procs tend to be "front loaded" onto the
       * list of nodes, as opposed to being "load balanced" in the span mode
       */
      if (ORTE_MAPPING_SPAN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
          return byobj_span(jdata, app, node_list, num_slots,
                            num_procs, target, cache_level);
      }
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr: mapping no-span by %s for job %s slots %d num_procs %lu",
                          hwloc_obj_type_string(target),
                          ORTE_JOBID_PRINT(jdata->jobid),
                          (int)num_slots, (unsigned long)num_procs);
  
      /* quick check to see if we can map all the procs */
      if (num_slots < (app->num_procs * orte_rmaps_base.cpus_per_rank)) {
          if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
              orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
                             true, app->num_procs, app->app);
              return ORTE_ERR_SILENT;
          }
      }
  
      /* we know we have enough slots, or that oversubscrption is allowed, so
       * start mapping procs onto objects, filling each object as we go until
       * all procs are mapped. If one pass doesn't catch all the required procs,
       * then loop thru the list again to handle the oversubscription
       */
      nprocs_mapped = 0;
      second_pass = false;
      do {
          add_one = false;
          OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
              if (NULL == node->topology) {
                  orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
                                 true, node->name);
                  return ORTE_ERR_SILENT;
              }
              start = 0;
              /* get the number of objects of this type on this node */
              nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target, cache_level, OPAL_HWLOC_AVAILABLE);
              if (0 == nobjs) {
                  continue;
              }
              opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                                  "mca:rmaps:rr: found %u %s objects on node %s",
                                  nobjs, hwloc_obj_type_string(target), node->name);
  
              /* compute the number of procs to go on this node */
              nprocs = (node->slots - node->slots_inuse) / orte_rmaps_base.cpus_per_rank;
              opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                                  "mca:rmaps:rr: calculated nprocs %d", nprocs);
              if (nprocs < 1) {
                  if (second_pass) {
                      /* already checked for oversubscription permission, so at least put
                       * one proc on it
                       */
                      nprocs = 1;
                      /* offset our starting object position to avoid always
                       * hitting the first one
                       */
                      start = node->num_procs % nobjs;
                  } else {
                      continue;
                  }
              }
              /* add this node to the map, if reqd */
              if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
                  if (ORTE_SUCCESS > (idx = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
                      ORTE_ERROR_LOG(idx);
                      return idx;
                  }
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
                  OBJ_RETAIN(node);  /* maintain accounting on object */
                  ++(jdata->map->num_nodes);
              }
              nmapped = 0;
              opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                                  "mca:rmaps:rr: assigning nprocs %d", nprocs);
              do {
                  /* loop through the number of objects */
                  for (i=0; i < (int)nobjs && nmapped < nprocs && nprocs_mapped < (int)app->num_procs; i++) {
                      opal_output_verbose(20, orte_rmaps_base_framework.framework_output,
                                          "mca:rmaps:rr: assigning proc to object %d", (i+start) % nobjs);
                      /* get the hwloc object */
                      if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology, target, cache_level, (i+start) % nobjs, OPAL_HWLOC_AVAILABLE))) {
                          ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
                          return ORTE_ERR_NOT_FOUND;
                      }
                      if (orte_rmaps_base.cpus_per_rank > (int)opal_hwloc_base_get_npus(node->topology, obj)) {
                          orte_show_help("help-orte-rmaps-base.txt", "mapping-too-low", true,
                                         orte_rmaps_base.cpus_per_rank, opal_hwloc_base_get_npus(node->topology, obj),
                                         orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
                          return ORTE_ERR_SILENT;
                      }
                      if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                          return ORTE_ERR_OUT_OF_RESOURCE;
                      }
                      nprocs_mapped++;
                      nmapped++;
                      orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
                  }
              } while (nmapped < nprocs && nprocs_mapped < (int)app->num_procs);
              add_one = true;
              /* not all nodes are equal, so only set oversubscribed for
               * this node if it is in that state
               */
              if (node->slots < (int)node->num_procs) {
                  /* flag the node as oversubscribed so that sched-yield gets
                   * properly set
                   */
                  ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
              }
              if (nprocs_mapped == app->num_procs) {
                  /* we are done */
                  break;
              }
          }
          second_pass = true;
      } while (add_one && nprocs_mapped < app->num_procs);
  
      if (nprocs_mapped < app->num_procs) {
          /* usually means there were no objects of the requested type */
          return ORTE_ERR_NOT_FOUND;
      }
  
      return ORTE_SUCCESS;
  }
  
  static int byobj_span(orte_job_t *jdata,
                        orte_app_context_t *app,
                        opal_list_t *node_list,
                        orte_std_cntr_t num_slots,
                        orte_vpid_t num_procs,
                        hwloc_obj_type_t target, unsigned cache_level)
  {
      int i, j, nprocs_mapped, navg;
      orte_node_t *node;
      orte_proc_t *proc;
      int nprocs, nxtra_objs;
      int idx;
      hwloc_obj_t obj=NULL;
      unsigned int nobjs;
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr: mapping span by %s for job %s slots %d num_procs %lu",
                          hwloc_obj_type_string(target),
                          ORTE_JOBID_PRINT(jdata->jobid),
                          (int)num_slots, (unsigned long)num_procs);
  
      /* quick check to see if we can map all the procs */
      if (num_slots < (int)app->num_procs * orte_rmaps_base.cpus_per_rank) {
          if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
              orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
                             true, app->num_procs, app->app);
              return ORTE_ERR_SILENT;
          }
      }
  
      /* we know we have enough slots, or that oversubscrption is allowed, so
       * next determine how many total objects we have to work with
       */
      nobjs = 0;
      OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
          if (NULL == node->topology) {
              orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
                             true, node->name);
              return ORTE_ERR_SILENT;
          }
          /* get the number of objects of this type on this node */
          nobjs += opal_hwloc_base_get_nbobjs_by_type(node->topology, target, cache_level, OPAL_HWLOC_AVAILABLE);
      }
  
      if (0 == nobjs) {
          return ORTE_ERR_NOT_FOUND;
      }
  
      /* divide the procs evenly across all objects */
      navg = app->num_procs / nobjs;
      if (0 == navg) {
          /* if there are less procs than objects, we have to
           * place at least one/obj
           */
          navg = 1;
      }
  
      /* compute how many objs need an extra proc */
      if (0 > (nxtra_objs = app->num_procs - (navg * nobjs))) {
          nxtra_objs = 0;
      }
  
      opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                          "mca:rmaps:rr: mapping by %s navg %d extra_objs %d",
                          hwloc_obj_type_string(target),
                          navg, nxtra_objs);
  
      nprocs_mapped = 0;
      OPAL_LIST_FOREACH(node, node_list, orte_node_t) {
          /* add this node to the map, if reqd */
          if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
              if (ORTE_SUCCESS > (idx = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
                  ORTE_ERROR_LOG(idx);
                  return idx;
              }
              ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
              OBJ_RETAIN(node);  /* maintain accounting on object */
              ++(jdata->map->num_nodes);
          }
          /* get the number of objects of this type on this node */
          nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target, cache_level, OPAL_HWLOC_AVAILABLE);
          opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
                              "mca:rmaps:rr:byobj: found %d objs on node %s", nobjs, node->name);
          /* loop through the number of objects */
          for (i=0; i < (int)nobjs && nprocs_mapped < (int)app->num_procs; i++) {
              /* get the hwloc object */
              if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology, target, cache_level, i, OPAL_HWLOC_AVAILABLE))) {
                  ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
                  return ORTE_ERR_NOT_FOUND;
              }
              if (orte_rmaps_base.cpus_per_rank > (int)opal_hwloc_base_get_npus(node->topology, obj)) {
                  orte_show_help("help-orte-rmaps-base.txt", "mapping-too-low", true,
                                 orte_rmaps_base.cpus_per_rank, opal_hwloc_base_get_npus(node->topology, obj),
                                 orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
                  return ORTE_ERR_SILENT;
              }
              /* determine how many to map */
              nprocs = navg;
              if (0 < nxtra_objs) {
                  nprocs++;
                  nxtra_objs--;
              }
              /* map the reqd number of procs */
              for (j=0; j < nprocs && nprocs_mapped < app->num_procs; j++) {
                  if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
                      return ORTE_ERR_OUT_OF_RESOURCE;
                  }
                  nprocs_mapped++;
                  orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
              }
              /* keep track of the node we last used */
              jdata->bookmark = node;
          }
          /* not all nodes are equal, so only set oversubscribed for
           * this node if it is in that state
           */
          if (node->slots < (int)node->num_procs * orte_rmaps_base.cpus_per_rank) {
              /* flag the node as oversubscribed so that sched-yield gets
               * properly set
               */
              ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
          }
          if (nprocs_mapped == app->num_procs) {
              /* we are done */
              break;
          }
      }
  
      return ORTE_SUCCESS;
  }
  #endif
  
  