text: /*
   * Copyright (c) 2013-2019 Triad National Security, LLC
   *                         All rights reserved.
   *
   * This file is part of the libquo project. See the LICENSE file at the
   * top-level directory of this distribution.
   */
  
  #include "quo.h"
  
  #include <stdio.h>
  #include <stdlib.h>
  #include <stdbool.h>
  #include <sys/types.h>
  #include <unistd.h>
  
  #include "mpi.h"
  
  /**
   * another quo demo code that shows how one might implement binding policies
   * based on node process information, hardware info, and code policy.
   */
  
  typedef struct context_t {
      /* my host's name */
      char hostname[MPI_MAX_PROCESSOR_NAME];
      /* my rank */
      int rank;
      /* number of ranks in MPI_COMM_WORLD */
      int nranks;
      /* number of nodes in our job */
      int nnodes;
      /* number of numa nodes on this node */
      int nnuma;
      /* number of ranks that share this node with me (includes myself) */
      int nnoderanks;
      /* my node rank */
      int noderank;
      /* whether or not mpi is initialized */
      bool mpi_inited;
      /* number of sockets on the node */
      int nsockets;
      /* number of cores on the node */
      int ncores;
      /* number of pus (processing units - e.g hw threads) */
      int npus;
      /* quo major version */
      int qv;
      /* quo minor version */
      int qsv;
      /* pointer to initial stringification of our cpuset */
      char *cbindstr;
      /* flag indicating whether or not we are initially bound */
      int bound;
      /* flag indicating whether or not i've pushed a policy that hasn't been
       * popped yet. this is hacky, but illustrates a nice point. */
      bool pushed_policy;
      /* our quo context (the thing that gets passed around all over the place).
       * filler words that make this comment line look mo better... */
      QUO_context quo;
  } context_t;
  
  static const char *
  stringify_type(QUO_obj_type_t typ)
  {
      switch (typ) {
          case QUO_OBJ_MACHINE: return "machine";
          case QUO_OBJ_NUMANODE: return "numa node";
          case QUO_OBJ_SOCKET: return "socket";
          case QUO_OBJ_CORE: return "core";
          case QUO_OBJ_PU: return "processing unit";
          default: return "???";
      }
      return "???";
  }
  
  /**
   * rudimentary "pretty print" routine. not needed in real life...
   */
  static inline void
  demo_emit_sync(const context_t *c)
  {
      MPI_Barrier(MPI_COMM_WORLD);
      usleep((c->rank) * 5000);
  }
  
  static int
  fini(context_t *c)
  {
      if (!c) return 1;
  
      int nerrs = 0;
      if (QUO_SUCCESS != QUO_free(c->quo)) nerrs++;
      /* finalize mpi AFTER QUO_destruct - we may mpi in our destruct */
      if (c->mpi_inited) MPI_Finalize();
      if (c->cbindstr) free(c->cbindstr);
      free(c);
  
      return (nerrs ? 1 : 0);
  }
  
  /**
   * i'm being really sloppy here. ideally, one should probably save the rc and
   * then display or do some other cool thing with it. don't be like this code. if
   * QUO_construct or QUO_init fail, then someone using this could just continue
   * without the awesomeness that is libquo. they cleanup after themselves, so
   * things *should* be in an okay state after an early failure. the failures may
   * be part of a larger problem, however. */
  static int
  init(context_t **c)
  {
      int name_len = 0;
      context_t *newc = NULL;
      /* alloc our context */
      if (NULL == (newc = calloc(1, sizeof(*newc)))) goto err;
      /* libquo requires that MPI be initialized before its init is called */
      if (MPI_SUCCESS != MPI_Init(NULL, NULL)) goto err;
      /* gather some basic job info from our mpi lib */
      if (MPI_SUCCESS != MPI_Comm_size(MPI_COMM_WORLD, &(newc->nranks))) goto err;
      /* ...and more */
      if (MPI_SUCCESS != MPI_Comm_rank(MPI_COMM_WORLD, &(newc->rank))) goto err;
      /* stash my host's name */
      if (MPI_SUCCESS != MPI_Get_processor_name(newc->hostname, &name_len)) {
          goto err;
      }
      /* can be called at any point -- even before init and construct. */
      if (QUO_SUCCESS != QUO_version(&(newc->qv), &(newc->qsv))) goto err;
      /* relatively expensive call. you only really want to do this once at the
       * beginning of time and pass the context all over the place within your
       * code. */
      if (QUO_SUCCESS != QUO_create(&newc->quo, MPI_COMM_WORLD)) goto err;
      newc->mpi_inited = true;
      *c = newc;
      return 0;
  err:
      (void)fini(newc);
      return 1;
  }
  
  /**
   * gather system and job info from libquo.
   */
  static int
  sys_grok(context_t *c)
  {
      char *bad_func = NULL;
  
      /* this interface is more powerful, but the other n* calls can be more
       * convenient. at any rate, this is an example of the
       * QUO_nobjs_in_type_by_type interface to get the number of sockets on
       * the machine. note: you can also use the QUO_nsockets or
       * QUO_nobjs_by_type to get the same info. */
      if (QUO_SUCCESS != QUO_nobjs_in_type_by_type(c->quo,
                                                   QUO_OBJ_MACHINE,
                                                   0,
                                                   QUO_OBJ_SOCKET,
                                                   &c->nsockets)) {
          bad_func = "QUO_nobjs_in_type_by_type";
          goto out;
      }
      if (QUO_SUCCESS != QUO_nobjs_in_type_by_type(c->quo,
                                                   QUO_OBJ_MACHINE,
                                                   0,
                                                   QUO_OBJ_NUMANODE,
                                                   &c->nnuma)) {
          bad_func = "QUO_nobjs_in_type_by_type";
          goto out;
      }
      if (QUO_SUCCESS != QUO_ncores(c->quo, &c->ncores)) {
          bad_func = "QUO_ncores";
          goto out;
      }
      if (QUO_SUCCESS != QUO_npus(c->quo, &c->npus)) {
          bad_func = "QUO_npus";
          goto out;
      }
      if (QUO_SUCCESS != QUO_bound(c->quo, &c->bound)) {
          bad_func = "QUO_bound";
          goto out;
      }
      if (QUO_SUCCESS != QUO_stringify_cbind(c->quo, &c->cbindstr)) {
          bad_func = "QUO_stringify_cbind";
          goto out;
      }
      if (QUO_SUCCESS != QUO_nnodes(c->quo, &c->nnodes)) {
          bad_func = "QUO_nnodes";
          goto out;
      }
      if (QUO_SUCCESS != QUO_nqids(c->quo, &c->nnoderanks)) {
          bad_func = "QUO_nqids";
          goto out;
      }
      if (QUO_SUCCESS != QUO_id(c->quo, &c->noderank)) {
          bad_func = "QUO_id";
          goto out;
      }
  out:
      if (bad_func) {
          fprintf(stderr, "%s: %s failure :-(\n", __func__, bad_func);
          return 1;
      }
      return 0;
  }
  
  static int
  emit_bind_state(const context_t *c)
  {
      char *cbindstr = NULL, *bad_func = NULL;
      int bound = 0;
  
      if (QUO_SUCCESS != QUO_stringify_cbind(c->quo, &cbindstr)) {
          bad_func = "QUO_stringify_cbind";
          goto out;
      }
      if (QUO_SUCCESS != QUO_bound(c->quo, &bound)) {
          bad_func = "QUO_bound";
          goto out;
      }
      printf("### process %d rank %d [%s] bound: %s\n",
             (int)getpid(), c->rank, cbindstr, bound ? "true" : "false");
      fflush(stdout);
  out:
      demo_emit_sync(c);
      if (cbindstr) free(cbindstr);
      if (bad_func) {
          fprintf(stderr, "%s: %s failure :-(\n", __func__, bad_func);
          return 1;
      }
      return 0;
  }
  
  static int
  emit_node_basics(const context_t *c)
  {
      /* one proc per node will emit this info */
      if (0 == c->noderank) {
          printf("### quo version: %d.%d ###\n", c->qv, c->qsv);
          printf("### nnodes: %d\n", c->nnodes);
          printf("### nnoderanks: %d\n", c->nnoderanks);
          printf("### nnumanodes: %d\n", c->nnuma);
          printf("### nsockets: %d\n", c->nsockets);
          printf("### ncores: %d\n", c->ncores);
          printf("### npus: %d\n", c->npus);
          fflush(stdout);
      }
      demo_emit_sync(c);
      return 0;
  }
  
  /**
   *
   */
  static int
  bindup_node(context_t *c)
  {
      /* choose any node rank here. i use node rank 0 (quid 0) because it's easy.
       * */
      if (0 == c->noderank) {
          printf("### [rank %d on %s] expanding my cpuset for threading!\n",
                  c->rank, c->hostname);
          /* QUO_BIND_PUSH_OBJ, so the last argument doesn't matter */
          int rc = QUO_bind_push(c->quo, QUO_BIND_PUSH_OBJ,
                                 QUO_OBJ_MACHINE, -1);
          if (QUO_SUCCESS != rc) {
              fprintf(stderr, "%s fails with rc: %d\n",
                      "QUO_bind_push", rc);
              return 1;
          }
          /* i pushed a policy */
          c->pushed_policy = true;
          demo_emit_sync(c);
      }
      else {
          printf("--- [rank %d on %s] going to sleep...\n",
                  c->rank, c->hostname);
          demo_emit_sync(c);
      }
      return 0;
  }
  
  /**
   * bind to what is provided. the assumption is that the binding policy for all
   * procs that are calling this are "binding up." this demo binds all ranks on a
   * node to the provided "this" that encloses the current policy. for example, if
   * i'm bound to a core and "bind up" to a socket, then i'll be bound to my
   * enclosing socket.
   */
  static int
  bindup_to_this(context_t *c,
                 QUO_obj_type_t this)
  {
      demo_emit_sync(c);
      printf("### [rank %d on %s] expanding my cpuset for threading!\n",
              c->rank, c->hostname);
      /* QUO_BIND_PUSH_OBJ, so the last argument doesn't matter. QUO will do smart
       * things here (ish). This is where how the things were launched matters. */
      int rc = QUO_bind_push(c->quo, QUO_BIND_PUSH_OBJ, this, -1);
      if (QUO_SUCCESS != rc) {
          fprintf(stderr, "%s fails with rc: %d\n",
                  "QUO_bind_push", rc);
          return 1;
      }
      c->pushed_policy = true;
      /* i pushed a policy */
      demo_emit_sync(c);
      return 0;
  }
  
  /**
   * we can only safely pop bindings that were pushed, so those who were elected
   * to be the socket master can now revert their binding by calling pop.
   */
  static int
  pop_bind_policy(context_t *c)
  {
      /* if i pushed a policy, then revert it. only one deep in our stack in this
       * demo, so that's why the bool works. more complicated binding stack
       * situations will require a bit more code. that is, quo let's you push an
       * arbitrary amount of binding policies, but this code only ever pushed one
       * at a time. */
      if (c->pushed_policy) {
          if (QUO_SUCCESS != QUO_bind_pop(c->quo)) {
              return 1;
          }
          c->pushed_policy = false;
      }
      return 0;
  }
  
  static int
  node_process_info(const context_t *c)
  {
      demo_emit_sync(c);
      /* if i'm node rank (from quo's perspective), then print out some info. */
      if (0 == c->noderank) {
          printf("### [rank %d] %04d mpi processes share this node: %s\n",
                c->rank, c->nnoderanks, c->hostname);
      }
      demo_emit_sync(c);
      return 0;
  }
  
  static int
  one_rank_all_res(context_t *context)
  {
      int erc = 0;
      char *bad_func = NULL;
      demo_emit_sync(context);
      /* for the first test, we will just use ALL the resources on the node for
       * threading and quiesce all but one mpi process on each node. NOTE: doing
       * so will show that the process is NOT bound -- because it isn't.  its
       * cpuset is the widest it can be on a particular server, so it is NOT
       * bound. */
      if (0 == context->rank) {
          fprintf(stdout,
                  "*****************************************\n"
                  "*** 1 rank given all node resources *****\n"
                  "*****************************************\n");
          fflush(stdout);
      }
      if (bindup_node(context)) {
          bad_func = "bindup_node";
          goto out;
      }
      if (emit_bind_state(context)) {
          bad_func = "emit_bind_state";
          goto out;
      }
      /* now revert the previous policy */
      if (pop_bind_policy(context)) {
          bad_func = "pop_bind_policy";
          goto out;
      }
      if (0 == context->rank) {
          fprintf(stdout, "reverted binding change...\n");
          fflush(stdout);
      }
      if (emit_bind_state(context)) {
          bad_func = "emit_bind_state";
          goto out;
      }
  out:
      if (bad_func) {
          fprintf(stderr, "XXX %s failure in: %s\n", __FILE__, bad_func);
          erc = 1;
      }
      return erc;
  }
  
  static int
  all_ranks_some_res(context_t *context)
  {
      int erc = 0;
      char *bad_func = NULL;
      demo_emit_sync(context);
      /* everyone has one of these -- default */
      QUO_obj_type_t what_to_bindup_to = QUO_OBJ_MACHINE;
      /* pick some resource to "bind up" to. */
      if (0 != context->nnuma) {
          what_to_bindup_to = QUO_OBJ_NUMANODE;
      }
      else if (0 != context->nsockets) {
          what_to_bindup_to = QUO_OBJ_SOCKET;
      }
      if (0 == context->rank) {
          fprintf(stdout,
                  "*****************************************\n"
                  "*** all ranks given some resources       \n"
                  "*** resource is: %s                      \n"
                  "*****************************************\n",
                  stringify_type(what_to_bindup_to));
          fflush(stdout);
      }
      if (bindup_to_this(context, what_to_bindup_to)) {
          bad_func = "bindup_node";
          goto out;
      }
      if (emit_bind_state(context)) {
          bad_func = "emit_bind_state";
          goto out;
      }
      /* now revert the previous policy */
      if (pop_bind_policy(context)) {
          bad_func = "pop_bind_policy";
          goto out;
      }
      if (0 == context->rank) {
          fprintf(stdout, "reverted binding change...\n");
          fflush(stdout);
      }
      if (emit_bind_state(context)) {
          bad_func = "emit_bind_state";
          goto out;
      }
  out:
      if (NULL != bad_func) {
          fprintf(stderr, "XXX %s failure in: %s\n", __FILE__, bad_func);
          erc = 1;
      }
      return erc;
  }
  
  int
  main(void)
  {
      int erc = EXIT_SUCCESS;
      char *bad_func = NULL;
      context_t *context = NULL;
      setbuf(stdout, NULL);
  
      /* ////////////////////////////////////////////////////////////////////// */
      /* init code */
      /* ////////////////////////////////////////////////////////////////////// */
      if (init(&context)) {
          bad_func = "init";
          goto out;
      }
      /* ////////////////////////////////////////////////////////////////////// */
      /* libquo is now ready for service */
      /* ////////////////////////////////////////////////////////////////////// */
  
      /* first gather some info so we can base our decisions on our current
       * situation. */
      if (sys_grok(context)) {
          bad_func = "sys_grok";
          goto out;
      }
      /* let the people know what's going on... one per node. this info is on a
       * per-node basis, folks... so, running on a set of heterogeneous nodes will
       * yeild different results. policy can be set on a per node basis. */
      if (emit_node_basics(context)) {
          bad_func = "emit_node_basics";
          goto out;
      }
      /* now show what the default bind state of each process in the job is */
      if (emit_bind_state(context)) {
          bad_func = "emit_bind_state";
          goto out;
      }
      /* ////////////////////////////////////////////////////////////////////// */
      /* ////////////////////////////////////////////////////////////////////// */
      /* this is where the real work begins ...                                 */
      /* ////////////////////////////////////////////////////////////////////// */
      /* ////////////////////////////////////////////////////////////////////// */
      demo_emit_sync(context);
      if (0 == context->rank) {
          fprintf(stdout,
                  "*****************************************\n"
                  "***      starting the demo pieces     ***\n"
                  "*****************************************\n");
          fflush(stdout);
      }
      demo_emit_sync(context);
      /* first emit how many mpi processes share a node with me.
       * NOTE: this is a per-node thing and the result may differ across nodes.
       * Processes that share a node will of course see the same answer, but the
       * answer may be different across nodes (i.e. servers/compute nodes). */
      if (node_process_info(context)) {
          bad_func = "node_process_info";
          goto out;
      }
      /* demo 0 */
      if (one_rank_all_res(context)) {
          bad_func = "one_rank_all_res";
          goto out;
      }
      /* demo 1 */
      if (all_ranks_some_res(context)) {
          bad_func = "all_ranks_some_res";
          goto out;
      }
  out:
      if (NULL != bad_func) {
          fprintf(stderr, "XXX %s failure in: %s\n", __FILE__, bad_func);
          erc = EXIT_FAILURE;
      }
      (void)fini(context);
      return erc;
  }
  