@@ -284,6 +284,8 @@ void finalize_internal( const bool all_spaces = false )
       Kokkos::Cuda::impl_finalize();
 #endif
   }
+#else
+  (void)all_spaces;
 #endif
 
 #if defined( KOKKOS_ENABLE_ROCM )
/*
//@HEADER
// ************************************************************************
//
//                        Kokkos v. 2.0
//              Copyright (2014) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/

#include <Kokkos_Core.hpp>
#include <impl/Kokkos_Error.hpp>
#include <cctype>
#include <cstring>
#include <iostream>
#include <sstream>
#include <cstdlib>
#include <stack>
#include <cerrno>

//----------------------------------------------------------------------------

namespace {
bool g_is_initialized = false;
bool g_show_warnings = true;
std::stack<std::function<void()> > finalize_hooks;
}

namespace Kokkos { namespace Impl { namespace {

bool is_unsigned_int(const char* str)
{
  const size_t len = strlen (str);
  for (size_t i = 0; i < len; ++i) {
    if (! isdigit (str[i])) {
      return false;
    }
  }
  return true;
}
void initialize_internal(const InitArguments& args)
{
// This is an experimental setting
// For KNL in Flat mode this variable should be set, so that
// memkind allocates high bandwidth memory correctly.
#ifdef KOKKOS_ENABLE_HBWSPACE
setenv("MEMKIND_HBW_NODES", "1", 0);
#endif

  if (args.disable_warnings) {
    g_show_warnings = false;
  }

  // Protect declarations, to prevent "unused variable" warnings.
#if defined( KOKKOS_ENABLE_OPENMP ) || defined( KOKKOS_ENABLE_THREADS ) ||\
    defined( KOKKOS_ENABLE_OPENMPTARGET ) || defined ( KOKKOS_ENABLE_HPX )
  const int num_threads = args.num_threads;
#endif
#if defined( KOKKOS_ENABLE_THREADS ) || defined( KOKKOS_ENABLE_OPENMPTARGET )
  const int use_numa = args.num_numa;
#endif
#if defined( KOKKOS_ENABLE_CUDA ) || defined( KOKKOS_ENABLE_ROCM )
  int use_gpu = args.device_id;
  const int ndevices = args.ndevices;
  const int skip_device = args.skip_device;
  // if the exact device is not set, but ndevices was given, assign round-robin using on-node MPI rank
  if (use_gpu < 0 && ndevices >= 0) {
    auto local_rank_str = std::getenv("OMPI_COMM_WORLD_LOCAL_RANK"); //OpenMPI
    if (!local_rank_str) local_rank_str = std::getenv("MV2_COMM_WORLD_LOCAL_RANK"); //MVAPICH2
    if (!local_rank_str) local_rank_str = std::getenv("SLURM_LOCALID"); //SLURM
    if (local_rank_str) {
      auto local_rank = std::atoi(local_rank_str);
      use_gpu = local_rank % ndevices;
    } else {
      // user only gave us ndevices, but the MPI environment variable wasn't set.
      // start with GPU 0 at this point
      use_gpu = 0;
    }
    // shift assignments over by one so no one is assigned to "skip_device"
    if (use_gpu >= skip_device) ++use_gpu;
  }
#endif // defined( KOKKOS_ENABLE_CUDA )

#if defined( KOKKOS_ENABLE_OPENMP )
  if( std::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value ) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
    Kokkos::OpenMP::initialize(num_threads);
#else
    Kokkos::OpenMP::impl_initialize(num_threads);
#endif
  }
  else {
    //std::cout << "Kokkos::initialize() fyi: OpenMP enabled but not initialized" << std::endl ;
  }
#endif

#if defined( KOKKOS_ENABLE_THREADS )
  if( std::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value ) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
    if(num_threads>0) {
      if(use_numa>0) {
        Kokkos::Threads::initialize(num_threads,use_numa);
      }
      else {
        Kokkos::Threads::initialize(num_threads);
      }
    } else {
      Kokkos::Threads::initialize();
    }
#else
    if(num_threads>0) {
      if(use_numa>0) {
        Kokkos::Threads::impl_initialize(num_threads,use_numa);
      }
      else {
        Kokkos::Threads::impl_initialize(num_threads);
      }
    } else {
      Kokkos::Threads::impl_initialize();
    }
#endif
    //std::cout << "Kokkos::initialize() fyi: Pthread enabled and initialized" << std::endl ;
  }
  else {
    //std::cout << "Kokkos::initialize() fyi: Pthread enabled but not initialized" << std::endl ;
  }
#endif

#if defined( KOKKOS_ENABLE_HPX )
  if( std::is_same< Kokkos::Experimental::HPX , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Experimental::HPX , Kokkos::HostSpace::execution_space >::value ) {
      if(num_threads>0) {
        Kokkos::Experimental::HPX::impl_initialize(num_threads);
      } else {
        Kokkos::Experimental::HPX::impl_initialize();
      }
      //std::cout << "Kokkos::initialize() fyi: HPX enabled and initialized" << std::endl ;
  }
  else {
      //std::cout << "Kokkos::initialize() fyi: HPX enabled but not initialized" << std::endl ;
  }
#endif

#if defined( KOKKOS_ENABLE_SERIAL )
  // Prevent "unused variable" warning for 'args' input struct.  If
  // Serial::initialize() ever needs to take arguments from the input
  // struct, you may remove this line of code.
  (void) args;

  // Always initialize Serial if it is configure time enabled
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
  Kokkos::Serial::initialize();
#else
  Kokkos::Serial::impl_initialize();
#endif
#endif

#if defined( KOKKOS_ENABLE_OPENMPTARGET )
  if( Impl::is_same< Kokkos::Experimental::OpenMPTarget , Kokkos::DefaultExecutionSpace >::value ) {
    if(num_threads>0) {
      if(use_numa>0) {
        Kokkos::Experimental::OpenMPTarget::initialize(num_threads,use_numa);
      }
      else {
        Kokkos::Experimental::OpenMPTarget::initialize(num_threads);
      }
    } else {
      Kokkos::Experimental::OpenMPTarget::initialize();
    }
    //std::cout << "Kokkos::initialize() fyi: OpenMP enabled and initialized" << std::endl ;
  }
  else {
    //std::cout << "Kokkos::initialize() fyi: OpenMP enabled but not initialized" << std::endl ;
  }
#endif

#if defined( KOKKOS_ENABLE_CUDA )
  if( std::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value || 0 < use_gpu ) {
    if (use_gpu > -1) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
      Kokkos::Cuda::initialize( Kokkos::Cuda::SelectDevice( use_gpu ) );
#else
      Kokkos::Cuda::impl_initialize( Kokkos::Cuda::SelectDevice( use_gpu ) );
#endif
    }
    else {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
      Kokkos::Cuda::initialize();
#else
      Kokkos::Cuda::impl_initialize();
#endif
    }
    //std::cout << "Kokkos::initialize() fyi: Cuda enabled and initialized" << std::endl ;
  }
#endif

#if defined( KOKKOS_ENABLE_ROCM )
  if( std::is_same< Kokkos::Experimental::ROCm , Kokkos::DefaultExecutionSpace >::value || 0 < use_gpu ) {
    if (use_gpu > -1) {
      Kokkos::Experimental::ROCm::initialize( Kokkos::Experimental::ROCm::SelectDevice( use_gpu ) );
    }
    else {
      Kokkos::Experimental::ROCm::initialize();
    }
    std::cout << "Kokkos::initialize() fyi: ROCm enabled and initialized" << std::endl ;
  }
#endif

#if defined(KOKKOS_ENABLE_PROFILING)
    Kokkos::Profiling::initialize();
#endif
    g_is_initialized = true;
}

void finalize_internal( const bool all_spaces = false )
{

  typename decltype(finalize_hooks)::size_type  numSuccessfulCalls = 0;
  while(! finalize_hooks.empty()) {
    auto f = finalize_hooks.top();
    try {
      f();
    }
    catch(...) {
      std::cerr << "Kokkos::finalize: A finalize hook (set via "
        "Kokkos::push_finalize_hook) threw an exception that it did not catch."
        "  Per std::atexit rules, this results in std::terminate.  This is "
        "finalize hook number " << numSuccessfulCalls << " (1-based indexing) "
        "out of " << finalize_hooks.size() << " to call.  Remember that "
        "Kokkos::finalize calls finalize hooks in reverse order from how they "
        "were pushed." << std::endl;
      std::terminate();
    }
    finalize_hooks.pop();
    ++numSuccessfulCalls;
  }

#if defined(KOKKOS_ENABLE_PROFILING)
    Kokkos::Profiling::finalize();
#endif

#if defined( KOKKOS_ENABLE_CUDA )
  if( std::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value || all_spaces ) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
    if(Kokkos::Cuda::is_initialized())
      Kokkos::Cuda::finalize();
#else
    if(Kokkos::Cuda::impl_is_initialized())
      Kokkos::Cuda::impl_finalize();
#endif
  }
#endif

#if defined( KOKKOS_ENABLE_ROCM )
  if( std::is_same< Kokkos::Experimental::ROCm , Kokkos::DefaultExecutionSpace >::value || all_spaces ) {
    if(Kokkos::Experimental::ROCm::is_initialized())
      Kokkos::Experimental::ROCm::finalize();
  }
#endif

#if defined( KOKKOS_ENABLE_OPENMPTARGET )
  if( std::is_same< Kokkos::Experimental::OpenMPTarget , Kokkos::DefaultExecutionSpace >::value || all_spaces ) {
    if(Kokkos::Experimental::OpenMPTarget::is_initialized())
      Kokkos::Experimental::OpenMPTarget::finalize();
  }
#endif

#if defined( KOKKOS_ENABLE_OPENMP )
  if( std::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value ||
      all_spaces ) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
    if(Kokkos::OpenMP::is_initialized())
      Kokkos::OpenMP::finalize();
#else
    if(Kokkos::OpenMP::impl_is_initialized())
      Kokkos::OpenMP::impl_finalize();
#endif
  }
#endif

#if defined( KOKKOS_ENABLE_HPX )
  if( std::is_same< Kokkos::Experimental::HPX , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Experimental::HPX , Kokkos::HostSpace::execution_space >::value ||
      all_spaces ) {
    if(Kokkos::Experimental::HPX::impl_is_initialized())
      Kokkos::Experimental::HPX::impl_finalize();
  }
#endif

#if defined( KOKKOS_ENABLE_THREADS )
  if( std::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value ||
      all_spaces ) {
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
    if(Kokkos::Threads::is_initialized())
      Kokkos::Threads::finalize();
#else
    if(Kokkos::Threads::impl_is_initialized())
      Kokkos::Threads::impl_finalize();
#endif
  }
#endif

#if defined( KOKKOS_ENABLE_SERIAL )
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
  if(Kokkos::Serial::is_initialized())
    Kokkos::Serial::finalize();
#else
  if(Kokkos::Serial::impl_is_initialized())
    Kokkos::Serial::impl_finalize();
#endif
#endif

  g_is_initialized = false;
  g_show_warnings = true;
}

void fence_internal()
{

#if defined( KOKKOS_ENABLE_CUDA )
  if( std::is_same< Kokkos::Cuda , Kokkos::DefaultExecutionSpace >::value ) {
    Kokkos::Cuda::fence();
  }
#endif

#if defined( KOKKOS_ENABLE_ROCM )
  if( std::is_same< Kokkos::Experimental::ROCm , Kokkos::DefaultExecutionSpace >::value ) {
    Kokkos::Experimental::ROCm::fence();
  }
#endif

#if defined( KOKKOS_ENABLE_OPENMP )
  if( std::is_same< Kokkos::OpenMP , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::OpenMP , Kokkos::HostSpace::execution_space >::value ) {
    Kokkos::OpenMP::fence();
  }
#endif

#if defined( KOKKOS_ENABLE_HPX )
  Kokkos::Experimental::HPX::fence();
#endif

#if defined( KOKKOS_ENABLE_THREADS )
  if( std::is_same< Kokkos::Threads , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Threads , Kokkos::HostSpace::execution_space >::value ) {
    Kokkos::Threads::fence();
  }
#endif

#if defined( KOKKOS_ENABLE_SERIAL )
  if( std::is_same< Kokkos::Serial , Kokkos::DefaultExecutionSpace >::value ||
      std::is_same< Kokkos::Serial , Kokkos::HostSpace::execution_space >::value ) {
    Kokkos::Serial::fence();
  }
#endif

}

bool check_arg(char const* arg, char const* expected) {
  std::size_t arg_len = std::strlen(arg);
  std::size_t exp_len = std::strlen(expected);
  if (arg_len < exp_len) return false;
  if (std::strncmp(arg, expected, exp_len) != 0) return false;
  if (arg_len == exp_len) return true;
  /* if expected is "--threads", ignore "--threads-for-application"
     by checking this character          ---------^
     to see if it continues to make a longer name */
  if (std::isalnum(arg[exp_len]) || arg[exp_len] == '-' || arg[exp_len] == '_') {
    return false;
  }
  return true;
}

bool check_int_arg(char const* arg, char const* expected, int* value) {
  if (!check_arg(arg, expected)) return false;
  std::size_t arg_len = std::strlen(arg);
  std::size_t exp_len = std::strlen(expected);
  bool okay = true;
  if (arg_len == exp_len || arg[exp_len] != '=') okay = false;
  char const* number = arg + exp_len + 1;
  if (!Impl::is_unsigned_int(number) || strlen(number) == 0) okay = false;
  *value = std::atoi(number);
  if (!okay) {
    std::ostringstream ss;
    ss << "Error: expecting an '=INT' after command line argument '" << expected << "'";
    ss << ". Raised by Kokkos::initialize(int narg, char* argc[]).";
    Impl::throw_runtime_exception( ss.str() );
  }
  return true;
}

}

}} // namespace Kokkos::Impl::{unnamed}

//----------------------------------------------------------------------------

namespace Kokkos {

void initialize(int& narg, char* arg[])
{
    int num_threads = -1;
    int numa = -1;
    int device = -1;
    int ndevices=-1;
    int skip_device = 9999;
    bool disable_warnings = false;

    int kokkos_threads_found = 0;
    int kokkos_numa_found = 0;
    int kokkos_device_found = 0;
    int kokkos_ndevices_found = 0;

    int iarg = 0;

    while (iarg < narg) {
      if (Impl::check_int_arg(arg[iarg], "--kokkos-threads", &num_threads)) {
        for(int k=iarg;k<narg-1;k++) {
          arg[k] = arg[k+1];
        }
        kokkos_threads_found=1;
        narg--;
      } else if (!kokkos_threads_found && Impl::check_int_arg(arg[iarg], "--threads", &num_threads)) {
        iarg++;
      } else if (Impl::check_int_arg(arg[iarg], "--kokkos-numa", &numa)) {
        for(int k=iarg;k<narg-1;k++) {
          arg[k] = arg[k+1];
        }
        kokkos_numa_found=1;
        narg--;
      } else if (!kokkos_numa_found && Impl::check_int_arg(arg[iarg], "--numa", &numa)) {
        iarg++;
      } else if (Impl::check_int_arg(arg[iarg], "--kokkos-device", &device)) {
        for(int k=iarg;k<narg-1;k++) {
          arg[k] = arg[k+1];
        }
        kokkos_device_found=1;
        narg--;
      } else if (!kokkos_device_found && Impl::check_int_arg(arg[iarg], "--device", &device)) {
        iarg++;
      } else if (Impl::check_arg(arg[iarg], "--kokkos-ndevices") || Impl::check_arg(arg[iarg], "--ndevices")) {

        //Find the number of device (expecting --device=XX)
        if (!((strncmp(arg[iarg],"--kokkos-ndevices=",18) == 0) || (strncmp(arg[iarg],"--ndevices=",11) == 0)))
          Impl::throw_runtime_exception("Error: expecting an '=INT[,INT]' after command line argument '--ndevices/--kokkos-ndevices'. Raised by Kokkos::initialize(int narg, char* argc[]).");

        char* num1 = strchr(arg[iarg],'=')+1;
        char* num2 = strpbrk(num1,",");
        int num1_len = num2==NULL?strlen(num1):num2-num1;
        char* num1_only = new char[num1_len+1];
        strncpy(num1_only,num1,num1_len);
        num1_only[num1_len]=0;

        if(!Impl::is_unsigned_int(num1_only) || (strlen(num1_only)==0)) {
          Impl::throw_runtime_exception("Error: expecting an integer number after command line argument '--kokkos-ndevices'. Raised by Kokkos::initialize(int narg, char* argc[]).");
        }
        if((strncmp(arg[iarg],"--kokkos-ndevices",17) == 0) || !kokkos_ndevices_found)
          ndevices = atoi(num1_only);
        delete [] num1_only;

        if( num2 != NULL ) {
          if(( !Impl::is_unsigned_int(num2+1) ) || (strlen(num2)==1) )
            Impl::throw_runtime_exception("Error: expecting an integer number after command line argument '--kokkos-ndevices=XX,'. Raised by Kokkos::initialize(int narg, char* argc[]).");

          if((strncmp(arg[iarg],"--kokkos-ndevices",17) == 0) || !kokkos_ndevices_found)
            skip_device = atoi(num2+1);
        }

        //Remove the --kokkos-ndevices argument from the list but leave --ndevices
        if(strncmp(arg[iarg],"--kokkos-ndevices",17) == 0) {
          for(int k=iarg;k<narg-1;k++) {
            arg[k] = arg[k+1];
          }
          kokkos_ndevices_found=1;
          narg--;
        } else {
          iarg++;
        }
      } else if ( strcmp(arg[iarg],"--kokkos-disable-warnings") == 0) {
        disable_warnings = true;
        for(int k=iarg;k<narg-1;k++) {
          arg[k] = arg[k+1];
        }
        narg--;
      } else if ((strcmp(arg[iarg],"--kokkos-help") == 0) || (strcmp(arg[iarg],"--help") == 0)) {
         std::cout << std::endl;
         std::cout << "--------------------------------------------------------------------------------" << std::endl;
         std::cout << "-------------Kokkos command line arguments--------------------------------------" << std::endl;
         std::cout << "--------------------------------------------------------------------------------" << std::endl;
         std::cout << "The following arguments exist also without prefix 'kokkos' (e.g. --help)." << std::endl;
         std::cout << "The prefixed arguments will be removed from the list by Kokkos::initialize()," << std::endl;
         std::cout << "the non-prefixed ones are not removed. Prefixed versions take precedence over " << std::endl;
         std::cout << "non prefixed ones, and the last occurrence of an argument overwrites prior" << std::endl;
         std::cout << "settings." << std::endl;
         std::cout << std::endl;
         std::cout << "--kokkos-help               : print this message" << std::endl;
         std::cout << "--kokkos-disable-warnings   : disable kokkos warning messages" << std::endl;
         std::cout << "--kokkos-threads=INT        : specify total number of threads or" << std::endl;
         std::cout << "                              number of threads per NUMA region if " << std::endl;
         std::cout << "                              used in conjunction with '--numa' option. " << std::endl;
         std::cout << "--kokkos-numa=INT           : specify number of NUMA regions used by process." << std::endl;
         std::cout << "--kokkos-device=INT         : specify device id to be used by Kokkos. " << std::endl;
         std::cout << "--kokkos-ndevices=INT[,INT] : used when running MPI jobs. Specify number of" << std::endl;
         std::cout << "                              devices per node to be used. Process to device" << std::endl;
         std::cout << "                              mapping happens by obtaining the local MPI rank" << std::endl;
         std::cout << "                              and assigning devices round-robin. The optional" << std::endl;
         std::cout << "                              second argument allows for an existing device" << std::endl;
         std::cout << "                              to be ignored. This is most useful on workstations" << std::endl;
         std::cout << "                              with multiple GPUs of which one is used to drive" << std::endl;
         std::cout << "                              screen output." << std::endl;
         std::cout << std::endl;
         std::cout << "--------------------------------------------------------------------------------" << std::endl;
         std::cout << std::endl;

         //Remove the --kokkos-help argument from the list but leave --ndevices
         if(strcmp(arg[iarg],"--kokkos-help") == 0) {
           for(int k=iarg;k<narg-1;k++) {
             arg[k] = arg[k+1];
           }
           narg--;
         } else {
           iarg++;
         }
      } else
      iarg++;
    }

    //Read environment variables
    char * endptr;
    auto env_num_threads_str = std::getenv("KOKKOS_NUM_THREADS");
    if (env_num_threads_str!=nullptr) {
        errno = 0;
        auto env_num_threads = std::strtol(env_num_threads_str,&endptr,10);
        if (endptr== env_num_threads_str) 
            Impl::throw_runtime_exception("Error: cannot convert KOKKOS_NUM_THREADS to an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if (errno == ERANGE)
            Impl::throw_runtime_exception("Error: KOKKOS_NUM_THREADS out of range of representable values by an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if ((num_threads != -1)&&(env_num_threads!=num_threads))
            Impl::throw_runtime_exception("Error: expecting a match between --kokkos-threads and KOKKOS_NUM_THREADS if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
        else
            num_threads = env_num_threads;
    }
    auto env_numa_str = std::getenv("KOKKOS_NUMA");
    if (env_numa_str!=nullptr) {
        errno = 0;
        auto env_numa = std::strtol(env_numa_str,&endptr,10);
        if (endptr== env_numa_str) 
            Impl::throw_runtime_exception("Error: cannot convert KOKKOS_NUMA to an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if (errno == ERANGE)
            Impl::throw_runtime_exception("Error: KOKKOS_NUMA out of range of representable values by an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if ((numa != -1)&&(env_numa!=numa))
            Impl::throw_runtime_exception("Error: expecting a match between --kokkos-numa and KOKKOS_NUMA if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
        else
            numa = env_numa;
    }
    auto env_device_str = std::getenv("KOKKOS_DEVICE_ID");
    if (env_device_str!=nullptr) {
        errno = 0;
        auto env_device = std::strtol(env_device_str,&endptr,10);
        if (endptr== env_device_str) 
            Impl::throw_runtime_exception("Error: cannot convert KOKKOS_DEVICE_ID to an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if (errno == ERANGE)
            Impl::throw_runtime_exception("Error: KOKKOS_DEVICE_ID out of range of representable values by an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if ((device != -1)&&(env_device!=device))
            Impl::throw_runtime_exception("Error: expecting a match between --kokkos-device and KOKKOS_DEVICE_ID if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
        else
            device = env_device;
    }
    auto env_ndevices_str = std::getenv("KOKKOS_NUM_DEVICES");
    if (env_ndevices_str!=nullptr) {
        errno = 0;
        auto env_ndevices = std::strtol(env_ndevices_str,&endptr,10);
        if (endptr== env_ndevices_str) 
            Impl::throw_runtime_exception("Error: cannot convert KOKKOS_NUM_DEVICES to an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if (errno == ERANGE)
            Impl::throw_runtime_exception("Error: KOKKOS_NUM_DEVICES out of range of representable values by an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
        if ((ndevices != -1)&&(env_ndevices!=ndevices))
            Impl::throw_runtime_exception("Error: expecting a match between --kokkos-ndevices and KOKKOS_NUM_DEVICES if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
        else
            ndevices = env_ndevices;
        //Skip device
        auto env_skip_device_str = std::getenv("KOKKOS_SKIP_DEVICE");
        if (env_skip_device_str!=nullptr) {
            errno = 0;
            auto env_skip_device = std::strtol(env_skip_device_str,&endptr,10);
            if (endptr== env_skip_device_str) 
                Impl::throw_runtime_exception("Error: cannot convert KOKKOS_SKIP_DEVICE to an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
            if (errno == ERANGE)
                Impl::throw_runtime_exception("Error: KOKKOS_SKIP_DEVICE out of range of representable values by an integer. Raised by Kokkos::initialize(int narg, char* argc[]).");
            if ((skip_device != 9999)&&(env_skip_device!=skip_device))
                Impl::throw_runtime_exception("Error: expecting a match between --kokkos-ndevices and KOKKOS_SKIP_DEVICE if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
            else
                skip_device = env_skip_device;
        }
    }
    char * env_disablewarnings_str = std::getenv("KOKKOS_DISABLE_WARNINGS");
    if (env_disablewarnings_str!=nullptr) {
        std::string env_str (env_disablewarnings_str); // deep-copies string
        for (char& c : env_str) { c = toupper (c); }
        if ((env_str == "TRUE") || (env_str == "ON") || (env_str == "1"))
            disable_warnings = true;
        else
            if (disable_warnings)
                Impl::throw_runtime_exception("Error: expecting a match between --kokkos-disable-warnings and KOKKOS_DISABLE_WARNINGS if both are set. Raised by Kokkos::initialize(int narg, char* argc[]).");
    }

    InitArguments arguments;
    arguments.num_threads = num_threads;
    arguments.num_numa = numa;
    arguments.device_id = device;
    arguments.ndevices = ndevices;
    arguments.skip_device = skip_device;
    arguments.disable_warnings = disable_warnings;
    Impl::initialize_internal(arguments);
}

void initialize(const InitArguments& arguments) {
  Impl::initialize_internal(arguments);
}

void push_finalize_hook(std::function<void()> f)
{
  finalize_hooks.push(f);
}

void finalize()
{
  Impl::finalize_internal();
}

void finalize_all()
{
  enum { all_spaces = true };
  Impl::finalize_internal( all_spaces );
}

void fence()
{
  Impl::fence_internal();
}

void print_configuration( std::ostream & out , const bool detail )
{
  std::ostringstream msg;

  msg << "Compiler:" << std::endl;
#ifdef KOKKOS_COMPILER_APPLECC
  msg << "  KOKKOS_COMPILER_APPLECC: " << KOKKOS_COMPILER_APPLECC << std::endl;
#endif
#ifdef KOKKOS_COMPILER_CLANG
  msg << "  KOKKOS_COMPILER_CLANG: " << KOKKOS_COMPILER_CLANG << std::endl;
#endif
#ifdef KOKKOS_COMPILER_CRAYC
  msg << "  KOKKOS_COMPILER_CRAYC: " << KOKKOS_COMPILER_CRAYC << std::endl;
#endif
#ifdef KOKKOS_COMPILER_GNU
  msg << "  KOKKOS_COMPILER_GNU: " << KOKKOS_COMPILER_GNU << std::endl;
#endif
#ifdef KOKKOS_COMPILER_IBM
  msg << "  KOKKOS_COMPILER_IBM: " << KOKKOS_COMPILER_IBM << std::endl;
#endif
#ifdef KOKKOS_COMPILER_INTEL
  msg << "  KOKKOS_COMPILER_INTEL: " << KOKKOS_COMPILER_INTEL << std::endl;
#endif
#ifdef KOKKOS_COMPILER_NVCC
  msg << "  KOKKOS_COMPILER_NVCC: " << KOKKOS_COMPILER_NVCC << std::endl;
#endif
#ifdef KOKKOS_COMPILER_PGI
  msg << "  KOKKOS_COMPILER_PGI: " << KOKKOS_COMPILER_PGI << std::endl;
#endif


  msg << "Architecture:" << std::endl;
#ifdef KOKKOS_ENABLE_ISA_KNC
  msg << "  KOKKOS_ENABLE_ISA_KNC: yes" << std::endl;
#else
  msg << "  KOKKOS_ENABLE_ISA_KNC: no" << std::endl;
#endif
#ifdef KOKKOS_ENABLE_ISA_POWERPCLE
  msg << "  KOKKOS_ENABLE_ISA_POWERPCLE: yes" << std::endl;
#else
  msg << "  KOKKOS_ENABLE_ISA_POWERPCLE: no" << std::endl;
#endif
#ifdef KOKKOS_ENABLE_ISA_X86_64
  msg << "  KOKKOS_ENABLE_ISA_X86_64: yes" << std::endl;
#else
  msg << "  KOKKOS_ENABLE_ISA_X86_64: no" << std::endl;
#endif


  msg << "Devices:" << std::endl;
  msg << "  KOKKOS_ENABLE_CUDA: ";
#ifdef KOKKOS_ENABLE_CUDA
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_OPENMP: ";
#ifdef KOKKOS_ENABLE_OPENMP
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_HPX: ";
#ifdef KOKKOS_ENABLE_HPX
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_THREADS: ";
#ifdef KOKKOS_ENABLE_THREADS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_QTHREADS: ";
#ifdef KOKKOS_ENABLE_QTHREADS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_SERIAL: ";
#ifdef KOKKOS_ENABLE_SERIAL
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif


  msg << "Default Device:" << std::endl;
  msg << "  KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_CUDA: ";
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_CUDA
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_OPENMP: ";
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_OPENMP
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_THREADS: ";
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_THREADS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_QTHREADS: ";
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_QTHREADS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL: ";
#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif


  msg << "Atomics:" << std::endl;
  msg << "  KOKKOS_ENABLE_CUDA_ATOMICS: ";
#ifdef KOKKOS_ENABLE_CUDA_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_GNU_ATOMICS: ";
#ifdef KOKKOS_ENABLE_GNU_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_INTEL_ATOMICS: ";
#ifdef KOKKOS_ENABLE_INTEL_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_OPENMP_ATOMICS: ";
#ifdef KOKKOS_ENABLE_OPENMP_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_WINDOWS_ATOMICS: ";
#ifdef KOKKOS_ENABLE_WINDOWS_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_SERIAL_ATOMICS: ";
#ifdef KOKKOS_ENABLE_SERIAL_ATOMICS
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif

  msg << "Vectorization:" << std::endl;
  msg << "  KOKKOS_ENABLE_PRAGMA_IVDEP: ";
#ifdef KOKKOS_ENABLE_PRAGMA_IVDEP
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_PRAGMA_LOOPCOUNT: ";
#ifdef KOKKOS_ENABLE_PRAGMA_LOOPCOUNT
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_PRAGMA_SIMD: ";
#ifdef KOKKOS_ENABLE_PRAGMA_SIMD
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_PRAGMA_UNROLL: ";
#ifdef KOKKOS_ENABLE_PRAGMA_UNROLL
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_PRAGMA_VECTOR: ";
#ifdef KOKKOS_ENABLE_PRAGMA_VECTOR
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif

  msg << "Memory:" << std::endl;
  msg << "  KOKKOS_ENABLE_HBWSPACE: ";
#ifdef KOKKOS_ENABLE_HBWSPACE
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_INTEL_MM_ALLOC: ";
#ifdef KOKKOS_ENABLE_INTEL_MM_ALLOC
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_POSIX_MEMALIGN: ";
#ifdef KOKKOS_ENABLE_POSIX_MEMALIGN
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif


  msg << "Options:" << std::endl;
  msg << "  KOKKOS_ENABLE_ASM: ";
#ifdef KOKKOS_ENABLE_ASM
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CXX14: ";
#ifdef KOKKOS_ENABLE_CXX14
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CXX17: ";
#ifdef KOKKOS_ENABLE_CXX17
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CXX20: ";
#ifdef KOKKOS_ENABLE_CXX20
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK: ";
#ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_HWLOC: ";
#ifdef KOKKOS_ENABLE_HWLOC
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_LIBRT: ";
#ifdef KOKKOS_ENABLE_LIBRT
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_MPI: ";
#ifdef KOKKOS_ENABLE_MPI
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_PROFILING: ";
#ifdef KOKKOS_ENABLE_PROFILING
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif

#ifdef KOKKOS_ENABLE_CUDA
  msg << "Cuda Options:" << std::endl;
  msg << "  KOKKOS_ENABLE_CUDA_LAMBDA: ";
#ifdef KOKKOS_ENABLE_CUDA_LAMBDA
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CUDA_LDG_INTRINSIC: ";
#ifdef KOKKOS_ENABLE_CUDA_LDG_INTRINSIC
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE: ";
#ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CUDA_UVM: ";
#ifdef KOKKOS_ENABLE_CUDA_UVM
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CUSPARSE: ";
#ifdef KOKKOS_ENABLE_CUSPARSE
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif
  msg << "  KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA: ";
#ifdef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA
  msg << "yes" << std::endl;
#else
  msg << "no" << std::endl;
#endif

#endif

  msg << "\nRuntime Configuration:" << std::endl;
#ifdef KOKKOS_ENABLE_CUDA
  Cuda::print_configuration(msg, detail);
#endif
#ifdef KOKKOS_ENABLE_OPENMP
  OpenMP::print_configuration(msg, detail);
#endif
#ifdef KOKKOS_ENABLE_HPX
  Experimental::HPX::print_configuration(msg, detail);
#endif
#if defined( KOKKOS_ENABLE_THREADS )
  Threads::print_configuration(msg, detail);
#endif
#ifdef KOKKOS_ENABLE_QTHREADS
  Qthreads::print_configuration(msg, detail);
#endif
#ifdef KOKKOS_ENABLE_SERIAL
  Serial::print_configuration(msg, detail);
#endif

  out << msg.str() << std::endl;
}

bool is_initialized() noexcept { return g_is_initialized; }

bool show_warnings() noexcept { return g_show_warnings; }

} // namespace Kokkos


Casting to void sometimes helps, but not always.