text: //  Copyright (c) 2007-2021 Hartmut Kaiser
  //  Copyright (c)      2011 Bryce Lelbach
  //
  //  SPDX-License-Identifier: BSL-1.0
  //  Distributed under the Boost Software License, Version 1.0. (See accompanying
  //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  
  #include <hpx/config.hpp>
  
  #if defined(HPX_HAVE_DISTRIBUTED_RUNTIME)
  
  #include <hpx/agas/addressing_service.hpp>
  #include <hpx/collectives/barrier.hpp>
  #include <hpx/collectives/detail/barrier_node.hpp>
  #include <hpx/collectives/latch.hpp>
  #include <hpx/components_base/agas_interface.hpp>
  #include <hpx/datastructures/tuple.hpp>
  #include <hpx/init_runtime/pre_main.hpp>
  #include <hpx/modules/errors.hpp>
  #include <hpx/modules/logging.hpp>
  #include <hpx/parcelset/message_handler_fwd.hpp>
  #include <hpx/performance_counters/agas_counter_types.hpp>
  #include <hpx/performance_counters/parcelhandler_counter_types.hpp>
  #include <hpx/performance_counters/threadmanager_counter_types.hpp>
  #include <hpx/runtime_components/console_logging.hpp>
  #include <hpx/runtime_configuration/runtime_mode.hpp>
  #include <hpx/runtime_distributed.hpp>
  #include <hpx/runtime_distributed/applier.hpp>
  #include <hpx/runtime_distributed/runtime_fwd.hpp>
  #include <hpx/runtime_distributed/runtime_support.hpp>
  #include <hpx/runtime_local/runtime_local_fwd.hpp>
  #include <hpx/runtime_local/shutdown_function.hpp>
  
  #include <string>
  #include <vector>
  
  namespace hpx { namespace detail {
  
      static void garbage_collect_non_blocking()
      {
          return ::hpx::agas::garbage_collect_non_blocking();
      }
  
      static void garbage_collect()
      {
          return ::hpx::agas::garbage_collect();
      }
  
      ///////////////////////////////////////////////////////////////////////////
      // Install performance counter startup functions for core subsystems.
      static void register_counter_types()
      {
          auto& agas_client = naming::get_agas_client();
          performance_counters::register_agas_counter_types(agas_client);
          agas_client.register_server_instances();
          lbt_ << "(2nd stage) pre_main: registered AGAS client-side "
                  "performance counter types";
  
          get_runtime_distributed().register_counter_types();
          lbt_ << "(2nd stage) pre_main: registered runtime performance "
                  "counter types";
  
          performance_counters::register_threadmanager_counter_types(
              threads::get_thread_manager());
          lbt_ << "(2nd stage) pre_main: registered thread-manager performance "
                  "counter types";
  
  #if defined(HPX_HAVE_NETWORKING)
          performance_counters::register_parcelhandler_counter_types(
              applier::get_applier().get_parcel_handler());
          lbt_ << "(2nd stage) pre_main: registered parcelset performance "
                  "counter types";
  #endif
      }
  
      ///////////////////////////////////////////////////////////////////////////
  #if defined(HPX_HAVE_NETWORKING)
      static void register_message_handlers()
      {
          runtime_distributed& rtd = get_runtime_distributed();
          for (auto const& t :
              parcelset::detail::get_message_handler_registrations())
          {
              error_code ec(lightweight);
              rtd.register_message_handler(hpx::get<0>(t), hpx::get<1>(t), ec);
          }
          lbt_ << "(3rd stage) pre_main: registered message handlers";
      }
  #endif
  
      ///////////////////////////////////////////////////////////////////////////
      // Implements second and third stage bootstrapping.
      int pre_main(runtime_mode mode)
      {
          // Register pre-shutdown and shutdown functions to flush pending
          // reference counting operations.
          register_pre_shutdown_function(&garbage_collect_non_blocking);
          register_shutdown_function(&garbage_collect);
  
          using components::stubs::runtime_support;
  
          naming::resolver_client& agas_client = naming::get_agas_client();
          runtime& rt = get_runtime();
  
          int exit_code = 0;
          if (runtime_mode::connect == mode)
          {
              lbt_ << "(2nd stage) pre_main: locality is in connect mode, "
                      "skipping 2nd and 3rd stage startup synchronization";
              lbt_ << "(2nd stage) pre_main: addressing services enabled";
  
              // Load components, so that we can use the barrier LCO.
              exit_code = runtime_support::load_components(find_here());
              lbt_ << "(2nd stage) pre_main: loaded components"
                   << (exit_code ? ", application exit has been requested" : "");
  
              // Work on registration requests for message handler plugins
  #if defined(HPX_HAVE_NETWORKING)
              register_message_handlers();
  #endif
              // Register all counter types before the startup functions are being
              // executed.
              register_counter_types();
  
              rt.set_state(state_pre_startup);
              runtime_support::call_startup_functions(find_here(), true);
              lbt_ << "(3rd stage) pre_main: ran pre-startup functions";
  
              rt.set_state(state_startup);
              runtime_support::call_startup_functions(find_here(), false);
              lbt_ << "(4th stage) pre_main: ran startup functions";
          }
          else
          {
              lbt_ << "(2nd stage) pre_main: addressing services enabled";
  
              // Load components, so that we can use the barrier LCO.
              exit_code = runtime_support::load_components(find_here());
              lbt_ << "(2nd stage) pre_main: loaded components"
                   << (exit_code ? ", application exit has been requested" : "");
  
              // Second and third stage barrier creation.
              if (agas_client.is_bootstrap())
              {
                  naming::gid_type console_;
                  if (HPX_UNLIKELY(!agas_client.get_console_locality(console_)))
                  {
                      HPX_THROW_EXCEPTION(network_error, "pre_main",
                          "no console locality registered");
                  }
  
                  lbt_ << "(2nd stage) pre_main: creating 2nd and 3rd stage boot "
                          "barriers";
              }
              else    // Hosted.
              {
                  lbt_ << "(2nd stage) pre_main: finding 2nd and 3rd stage boot "
                          "barriers";
              }
  
              // create our global barrier...
              hpx::lcos::barrier::get_global_barrier() =
                  hpx::lcos::barrier::create_global_barrier();
  
              // Second stage bootstrap synchronizes component loading across all
              // localities, ensuring that the component namespace tables are fully
              // populated before user code is executed.
              lcos::barrier::synchronize();
              lbt_ << "(2nd stage) pre_main: passed 2nd stage boot barrier";
  
              // Work on registration requests for message handler plugins
  #if defined(HPX_HAVE_NETWORKING)
              register_message_handlers();
  #endif
              // Register all counter types before the startup functions are being
              // executed.
              register_counter_types();
  
              // Second stage bootstrap synchronizes performance counter loading
              // across all localities.
              lcos::barrier::synchronize();
              lbt_ << "(3rd stage) pre_main: passed 3rd stage boot barrier";
  
              runtime_support::call_startup_functions(find_here(), true);
              lbt_ << "(3rd stage) pre_main: ran pre-startup functions";
  
              // Third stage separates pre-startup and startup function phase.
              lcos::barrier::synchronize();
              lbt_ << "(4th stage) pre_main: passed 4th stage boot barrier";
  
              runtime_support::call_startup_functions(find_here(), false);
              lbt_ << "(4th stage) pre_main: ran startup functions";
  
              // Forth stage bootstrap synchronizes startup functions across all
              // localities. This is done after component loading to guarantee that
              // all user code, including startup functions, are only run after the
              // component tables are populated.
              lcos::barrier::synchronize();
              lbt_ << "(5th stage) pre_main: passed 4th stage boot barrier";
          }
  
          // Enable logging. Even if we terminate at this point we will see all
          // pending log messages so far.
          components::activate_logging();
          lbt_ << "(last stage) pre_main: activated logging";
  
          // Any error in post-command line handling or any explicit --exit command
          // line option will cause the application to terminate at this point.
          if (exit_code)
          {
              // If load_components returns false, shutdown the system. This
              // essentially only happens if the command line contained --exit.
              runtime_support::shutdown_all(
                  naming::get_id_from_locality_id(agas::booststrap_prefix), -1.0);
              return exit_code;
          }
  
          // Connect back to given latch if specified
          std::string connect_back_to(
              get_config_entry("hpx.on_startup.wait_on_latch", ""));
          if (!connect_back_to.empty())
          {
              lbt_ << "(6th stage) runtime::run_helper: about to "
                      "synchronize with latch: "
                   << connect_back_to;
  
              // inform launching process that this locality is up and running
              hpx::lcos::latch l;
              l.connect_to(connect_back_to);
              l.count_down_and_wait();
  
              lbt_ << "(6th stage) runtime::run_helper: "
                      "synchronized with latch: "
                   << connect_back_to;
          }
  
          return 0;
      }
  
      void post_main()
      {
          // simply destroy global barrier
          hpx::lcos::barrier::get_global_barrier().detach();
      }
  }}    // namespace hpx::detail
  
  #endif
  