text: /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
  /*
   * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
   *                         University Research and Technology
   *                         Corporation.  All rights reserved.
   * Copyright (c) 2004-2009 The University of Tennessee and The University
   *                         of Tennessee Research Foundation.  All rights
   *                         reserved.
   * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   *                         University of Stuttgart.  All rights reserved.
   * Copyright (c) 2004-2005 The Regents of the University of California.
   *                         All rights reserved.
   * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
   *                         reserved.
   * $COPYRIGHT$
   *
   * Additional copyrights may follow
   *
   * $HEADER$
   */
  /**
   * @file
   */
  #ifndef MCA_BTL_TEMPLATE_H
  #define MCA_BTL_TEMPLATE_H
  
  #include "opal_config.h"
  #include <sys/types.h>
  #include <string.h>
  
  /* Open MPI includes */
  #include "opal/mca/event/event.h"
  #include "opal/mca/btl/btl.h"
  #include "opal/mca/btl/base/base.h"
  #include "opal/mca/mpool/mpool.h"
  
  BEGIN_C_DECLS
  
  #define MCA_BTL_HAS_MPOOL 1
  
  /**
   * Infiniband (TEMPLATE) BTL component.
   */
  
  struct mca_btl_template_component_t {
      mca_btl_base_component_3_0_0_t          super;  /**< base BTL component */
  
      uint32_t                                template_num_btls;
      /**< number of hcas available to the TEMPLATE component */
  
      struct mca_btl_template_module_t       *template_btls;
      /**< array of available BTL modules */
  
      int template_free_list_num;
      /**< initial size of free lists */
  
      int template_free_list_max;
      /**< maximum size of free lists */
  
      int template_free_list_inc;
      /**< number of elements to alloc when growing free lists */
  
      opal_list_t template_procs;
      /**< list of template proc structures */
  
      opal_mutex_t template_lock;
      /**< lock for accessing module state */
  
      char* template_mpool_name;
      /**< name of memory pool */
  
      bool leave_pinned;
      /**< pin memory on first use and leave pinned */
  };
  typedef struct mca_btl_template_component_t mca_btl_template_component_t;
  
  OPAL_MODULE_DECLSPEC extern mca_btl_template_component_t mca_btl_template_component;
  
  /**
   * BTL Module Interface
   */
  struct mca_btl_template_module_t {
      mca_btl_base_module_t  super;  /**< base BTL interface */
  
      /* free list of fragment descriptors */
      opal_free_list_t template_frag_eager;
      opal_free_list_t template_frag_max;
      opal_free_list_t template_frag_user;
  
      /* lock for accessing module state */
      opal_mutex_t template_lock;
  
  #if MCA_BTL_HAS_MPOOL
      struct mca_mpool_base_module_t* template_mpool;
  #endif
  };
  typedef struct mca_btl_template_module_t mca_btl_template_module_t;
  extern mca_btl_template_module_t mca_btl_template_module;
  
  
  /**
   * TEMPLATE component initialization.
   *
   * @param num_btl_modules (OUT)           Number of BTLs returned in BTL array.
   * @param allow_multi_user_threads (OUT)  Flag indicating wether BTL supports user threads (TRUE)
   * @param have_hidden_threads (OUT)       Flag indicating wether BTL uses threads (TRUE)
   */
  extern mca_btl_base_module_t** mca_btl_template_component_init(
      int *num_btl_modules,
      bool allow_multi_user_threads,
      bool have_hidden_threads
  );
  
  
  /**
   * TEMPLATE component progress.
   */
  extern int mca_btl_template_component_progress(void);
  
  
  
  /**
   * Cleanup any resources held by the BTL.
   *
   * @param btl  BTL instance.
   * @return     OPAL_SUCCESS or error status on failure.
   */
  
  extern int mca_btl_template_finalize(
      struct mca_btl_base_module_t* btl
  );
  
  
  /**
   * PML->BTL notification of change in the process list.
   *
   * @param btl (IN)
   * @param nprocs (IN)     Number of processes
   * @param procs (IN)      Set of processes
   * @param peers (OUT)     Set of (optional) peer addressing info.
   * @param peers (IN/OUT)  Set of processes that are reachable via this BTL.
   * @return     OPAL_SUCCESS or error status on failure.
   *
   */
  
  extern int mca_btl_template_add_procs(
      struct mca_btl_base_module_t* btl,
      size_t nprocs,
      struct opal_proc_t **procs,
      struct mca_btl_base_endpoint_t** peers,
      opal_bitmap_t* reachable
  );
  
  /**
   * PML->BTL notification of change in the process list.
   *
   * @param btl (IN)     BTL instance
   * @param nproc (IN)   Number of processes.
   * @param procs (IN)   Set of processes.
   * @param peers (IN)   Set of peer data structures.
   * @return             Status indicating if cleanup was successful
   *
   */
  
  extern int mca_btl_template_del_procs(
      struct mca_btl_base_module_t* btl,
      size_t nprocs,
      struct opal_proc_t **procs,
      struct mca_btl_base_endpoint_t** peers
  );
  
  
  /**
   * Initiate an asynchronous send.
   *
   * @param btl (IN)         BTL module
   * @param endpoint (IN)    BTL addressing information
   * @param descriptor (IN)  Description of the data to be transfered
   * @param tag (IN)         The tag value used to notify the peer.
   */
  
  extern int mca_btl_template_send(
      struct mca_btl_base_module_t* btl,
      struct mca_btl_base_endpoint_t* btl_peer,
      struct mca_btl_base_descriptor_t* descriptor,
      mca_btl_base_tag_t tag
  );
  
  
  /**
   * Initiate an asynchronous put.
   * Completion Semantics: if this function returns a 1 then the operation
   *                       is complete. a return of OPAL_SUCCESS indicates
   *                       the put operation has been queued with the
   *                       network. the local_handle can not be deregistered
   *                       until all outstanding operations on that handle
   *                       have been completed.
   *
   * @param btl (IN)            BTL module
   * @param endpoint (IN)       BTL addressing information
   * @param local_address (IN)  Local address to put from (registered)
   * @param remote_address (IN) Remote address to put to (registered remotely)
   * @param local_handle (IN)   Registration handle for region containing
   *                            (local_address, local_address + size)
   * @param remote_handle (IN)  Remote registration handle for region containing
   *                            (remote_address, remote_address + size)
   * @param size (IN)           Number of bytes to put
   * @param flags (IN)          Flags for this put operation
   * @param order (IN)          Ordering
   * @param cbfunc (IN)         Function to call on completion (if queued)
   * @param cbcontext (IN)      Context for the callback
   * @param cbdata (IN)         Data for callback
   *
   * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
   * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
   * @retval OPAL_ERR_OUT_OF_RESOURCE  Insufficient resources to queue the put
   *                         operation. Try again later
   * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
   *                         alignment restrictions.
   */
  int mca_btl_template_put (struct mca_btl_base_module_t *btl,
      struct mca_btl_base_endpoint_t *endpoint, void *local_address,
      uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
      struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
  
  /**
   * Initiate an asynchronous get.
   * Completion Semantics: if this function returns a 1 then the operation
   *                       is complete. a return of OPAL_SUCCESS indicates
   *                       the get operation has been queued with the
   *                       network. the local_handle can not be deregistered
   *                       until all outstanding operations on that handle
   *                       have been completed.
   *
   * @param btl (IN)            BTL module
   * @param endpoint (IN)       BTL addressing information
   * @param local_address (IN)  Local address to put from (registered)
   * @param remote_address (IN) Remote address to put to (registered remotely)
   * @param local_handle (IN)   Registration handle for region containing
   *                            (local_address, local_address + size)
   * @param remote_handle (IN)  Remote registration handle for region containing
   *                            (remote_address, remote_address + size)
   * @param size (IN)           Number of bytes to put
   * @param flags (IN)          Flags for this put operation
   * @param order (IN)          Ordering
   * @param cbfunc (IN)         Function to call on completion (if queued)
   * @param cbcontext (IN)      Context for the callback
   * @param cbdata (IN)         Data for callback
   *
   * @retval OPAL_SUCCESS    The descriptor was successfully queued for a put
   * @retval OPAL_ERROR      The descriptor was NOT successfully queued for a put
   * @retval OPAL_ERR_OUT_OF_RESOURCE  Insufficient resources to queue the put
   *                         operation. Try again later
   * @retval OPAL_ERR_NOT_AVAILABLE  Put can not be performed due to size or
   *                         alignment restrictions.
   */
  int mca_btl_template_get (struct mca_btl_base_module_t *btl,
      struct mca_btl_base_endpoint_t *endpoint, void *local_address,
      uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle,
      struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
      int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
  
  /**
   * @brief Register a memory region for put/get/atomic operations.
   *
   * @param btl (IN)         BTL module
   * @param endpoint(IN)     BTL addressing information (or NULL for all endpoints)
   * @param base (IN)        Pointer to start of region
   * @param size (IN)        Size of region
   * @param flags (IN)       Flags indicating what operation will be performed. Valid
   *                         values are MCA_BTL_DES_FLAGS_PUT, MCA_BTL_DES_FLAGS_GET,
   *                         and MCA_BTL_DES_FLAGS_ATOMIC
   *
   * @returns a memory registration handle valid for both local and remote operations
   * @returns NULL if the region could not be registered
   *
   * This function registers the specified region with the hardware for use with
   * the btl_put, btl_get, btl_atomic_cas, btl_atomic_op, and btl_atomic_fop
   * functions. Care should be taken to not hold an excessive number of registrations
   * as they may use limited system/NIC resources.
   */
  struct mca_btl_base_registration_handle_t *mca_btl_template_register_mem (
      struct mca_btl_base_module_t* btl, struct mca_btl_base_endpoint_t *endpoint, void *base,
      size_t size, uint32_t flags);
  
  /**
   * @brief Deregister a memory region
   *
   * @param btl (IN)         BTL module region was registered with
   * @param handle (IN)      BTL registration handle to deregister
   *
   * This function deregisters the memory region associated with the specified handle. Care
   * should be taken to not perform any RDMA or atomic operation on this memory region
   * after it is deregistered. It is erroneous to specify a memory handle associated with
   * a remote node.
   */
  int mca_btl_template_deregister_mem (struct mca_btl_base_module_t* btl,
                                       struct mca_btl_base_registration_handle_t *handle);
  
  /**
   * Register a callback function that is called on receipt
   * of a fragment.
   *
   * @param btl (IN)     BTL module
   * @return             Status indicating if registration was successful
   *
   */
  
  extern int mca_btl_template_register(
      struct mca_btl_base_module_t* btl,
      mca_btl_base_tag_t tag,
      mca_btl_base_module_recv_cb_fn_t cbfunc,
      void* cbdata);
  
  /**
   * Allocate a descriptor with a segment of the requested size.
   * Note that the BTL layer may choose to return a smaller size
   * if it cannot support the request.
   *
   * @param btl (IN)      BTL module
   * @param size (IN)     Request segment size.
   */
  
  extern mca_btl_base_descriptor_t* mca_btl_template_alloc(
      struct mca_btl_base_module_t* btl,
      struct mca_btl_base_endpoint_t* endpoint,
      uint8_t order,
      size_t size,
      uint32_t flags);
  
  
  /**
   * Return a segment allocated by this BTL.
   *
   * @param btl (IN)      BTL module
   * @param descriptor (IN)  Allocated descriptor.
   */
  
  extern int mca_btl_template_free(
      struct mca_btl_base_module_t* btl,
      mca_btl_base_descriptor_t* des);
  
  
  /**
   * Prepare a descriptor for send/rdma using the supplied
   * convertor. If the convertor references data that is contigous,
   * the descriptor may simply point to the user buffer. Otherwise,
   * this routine is responsible for allocating buffer space and
   * packing if required.
   *
   * @param btl (IN)          BTL module
   * @param endpoint (IN)     BTL peer addressing
   * @param convertor (IN)    Data type convertor
   * @param reserve (IN)      Additional bytes requested by upper layer to precede user data
   * @param size (IN/OUT)     Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
  */
  
  mca_btl_base_descriptor_t* mca_btl_template_prepare_src(
      struct mca_btl_base_module_t* btl,
      struct mca_btl_base_endpoint_t* peer,
      struct opal_convertor_t* convertor,
      uint8_t order,
      size_t reserve,
      size_t* size,
      uint32_t flags
  );
  
   /**
    * Fault Tolerance Event Notification Function
    * @param state Checkpoint Stae
    * @return OPAL_SUCCESS or failure status
    */
  int mca_btl_template_ft_event(int state);
  
  END_C_DECLS
  #endif
  