text: /*
   * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   * Copyright (c) 2004, 2011-2012 Intel Corporation.  All rights reserved.
   * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.  All rights reserved.
   * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
   * Copyright (c) Microsoft Corporation. All rights reserved.
   *
   * This software is available to you under a choice of one of two
   * licenses.  You may choose to be licensed under the terms of the GNU
   * General Public License (GPL) Version 2, available from the file
   * COPYING in the main directory of this source tree, or the
   * OpenIB.org BSD license below:
   *
   *     Redistribution and use in source and binary forms, with or
   *     without modification, are permitted provided that the following
   *     conditions are met:
   *
   *      - Redistributions of source code must retain the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer.
   *
   *      - Redistributions in binary form must reproduce the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer in the documentation and/or other materials
   *        provided with the distribution.
   *
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   * SOFTWARE.
   */
  
  #ifndef INFINIBAND_VERBS_EXP_H
  #define INFINIBAND_VERBS_EXP_H
  
  #include <infiniband/verbs.h>
  #include <stdio.h>
  #include <stdlib.h>
  
  #if __GNUC__ >= 3
  #  define __attribute_const __attribute__((const))
  #else
  #  define __attribute_const
  #endif
  
  BEGIN_C_DECLS
  
  #define IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, ret)						\
  	if ((val) > (valid_mask)) {									\
  		fprintf(stderr, "%s: invalid comp_mask !!! (comp_mask = 0x%x valid_mask = 0x%x)\n",	\
  			__FUNCTION__, val, valid_mask);							\
  		errno = EINVAL;										\
  		return ret;										\
  	}
  
  #define IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(val, valid_mask)	\
  		IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, NULL)
  
  #define IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(val, valid_mask)	\
  		IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, EINVAL)
  
  
  #define IBV_EXP_IMPLICIT_MR_SIZE (~((size_t)0))
  
  enum ibv_exp_func_name {
  	IBV_EXP_POST_SEND_FUNC,
  	IBV_EXP_POLL_CQ_FUNC,
  	IBV_POST_SEND_FUNC,
  	IBV_POLL_CQ_FUNC,
  	IBV_POST_RECV_FUNC
  };
  
  enum ibv_exp_start_values {
  	IBV_EXP_START_ENUM	= 0x40,
  	IBV_EXP_START_FLAG_LOC	= 0x20,
  	IBV_EXP_START_FLAG	= (1ULL << IBV_EXP_START_FLAG_LOC),
  };
  
  /*
   * Capabilities for exp_atomic_cap field in ibv_exp_device_attr struct
   */
  enum ibv_exp_atomic_cap {
  	IBV_EXP_ATOMIC_NONE		= IBV_ATOMIC_NONE,
  	IBV_EXP_ATOMIC_HCA		= IBV_ATOMIC_HCA,
  	IBV_EXP_ATOMIC_GLOB		= IBV_ATOMIC_GLOB,
  
  	IBV_EXP_ATOMIC_HCA_REPLY_BE	= IBV_EXP_START_ENUM /* HOST is LE and atomic reply is BE */
  };
  
  /*
   * Flags for exp_device_cap_flags field in ibv_exp_device_attr struct
   */
  enum ibv_exp_device_cap_flags {
  	IBV_EXP_DEVICE_RESIZE_MAX_WR		= IBV_DEVICE_RESIZE_MAX_WR,
  	IBV_EXP_DEVICE_BAD_PKEY_CNTR		= IBV_DEVICE_BAD_PKEY_CNTR,
  	IBV_EXP_DEVICE_BAD_QKEY_CNTR		= IBV_DEVICE_BAD_QKEY_CNTR,
  	IBV_EXP_DEVICE_RAW_MULTI		= IBV_DEVICE_RAW_MULTI,
  	IBV_EXP_DEVICE_AUTO_PATH_MIG		= IBV_DEVICE_AUTO_PATH_MIG,
  	IBV_EXP_DEVICE_CHANGE_PHY_PORT		= IBV_DEVICE_CHANGE_PHY_PORT,
  	IBV_EXP_DEVICE_UD_AV_PORT_ENFORCE	= IBV_DEVICE_UD_AV_PORT_ENFORCE,
  	IBV_EXP_DEVICE_CURR_QP_STATE_MOD	= IBV_DEVICE_CURR_QP_STATE_MOD,
  	IBV_EXP_DEVICE_SHUTDOWN_PORT		= IBV_DEVICE_SHUTDOWN_PORT,
  	IBV_EXP_DEVICE_INIT_TYPE		= IBV_DEVICE_INIT_TYPE,
  	IBV_EXP_DEVICE_PORT_ACTIVE_EVENT	= IBV_DEVICE_PORT_ACTIVE_EVENT,
  	IBV_EXP_DEVICE_SYS_IMAGE_GUID		= IBV_DEVICE_SYS_IMAGE_GUID,
  	IBV_EXP_DEVICE_RC_RNR_NAK_GEN		= IBV_DEVICE_RC_RNR_NAK_GEN,
  	IBV_EXP_DEVICE_SRQ_RESIZE		= IBV_DEVICE_SRQ_RESIZE,
  	IBV_EXP_DEVICE_N_NOTIFY_CQ		= IBV_DEVICE_N_NOTIFY_CQ,
  	IBV_EXP_DEVICE_XRC			= IBV_DEVICE_XRC,
  
  	IBV_EXP_DEVICE_DC_TRANSPORT		= (IBV_EXP_START_FLAG << 0),
  	IBV_EXP_DEVICE_QPG			= (IBV_EXP_START_FLAG << 1),
  	IBV_EXP_DEVICE_UD_RSS			= (IBV_EXP_START_FLAG << 2),
  	IBV_EXP_DEVICE_UD_TSS			= (IBV_EXP_START_FLAG << 3),
  	IBV_EXP_DEVICE_EXT_ATOMICS		= (IBV_EXP_START_FLAG << 4),
  	IBV_EXP_DEVICE_NOP			= (IBV_EXP_START_FLAG << 5),
  	IBV_EXP_DEVICE_UMR			= (IBV_EXP_START_FLAG << 6),
  	IBV_EXP_DEVICE_ODP			= (IBV_EXP_START_FLAG << 7),
  	IBV_EXP_DEVICE_VXLAN_SUPPORT		= (IBV_EXP_START_FLAG << 10),
  	IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT	= (IBV_EXP_START_FLAG << 11),
  	IBV_EXP_DEVICE_RX_CSUM_IP_PKT		= (IBV_EXP_START_FLAG << 12),
  	IBV_EXP_DEVICE_EC_OFFLOAD		= (IBV_EXP_START_FLAG << 13),
  	IBV_EXP_DEVICE_EXT_MASKED_ATOMICS	= (IBV_EXP_START_FLAG << 14),
  	IBV_EXP_DEVICE_RX_TCP_UDP_PKT_TYPE	= (IBV_EXP_START_FLAG << 15),
  	IBV_EXP_DEVICE_SCATTER_FCS		= (IBV_EXP_START_FLAG << 16),
  	IBV_EXP_DEVICE_MEM_WINDOW		= (IBV_EXP_START_FLAG << 17),
  	IBV_EXP_DEVICE_MEM_MGT_EXTENSIONS	= (IBV_EXP_START_FLAG << 21),
  	IBV_EXP_DEVICE_DC_INFO			= (IBV_EXP_START_FLAG << 22),
  	/* Jumping to 23 as of next capability in include/rdma/ib_verbs.h */
  	IBV_EXP_DEVICE_MW_TYPE_2A		= (IBV_EXP_START_FLAG << 23),
  	IBV_EXP_DEVICE_MW_TYPE_2B		= (IBV_EXP_START_FLAG << 24),
  	IBV_EXP_DEVICE_CROSS_CHANNEL		= (IBV_EXP_START_FLAG << 28),
  	IBV_EXP_DEVICE_MANAGED_FLOW_STEERING	= (IBV_EXP_START_FLAG << 29),
  	IBV_EXP_DEVICE_MR_ALLOCATE		= (IBV_EXP_START_FLAG << 30),
  	IBV_EXP_DEVICE_SHARED_MR		= (IBV_EXP_START_FLAG << 31),
  };
  
  /*
   * Flags for ibv_exp_device_attr struct comp_mask.
   */
  enum ibv_exp_device_attr_comp_mask {
  	IBV_EXP_DEVICE_ATTR_CALC_CAP		= (1 << 0),
  	IBV_EXP_DEVICE_ATTR_WITH_TIMESTAMP_MASK	= (1 << 1),
  	IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK	= (1 << 2),
  	IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS	= (1 << 3),
  	IBV_EXP_DEVICE_DC_RD_REQ		= (1 << 4),
  	IBV_EXP_DEVICE_DC_RD_RES		= (1 << 5),
  	IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ	= (1 << 6),
  	IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ		= (1 << 7),
  	IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS	= (1 << 8),
  	IBV_EXP_DEVICE_ATTR_UMR                 = (1 << 9),
  	IBV_EXP_DEVICE_ATTR_ODP			= (1 << 10),
  	IBV_EXP_DEVICE_ATTR_MAX_DCT		= (1 << 11),
  	IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN	= (1 << 12),
  	IBV_EXP_DEVICE_ATTR_RX_HASH		= (1 << 13),
  	IBV_EXP_DEVICE_ATTR_MAX_WQ_TYPE_RQ	= (1 << 14),
  	IBV_EXP_DEVICE_ATTR_MAX_DEVICE_CTX	= (1 << 15),
  	IBV_EXP_DEVICE_ATTR_MP_RQ		= (1 << 16),
  	IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS	= (1 << 17),
  	IBV_EXP_DEVICE_ATTR_EC_CAPS		= (1 << 18),
  	IBV_EXP_DEVICE_ATTR_MASKED_ATOMICS	= (1 << 19),
  	IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN	= (1 << 20),
  	IBV_EXP_DEVICE_ATTR_TSO_CAPS		= (1 << 21),
  	IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS	= (1 << 22),
  	IBV_EXP_DEVICE_ATTR_EC_GF_BASE		= (1 << 23),
  	/* set supported bits for validity check */
  	IBV_EXP_DEVICE_ATTR_RESERVED		= (1 << 24),
  };
  
  struct ibv_exp_device_calc_cap {
  	uint64_t		data_types;
  	uint64_t		data_sizes;
  	uint64_t		int_ops;
  	uint64_t		uint_ops;
  	uint64_t		fp_ops;
  };
  
  struct ibv_exp_ext_atomics_params {
  	/* defines which masked operation sizes are supported with same
  	 * endianness as stated in atomic_cap field
  	 */
  	uint64_t		log_atomic_arg_sizes; /* bit-mask of supported sizes */
  	uint32_t		max_fa_bit_boundary;
  	uint32_t		log_max_atomic_inline;
  };
  
  struct ibv_exp_masked_atomic_params {
  	uint32_t	max_fa_bit_boundary;
  	uint32_t	log_max_atomic_inline;
  	uint64_t	masked_log_atomic_arg_sizes;
  	uint64_t	masked_log_atomic_arg_sizes_network_endianness;
  };
  
  enum ibv_odp_general_cap_bits {
  	IBV_EXP_ODP_SUPPORT = 1 << 0,
  };
  
  enum ibv_exp_odp_transport_cap_bits {
  	IBV_EXP_ODP_SUPPORT_SEND     = 1 << 0,
  	IBV_EXP_ODP_SUPPORT_RECV     = 1 << 1,
  	IBV_EXP_ODP_SUPPORT_WRITE    = 1 << 2,
  	IBV_EXP_ODP_SUPPORT_READ     = 1 << 3,
  	IBV_EXP_ODP_SUPPORT_ATOMIC   = 1 << 4,
  	IBV_EXP_ODP_SUPPORT_SRQ_RECV = 1 << 5,
  };
  
  struct ibv_exp_umr_caps {
  	uint32_t                max_klm_list_size;
  	uint32_t                max_send_wqe_inline_klms;
  	uint32_t                max_umr_recursion_depth;
  	uint32_t                max_umr_stride_dimension;
  };
  
  struct ibv_exp_odp_caps {
  	uint64_t	general_odp_caps;
  	struct {
  		uint32_t	rc_odp_caps;
  		uint32_t	uc_odp_caps;
  		uint32_t	ud_odp_caps;
  		uint32_t	dc_odp_caps;
  		uint32_t	xrc_odp_caps;
  		uint32_t	raw_eth_odp_caps;
  	} per_transport_caps;
  };
  
  enum ibv_exp_supported_qp_types {
  	IBV_EXP_QPT_RC		= 1ULL << 0,
  	IBV_EXP_QPT_UC		= 1ULL << 1,
  	IBV_EXP_QPT_UD		= 1ULL << 2,
  	IBV_EXP_QPT_XRC_INIT	= 1ULL << 3,
  	IBV_EXP_QPT_XRC_TGT	= 1ULL << 4,
  	IBV_EXP_QPT_RAW_PACKET	= 1ULL << 5,
  	IBV_EXP_QPT_RESERVED	= 1ULL << 6
  };
  
  struct ibv_exp_rx_hash_caps {
  	uint32_t max_rwq_indirection_tables;
  	uint32_t max_rwq_indirection_table_size;
  	uint8_t  supported_hash_functions; /* from ibv_exp_rx_hash_function_flags */
  	uint64_t supported_packet_fields;  /* from ibv_exp_rx_hash_fields */
  	uint32_t supported_qps;		   /* from ibv_exp_supported_qp_types */
  };
  
  enum ibv_exp_mp_rq_shifts {
  	IBV_EXP_MP_RQ_NO_SHIFT		= 0,
  	IBV_EXP_MP_RQ_2BYTES_SHIFT	= 1 << 0
  };
  
  struct ibv_exp_mp_rq_caps {
  	uint32_t supported_qps; /* use ibv_exp_supported_qp_types */
  	uint32_t allowed_shifts; /* use ibv_exp_mp_rq_shifts */
  	uint8_t min_single_wqe_log_num_of_strides;
  	uint8_t max_single_wqe_log_num_of_strides;
  	uint8_t min_single_stride_log_num_of_bytes;
  	uint8_t max_single_stride_log_num_of_bytes;
  };
  
  struct ibv_exp_ec_caps {
  	uint32_t	max_ec_data_vector_count;
  	uint32_t	max_ec_calc_inflight_calcs;
  };
  
  #define ibv_is_qpt_supported(caps, qpt) ((caps) & (1 << (qpt)))
  
  struct ibv_exp_tso_caps {
  	uint32_t max_tso;
  	uint32_t supported_qpts;
  };
  
  struct ibv_exp_packet_pacing_caps {
  	uint32_t qp_rate_limit_min;
  	uint32_t qp_rate_limit_max; /* In kbps */
  	uint32_t supported_qpts;
  	uint32_t reserved;
  };
  
  struct ibv_exp_device_attr {
  	char			fw_ver[64];
  	uint64_t		node_guid;
  	uint64_t		sys_image_guid;
  	uint64_t		max_mr_size;
  	uint64_t		page_size_cap;
  	uint32_t		vendor_id;
  	uint32_t		vendor_part_id;
  	uint32_t		hw_ver;
  	int			max_qp;
  	int			max_qp_wr;
  	int			reserved; /* place holder to align with ibv_device_attr */
  	int			max_sge;
  	int			max_sge_rd;
  	int			max_cq;
  	int			max_cqe;
  	int			max_mr;
  	int			max_pd;
  	int			max_qp_rd_atom;
  	int			max_ee_rd_atom;
  	int			max_res_rd_atom;
  	int			max_qp_init_rd_atom;
  	int			max_ee_init_rd_atom;
  	enum ibv_exp_atomic_cap	exp_atomic_cap;
  	int			max_ee;
  	int			max_rdd;
  	int			max_mw;
  	int			max_raw_ipv6_qp;
  	int			max_raw_ethy_qp;
  	int			max_mcast_grp;
  	int			max_mcast_qp_attach;
  	int			max_total_mcast_qp_attach;
  	int			max_ah;
  	int			max_fmr;
  	int			max_map_per_fmr;
  	int			max_srq;
  	int			max_srq_wr;
  	int			max_srq_sge;
  	uint16_t		max_pkeys;
  	uint8_t			local_ca_ack_delay;
  	uint8_t			phys_port_cnt;
  	uint32_t                comp_mask;
  	struct ibv_exp_device_calc_cap calc_cap;
  	uint64_t		timestamp_mask;
  	uint64_t		hca_core_clock;
  	uint64_t		exp_device_cap_flags; /* use ibv_exp_device_cap_flags */
  	int			max_dc_req_rd_atom;
  	int			max_dc_res_rd_atom;
  	int			inline_recv_sz;
  	uint32_t		max_rss_tbl_sz;
  	struct ibv_exp_ext_atomics_params ext_atom;
  	struct ibv_exp_umr_caps umr_caps;
  	struct ibv_exp_odp_caps	odp_caps;
  	int			max_dct;
  	int			max_ctx_res_domain;
  	struct ibv_exp_rx_hash_caps	rx_hash_caps;
  	uint32_t			max_wq_type_rq;
  	int 				max_device_ctx;
  	struct ibv_exp_mp_rq_caps	mp_rq_caps;
  	uint16_t		wq_vlan_offloads_cap; /* use ibv_exp_vlan_offloads enum */
  	struct ibv_exp_ec_caps         ec_caps;
  	struct ibv_exp_masked_atomic_params masked_atomic;
  	/*
  	 * The alignment of the padding end address.
  	 * When RX end of packet padding is enabled the device will pad the end
  	 * of RX packet up until the next address which is aligned to the
  	 * rx_pad_end_addr_align size.
  	 * Expected size for this field is according to system cache line size,
  	 * for example 64 or 128. When field is 0 padding is not supported.
  	 */
  	int			rx_pad_end_addr_align;
  	struct ibv_exp_tso_caps	tso_caps;
  	struct ibv_exp_packet_pacing_caps packet_pacing_caps;
  	uint32_t ec_w_mask;
  };
  
  enum {
  	IBV_EXP_ACCESS_FLAGS_SHIFT = 0x0F
  };
  
  enum ibv_exp_access_flags {
  	IBV_EXP_ACCESS_LOCAL_WRITE	= IBV_ACCESS_LOCAL_WRITE,
  	IBV_EXP_ACCESS_REMOTE_WRITE	= IBV_ACCESS_REMOTE_WRITE,
  	IBV_EXP_ACCESS_REMOTE_READ	= IBV_ACCESS_REMOTE_READ,
  	IBV_EXP_ACCESS_REMOTE_ATOMIC	= IBV_ACCESS_REMOTE_ATOMIC,
  	IBV_EXP_ACCESS_MW_BIND		= IBV_ACCESS_MW_BIND,
  
  	IBV_EXP_ACCESS_ALLOCATE_MR		= (IBV_EXP_START_FLAG << 5),
  	IBV_EXP_ACCESS_SHARED_MR_USER_READ	= (IBV_EXP_START_FLAG << 6),
  	IBV_EXP_ACCESS_SHARED_MR_USER_WRITE	= (IBV_EXP_START_FLAG << 7),
  	IBV_EXP_ACCESS_SHARED_MR_GROUP_READ	= (IBV_EXP_START_FLAG << 8),
  	IBV_EXP_ACCESS_SHARED_MR_GROUP_WRITE	= (IBV_EXP_START_FLAG << 9),
  	IBV_EXP_ACCESS_SHARED_MR_OTHER_READ	= (IBV_EXP_START_FLAG << 10),
  	IBV_EXP_ACCESS_SHARED_MR_OTHER_WRITE	= (IBV_EXP_START_FLAG << 11),
  	IBV_EXP_ACCESS_NO_RDMA			= (IBV_EXP_START_FLAG << 12),
  	IBV_EXP_ACCESS_MW_ZERO_BASED		= (IBV_EXP_START_FLAG << 13),
  	IBV_EXP_ACCESS_ON_DEMAND		= (IBV_EXP_START_FLAG << 14),
  	IBV_EXP_ACCESS_RELAXED			= (IBV_EXP_START_FLAG << 15),
  	IBV_EXP_ACCESS_PHYSICAL_ADDR		= (IBV_EXP_START_FLAG << 16),
  	/* set supported bits for validity check */
  	IBV_EXP_ACCESS_RESERVED			= (IBV_EXP_START_FLAG << 17)
  };
  
  /* memory window information struct that is common to types 1 and 2 */
  struct ibv_exp_mw_bind_info {
  	struct ibv_mr	*mr;
  	uint64_t	 addr;
  	uint64_t	 length;
  	uint64_t	 exp_mw_access_flags; /* use ibv_exp_access_flags */
  };
  
  /*
   * Flags for ibv_exp_mw_bind struct comp_mask
   */
  enum ibv_exp_bind_mw_comp_mask {
  	IBV_EXP_BIND_MW_RESERVED	= (1 << 0)
  };
  
  /* type 1 specific info */
  struct ibv_exp_mw_bind {
  	struct ibv_qp			*qp;
  	struct ibv_mw			*mw;
  	uint64_t			 wr_id;
  	uint64_t			 exp_send_flags; /* use ibv_exp_send_flags */
  	struct ibv_exp_mw_bind_info	 bind_info;
  	uint32_t			 comp_mask; /* reserved for future growth (must be 0) */
  };
  
  enum ibv_exp_calc_op {
  	IBV_EXP_CALC_OP_ADD	= 0,
  	IBV_EXP_CALC_OP_MAXLOC,
  	IBV_EXP_CALC_OP_BAND,
  	IBV_EXP_CALC_OP_BXOR,
  	IBV_EXP_CALC_OP_BOR,
  	IBV_EXP_CALC_OP_NUMBER
  };
  
  enum ibv_exp_calc_data_type {
  	IBV_EXP_CALC_DATA_TYPE_INT	= 0,
  	IBV_EXP_CALC_DATA_TYPE_UINT,
  	IBV_EXP_CALC_DATA_TYPE_FLOAT,
  	IBV_EXP_CALC_DATA_TYPE_NUMBER
  };
  
  enum ibv_exp_calc_data_size {
  	IBV_EXP_CALC_DATA_SIZE_64_BIT	= 0,
  	IBV_EXP_CALC_DATA_SIZE_NUMBER
  };
  
  enum ibv_exp_wr_opcode {
  	IBV_EXP_WR_RDMA_WRITE		= IBV_WR_RDMA_WRITE,
  	IBV_EXP_WR_RDMA_WRITE_WITH_IMM	= IBV_WR_RDMA_WRITE_WITH_IMM,
  	IBV_EXP_WR_SEND			= IBV_WR_SEND,
  	IBV_EXP_WR_SEND_WITH_IMM	= IBV_WR_SEND_WITH_IMM,
  	IBV_EXP_WR_RDMA_READ		= IBV_WR_RDMA_READ,
  	IBV_EXP_WR_ATOMIC_CMP_AND_SWP	= IBV_WR_ATOMIC_CMP_AND_SWP,
  	IBV_EXP_WR_ATOMIC_FETCH_AND_ADD	= IBV_WR_ATOMIC_FETCH_AND_ADD,
  	IBV_EXP_WR_LOCAL_INV		= IBV_WR_LOCAL_INV,
  	IBV_EXP_WR_BIND_MW		= IBV_WR_BIND_MW,
  	IBV_EXP_WR_SEND_WITH_INV	= IBV_WR_SEND_WITH_INV,
  
  	IBV_EXP_WR_TSO			= 15 + IBV_EXP_START_ENUM,
  	IBV_EXP_WR_SEND_ENABLE		= 0x20 + IBV_EXP_START_ENUM,
  	IBV_EXP_WR_RECV_ENABLE,
  	IBV_EXP_WR_CQE_WAIT,
  	IBV_EXP_WR_EXT_MASKED_ATOMIC_CMP_AND_SWP,
  	IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD,
  	IBV_EXP_WR_NOP,
  	IBV_EXP_WR_UMR_FILL,
  	IBV_EXP_WR_UMR_INVALIDATE,
  };
  
  enum ibv_exp_send_flags {
  	IBV_EXP_SEND_FENCE		= IBV_SEND_FENCE,
  	IBV_EXP_SEND_SIGNALED		= IBV_SEND_SIGNALED,
  	IBV_EXP_SEND_SOLICITED		= IBV_SEND_SOLICITED,
  	IBV_EXP_SEND_INLINE		= IBV_SEND_INLINE,
  
  	IBV_EXP_SEND_IP_CSUM		= (IBV_EXP_START_FLAG << 0),
  	IBV_EXP_SEND_WITH_CALC		= (IBV_EXP_START_FLAG << 1),
  	IBV_EXP_SEND_WAIT_EN_LAST	= (IBV_EXP_START_FLAG << 2),
  	IBV_EXP_SEND_EXT_ATOMIC_INLINE	= (IBV_EXP_START_FLAG << 3),
  };
  
  struct ibv_exp_cmp_swap {
  	uint64_t        compare_mask;
  	uint64_t        compare_val;
  	uint64_t        swap_val;
  	uint64_t        swap_mask;
  };
  
  struct ibv_exp_fetch_add {
  	uint64_t        add_val;
  	uint64_t        field_boundary;
  };
  
  /*
   * Flags for ibv_exp_send_wr struct comp_mask
   */
  enum ibv_exp_send_wr_comp_mask {
  	IBV_EXP_SEND_WR_ATTR_RESERVED	= 1 << 0
  };
  
  struct ibv_exp_mem_region {
  	uint64_t base_addr;
  	struct ibv_mr *mr;
  	size_t length;
  };
  
  struct ibv_exp_mem_repeat_block {
  	uint64_t base_addr; /* array, size corresponds to ndim */
  	struct ibv_mr *mr;
  	size_t *byte_count; /* array, size corresponds to ndim */
  	size_t *stride; /* array, size corresponds to ndim */
  };
  
  enum ibv_exp_umr_wr_type {
  	IBV_EXP_UMR_MR_LIST,
  	IBV_EXP_UMR_REPEAT
  };
  
  struct ibv_exp_send_wr {
  	uint64_t		wr_id;
  	struct ibv_exp_send_wr *next;
  	struct ibv_sge	       *sg_list;
  	int			num_sge;
  	enum ibv_exp_wr_opcode	exp_opcode; /* use ibv_exp_wr_opcode */
  	int			reserved; /* place holder to align with ibv_send_wr */
  	union {
  		uint32_t	imm_data; /* in network byte order */
  		uint32_t	invalidate_rkey;
  	} ex;
  	union {
  		struct {
  			uint64_t	remote_addr;
  			uint32_t	rkey;
  		} rdma;
  		struct {
  			uint64_t	remote_addr;
  			uint64_t	compare_add;
  			uint64_t	swap;
  			uint32_t	rkey;
  		} atomic;
  		struct {
  			struct ibv_ah  *ah;
  			uint32_t	remote_qpn;
  			uint32_t	remote_qkey;
  		} ud;
  	} wr;
  	union {
  		union {
  			struct {
  				uint32_t    remote_srqn;
  			} xrc;
  		} qp_type;
  
  		uint32_t		xrc_remote_srq_num;
  	};
  	union {
  		struct {
  			uint64_t		remote_addr;
  			uint32_t		rkey;
  		} rdma;
  		struct {
  			uint64_t		remote_addr;
  			uint64_t		compare_add;
  			uint64_t		swap;
  			uint32_t		rkey;
  		} atomic;
  		struct {
  			struct ibv_cq	*cq;
  			int32_t  cq_count;
  		} cqe_wait;
  		struct {
  			struct ibv_qp	*qp;
  			int32_t  wqe_count;
  		} wqe_enable;
  	} task;
  	union {
  		struct {
  			enum ibv_exp_calc_op        calc_op;
  			enum ibv_exp_calc_data_type data_type;
  			enum ibv_exp_calc_data_size data_size;
  		} calc;
  	} op;
  	struct {
  		struct ibv_ah   *ah;
  		uint64_t        dct_access_key;
  		uint32_t        dct_number;
  	} dc;
  	union {
  		struct {
  			struct ibv_mw			*mw;
  			uint32_t			rkey;
  			struct ibv_exp_mw_bind_info	bind_info;
  		} bind_mw;
  		struct {
  			void				*hdr;
  			uint16_t			hdr_sz;
  			uint16_t			mss;
  		} tso;
  	};
  	uint64_t	exp_send_flags; /* use ibv_exp_send_flags */
  	uint32_t	comp_mask; /* reserved for future growth (must be 0) */
  	union {
  		struct {
  			uint32_t umr_type; /* use ibv_exp_umr_wr_type */
  			struct ibv_exp_mkey_list_container *memory_objects; /* used when IBV_EXP_SEND_INLINE is not set */
  			uint64_t exp_access; /* use ibv_exp_access_flags */
  			struct ibv_mr *modified_mr;
  			uint64_t base_addr;
  			uint32_t num_mrs; /* array size of mem_repeat_block_list or mem_reg_list */
  			union {
  				struct ibv_exp_mem_region *mem_reg_list; /* array, size corresponds to num_mrs */
  				struct {
  					struct ibv_exp_mem_repeat_block *mem_repeat_block_list; /* array,  size corresponds to num_mr */
  					size_t *repeat_count; /* array size corresponds to stride_dim */
  					uint32_t stride_dim;
  				} rb;
  			} mem_list;
  		} umr;
  		struct {
  			uint32_t        log_arg_sz;
  			uint64_t	remote_addr;
  			uint32_t	rkey;
  			union {
  				struct {
  					/* For the next four fields:
  					 * If operand_size <= 8 then inline data is immediate
  					 * from the corresponding field; for small opernands,
  					 * ls bits are used.
  					 * Else the fields are pointers in the process's address space
  					 * where arguments are stored
  					 */
  					union {
  						struct ibv_exp_cmp_swap cmp_swap;
  						struct ibv_exp_fetch_add fetch_add;
  					} op;
  				} inline_data;       /* IBV_EXP_SEND_EXT_ATOMIC_INLINE is set */
  				/* in the future add support for non-inline argument provisioning */
  			} wr_data;
  		} masked_atomics;
  	} ext_op;
  };
  
  /*
   * Flags for ibv_exp_values struct comp_mask
   */
  enum ibv_exp_values_comp_mask {
  		IBV_EXP_VALUES_HW_CLOCK_NS	= 1 << 0,
  		IBV_EXP_VALUES_HW_CLOCK		= 1 << 1,
  		IBV_EXP_VALUES_RESERVED		= 1 << 2
  };
  
  struct ibv_exp_values {
  	uint32_t comp_mask;
  	uint64_t hwclock_ns;
  	uint64_t hwclock;
  };
  
  /*
   * Flags for flags field in the ibv_exp_cq_init_attr struct
   */
  enum ibv_exp_cq_create_flags {
  	IBV_EXP_CQ_CREATE_CROSS_CHANNEL		= 1 << 0,
  	IBV_EXP_CQ_TIMESTAMP			= 1 << 1,
  	IBV_EXP_CQ_TIMESTAMP_TO_SYS_TIME	= 1 << 2,
  	IBV_EXP_CQ_COMPRESSED_CQE		= 1 << 3,
  	/*
  	 * note: update IBV_EXP_CQ_CREATE_FLAGS_MASK when adding new fields
  	 */
  };
  
  enum {
  	IBV_EXP_CQ_CREATE_FLAGS_MASK	= IBV_EXP_CQ_CREATE_CROSS_CHANNEL |
  					  IBV_EXP_CQ_TIMESTAMP |
  					  IBV_EXP_CQ_TIMESTAMP_TO_SYS_TIME |
  					  IBV_EXP_CQ_COMPRESSED_CQE,
  };
  
  /*
   * Flags for ibv_exp_cq_init_attr struct comp_mask
   * Set flags only when relevant field is valid
   */
  enum ibv_exp_cq_init_attr_mask {
  	IBV_EXP_CQ_INIT_ATTR_FLAGS		= 1 << 0,
  	IBV_EXP_CQ_INIT_ATTR_RESERVED		= 1 << 1, /* This field is kept for backward compatibility
  							   * of application which use the following to set comp_mask:
  							   * cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_RESERVED - 1
  							   * This kind of setting is no longer accepted and application
  							   * may set only valid known fields, for example:
  							   * cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS |
  							   *				IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN
  							   */
  	IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN		= 1 << 1,
  	IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT	= 1 << 2,
  	IBV_EXP_CQ_INIT_ATTR_RESERVED1		= 1 << 3,
  };
  
  struct ibv_exp_res_domain {
  	struct ibv_context     *context;
  };
  
  struct ibv_exp_cq_init_attr {
  	uint32_t			 comp_mask;
  	uint32_t			 flags;
  	struct ibv_exp_res_domain	*res_domain;
  	struct ibv_exp_peer_direct_attr *peer_direct_attrs;
  };
  
  /*
   * Flags for ibv_exp_ah_attr struct comp_mask
   */
  enum ibv_exp_ah_attr_attr_comp_mask {
  	IBV_EXP_AH_ATTR_LL		= 1 << 0,
  	IBV_EXP_AH_ATTR_VID		= 1 << 1,
  	IBV_EXP_AH_ATTR_RESERVED	= 1 << 2
  };
  
  enum ll_address_type {
  	LL_ADDRESS_UNKNOWN,
  	LL_ADDRESS_IB,
  	LL_ADDRESS_ETH,
  	LL_ADDRESS_SIZE
  };
  
  struct ibv_exp_ah_attr {
  	struct ibv_global_route	grh;
  	uint16_t		dlid;
  	uint8_t			sl;
  	uint8_t			src_path_bits;
  	uint8_t			static_rate;
  	uint8_t			is_global;
  	uint8_t			port_num;
  	uint32_t		comp_mask;
  	struct {
  		enum ll_address_type type;
  		uint32_t	len;
  		char		*address;
  	} ll_address;
  	uint16_t		vid;
  };
  
  /*
   * Flags for exp_attr_mask argument of ibv_exp_modify_qp
   */
  enum ibv_exp_qp_attr_mask {
  	IBV_EXP_QP_STATE		= IBV_QP_STATE,
  	IBV_EXP_QP_CUR_STATE		= IBV_QP_CUR_STATE,
  	IBV_EXP_QP_EN_SQD_ASYNC_NOTIFY	= IBV_QP_EN_SQD_ASYNC_NOTIFY,
  	IBV_EXP_QP_ACCESS_FLAGS		= IBV_QP_ACCESS_FLAGS,
  	IBV_EXP_QP_PKEY_INDEX		= IBV_QP_PKEY_INDEX,
  	IBV_EXP_QP_PORT			= IBV_QP_PORT,
  	IBV_EXP_QP_QKEY			= IBV_QP_QKEY,
  	IBV_EXP_QP_AV			= IBV_QP_AV,
  	IBV_EXP_QP_PATH_MTU		= IBV_QP_PATH_MTU,
  	IBV_EXP_QP_TIMEOUT		= IBV_QP_TIMEOUT,
  	IBV_EXP_QP_RETRY_CNT		= IBV_QP_RETRY_CNT,
  	IBV_EXP_QP_RNR_RETRY		= IBV_QP_RNR_RETRY,
  	IBV_EXP_QP_RQ_PSN		= IBV_QP_RQ_PSN,
  	IBV_EXP_QP_MAX_QP_RD_ATOMIC	= IBV_QP_MAX_QP_RD_ATOMIC,
  	IBV_EXP_QP_ALT_PATH		= IBV_QP_ALT_PATH,
  	IBV_EXP_QP_MIN_RNR_TIMER	= IBV_QP_MIN_RNR_TIMER,
  	IBV_EXP_QP_SQ_PSN		= IBV_QP_SQ_PSN,
  	IBV_EXP_QP_MAX_DEST_RD_ATOMIC	= IBV_QP_MAX_DEST_RD_ATOMIC,
  	IBV_EXP_QP_PATH_MIG_STATE	= IBV_QP_PATH_MIG_STATE,
  	IBV_EXP_QP_CAP			= IBV_QP_CAP,
  	IBV_EXP_QP_DEST_QPN		= IBV_QP_DEST_QPN,
  
  	IBV_EXP_QP_GROUP_RSS		= IBV_EXP_START_FLAG << 21,
  	IBV_EXP_QP_DC_KEY		= IBV_EXP_START_FLAG << 22,
  	IBV_EXP_QP_FLOW_ENTROPY		= IBV_EXP_START_FLAG << 23,
  	IBV_EXP_QP_RATE_LIMIT		= IBV_EXP_START_FLAG << 25,
  };
  
  /*
   * Flags for ibv_exp_qp_attr struct comp_mask
   * Set flags only when relevant field is valid
   */
  enum ibv_exp_qp_attr_comp_mask {
  	IBV_EXP_QP_ATTR_FLOW_ENTROPY	= 1UL << 0,
  	IBV_EXP_QP_ATTR_RESERVED	= 1UL << 1
  };
  
  struct ibv_exp_qp_attr {
  	enum ibv_qp_state	qp_state;
  	enum ibv_qp_state	cur_qp_state;
  	enum ibv_mtu		path_mtu;
  	enum ibv_mig_state	path_mig_state;
  	uint32_t		qkey;
  	uint32_t		rq_psn;
  	uint32_t		sq_psn;
  	uint32_t		dest_qp_num;
  	int			qp_access_flags; /* use ibv_access_flags form verbs.h */
  	struct ibv_qp_cap	cap;
  	struct ibv_ah_attr	ah_attr;
  	struct ibv_ah_attr	alt_ah_attr;
  	uint16_t		pkey_index;
  	uint16_t		alt_pkey_index;
  	uint8_t			en_sqd_async_notify;
  	uint8_t			sq_draining;
  	uint8_t			max_rd_atomic;
  	uint8_t			max_dest_rd_atomic;
  	uint8_t			min_rnr_timer;
  	uint8_t			port_num;
  	uint8_t			timeout;
  	uint8_t			retry_cnt;
  	uint8_t			rnr_retry;
  	uint8_t			alt_port_num;
  	uint8_t			alt_timeout;
  	uint64_t		dct_key;
  	uint32_t		comp_mask; /* reserved for future growth (must be 0) */
  	uint32_t		flow_entropy;
  	uint32_t		rate_limit;
  };
  
  /*
   * Flags for ibv_exp_qp_init_attr struct comp_mask
   * Set flags only when relevant field is valid
   */
  enum ibv_exp_qp_init_attr_comp_mask {
  	IBV_EXP_QP_INIT_ATTR_PD			= 1 << 0,
  	IBV_EXP_QP_INIT_ATTR_XRCD		= 1 << 1,
  	IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS	= 1 << 2,
  	IBV_EXP_QP_INIT_ATTR_INL_RECV		= 1 << 3,
  	IBV_EXP_QP_INIT_ATTR_QPG		= 1 << 4,
  	IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG	= 1 << 5,
  	IBV_EXP_QP_INIT_ATTR_MAX_INL_KLMS       = 1 << 6,
  	IBV_EXP_QP_INIT_ATTR_RESERVED		= 1 << 7, /* This field is kept for backward compatibility
  							   * of application which use the following to set comp_mask:
  							   * qp_init_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_RESERVED - 1
  							   * This kind of setting is no longer accepted and application
  							   * may set only valid known fields, for example:
  							   * qp_init_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD |
  							   *				IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS
  							   */
  	IBV_EXP_QP_INIT_ATTR_RES_DOMAIN         = 1 << 7,
  	IBV_EXP_QP_INIT_ATTR_RX_HASH		= 1 << 8,
  	IBV_EXP_QP_INIT_ATTR_PORT		= 1 << 9,
  	IBV_EXP_QP_INIT_ATTR_PEER_DIRECT	= 1 << 10,
  	IBV_EXP_QP_INIT_ATTR_MAX_TSO_HEADER	= 1 << 11,
  	IBV_EXP_QP_INIT_ATTR_RESERVED1		= 1 << 12,
  };
  
  enum ibv_exp_qpg_type {
  	IBV_EXP_QPG_NONE	= 0,
  	IBV_EXP_QPG_PARENT	= (1<<0),
  	IBV_EXP_QPG_CHILD_RX	= (1<<1),
  	IBV_EXP_QPG_CHILD_TX	= (1<<2)
  };
  
  struct ibv_exp_qpg_init_attrib {
  	uint32_t tss_child_count;
  	uint32_t rss_child_count;
  };
  
  struct ibv_exp_qpg {
  	uint32_t qpg_type;
  	union {
  		struct ibv_qp *qpg_parent; /* see qpg_type */
  		struct ibv_exp_qpg_init_attrib parent_attrib;
  	};
  };
  
  /*
   * RX Hash Function flags.
  */
  enum ibv_exp_rx_hash_function_flags {
  	IBV_EXP_RX_HASH_FUNC_TOEPLITZ	= 1 << 0,
  	IBV_EXP_RX_HASH_FUNC_XOR	= 1 << 1
  };
  
  /*
   * RX Hash flags, these flags allows to set which incoming packet field should
   * participates in RX Hash. Each flag represent certain packet's field,
   * when the flag is set the field that is represented by the flag will
   * participate in RX Hash calculation.
   * Notice: *IPV4 and *IPV6 flags can't be enabled together on the same QP
   * and *TCP and *UDP flags can't be enabled together on the same QP.
  */
  enum ibv_exp_rx_hash_fields {
  	IBV_EXP_RX_HASH_SRC_IPV4		= 1 << 0,
  	IBV_EXP_RX_HASH_DST_IPV4		= 1 << 1,
  	IBV_EXP_RX_HASH_SRC_IPV6		= 1 << 2,
  	IBV_EXP_RX_HASH_DST_IPV6		= 1 << 3,
  	IBV_EXP_RX_HASH_SRC_PORT_TCP	= 1 << 4,
  	IBV_EXP_RX_HASH_DST_PORT_TCP	= 1 << 5,
  	IBV_EXP_RX_HASH_SRC_PORT_UDP	= 1 << 6,
  	IBV_EXP_RX_HASH_DST_PORT_UDP	= 1 << 7
  };
  
  /*
   * RX Hash QP configuration. Sets hash function, hash types and
   * Indirection table for QPs with enabled IBV_QP_INIT_ATTR_RX_HASH flag.
  */
  struct ibv_exp_rx_hash_conf {
  	/* enum ibv_exp_rx_hash_function_flags */
  	uint8_t				rx_hash_function;
  	/* valid only for Toeplitz */
  	uint8_t			rx_hash_key_len;
  	uint8_t                        *rx_hash_key;
  	/* enum ibv_exp_rx_hash_fields */
  	uint64_t			rx_hash_fields_mask;
  	struct ibv_exp_rwq_ind_table       *rwq_ind_tbl;
  };
  
  /*
   * Flags for exp_create_flags field in ibv_exp_qp_init_attr struct
   */
  enum ibv_exp_qp_create_flags {
  	IBV_EXP_QP_CREATE_CROSS_CHANNEL        = (1 << 2),
  	IBV_EXP_QP_CREATE_MANAGED_SEND         = (1 << 3),
  	IBV_EXP_QP_CREATE_MANAGED_RECV         = (1 << 4),
  	IBV_EXP_QP_CREATE_IGNORE_SQ_OVERFLOW   = (1 << 6),
  	IBV_EXP_QP_CREATE_IGNORE_RQ_OVERFLOW   = (1 << 7),
  	IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY      = (1 << 8),
  	IBV_EXP_QP_CREATE_UMR		       = (1 << 9),
  	IBV_EXP_QP_CREATE_EC_PARITY_EN	       = (1 << 10),
  	IBV_EXP_QP_CREATE_RX_END_PADDING       = (1 << 11),
  	IBV_EXP_QP_CREATE_SCATTER_FCS          = (1 << 12),
  	IBV_EXP_QP_CREATE_INTERNAL_USE	       = (1 << 15),
  	/* set supported bits for validity check */
  	IBV_EXP_QP_CREATE_MASK                 = (0x00001FDC)
  };
  
  struct ibv_exp_qp_init_attr {
  	void		       *qp_context;
  	struct ibv_cq	       *send_cq;
  	struct ibv_cq	       *recv_cq;
  	struct ibv_srq	       *srq;
  	struct ibv_qp_cap	cap;
  	enum ibv_qp_type	qp_type;
  	int			sq_sig_all;
  
  	uint32_t		comp_mask; /* use ibv_exp_qp_init_attr_comp_mask */
  	struct ibv_pd	       *pd;
  	struct ibv_xrcd	       *xrcd;
  	uint32_t		exp_create_flags; /* use ibv_exp_qp_create_flags */
  
  	uint32_t		max_inl_recv;
  	struct ibv_exp_qpg	qpg;
  	uint32_t		max_atomic_arg;
  	uint32_t                max_inl_send_klms;
  	struct ibv_exp_res_domain *res_domain;
  	struct ibv_exp_rx_hash_conf *rx_hash_conf;
  	uint8_t port_num;
  	struct ibv_exp_peer_direct_attr *peer_direct_attrs;
  	uint16_t		max_tso_header;
  };
  
  /*
   * Flags for ibv_exp_dct_init_attr struct comp_mask
   */
  enum ibv_exp_dct_init_attr_comp_mask {
  	IBV_EXP_DCT_INIT_ATTR_RESERVED	= 1 << 0
  };
  
  enum {
  	IBV_EXP_DCT_CREATE_FLAGS_MASK		= (1 << 0) - 1,
  };
  
  struct ibv_exp_dct_init_attr {
  	struct ibv_pd	       *pd;
  	struct ibv_cq	       *cq;
  	struct ibv_srq	       *srq;
  	uint64_t		dc_key;
  	uint8_t			port;
  	uint32_t		access_flags; /* use ibv_access_flags form verbs.h */
  	uint8_t			min_rnr_timer;
  	uint8_t			tclass;
  	uint32_t		flow_label;
  	enum ibv_mtu		mtu;
  	uint8_t			pkey_index;
  	uint8_t			gid_index;
  	uint8_t			hop_limit;
  	uint32_t		inline_size;
  	uint32_t		create_flags;
  	uint32_t		comp_mask; /* reserved for future growth (must be 0) */
  };
  
  enum {
  	IBV_EXP_DCT_STATE_ACTIVE	= 0,
  	IBV_EXP_DCT_STATE_DRAINING	= 1,
  	IBV_EXP_DCT_STATE_DRAINED	= 2
  };
  
  /*
   * Flags for ibv_exp_dct_attr struct comp_mask
   */
  enum ibv_exp_dct_attr_comp_mask {
  	IBV_EXP_DCT_ATTR_RESERVED	= 1 << 0
  };
  
  struct ibv_exp_dct_attr {
  	uint64_t		dc_key;
  	uint8_t			port;
  	uint32_t		access_flags; /* use ibv_access_flags form verbs.h */
  	uint8_t			min_rnr_timer;
  	uint8_t			tclass;
  	uint32_t		flow_label;
  	enum ibv_mtu		mtu;
  	uint8_t			pkey_index;
  	uint8_t			gid_index;
  	uint8_t			hop_limit;
  	uint32_t		key_violations;
  	uint8_t			state;
  	struct ibv_srq	       *srq;
  	struct ibv_cq	       *cq;
  	struct ibv_pd	       *pd;
  	uint32_t		comp_mask; /* reserved for future growth (must be 0) */
  };
  
  enum {
  	IBV_EXP_QUERY_PORT_STATE		= 1 << 0,
  	IBV_EXP_QUERY_PORT_MAX_MTU		= 1 << 1,
  	IBV_EXP_QUERY_PORT_ACTIVE_MTU		= 1 << 2,
  	IBV_EXP_QUERY_PORT_GID_TBL_LEN		= 1 << 3,
  	IBV_EXP_QUERY_PORT_CAP_FLAGS		= 1 << 4,
  	IBV_EXP_QUERY_PORT_MAX_MSG_SZ		= 1 << 5,
  	IBV_EXP_QUERY_PORT_BAD_PKEY_CNTR	= 1 << 6,
  	IBV_EXP_QUERY_PORT_QKEY_VIOL_CNTR	= 1 << 7,
  	IBV_EXP_QUERY_PORT_PKEY_TBL_LEN		= 1 << 8,
  	IBV_EXP_QUERY_PORT_LID			= 1 << 9,
  	IBV_EXP_QUERY_PORT_SM_LID		= 1 << 10,
  	IBV_EXP_QUERY_PORT_LMC			= 1 << 11,
  	IBV_EXP_QUERY_PORT_MAX_VL_NUM		= 1 << 12,
  	IBV_EXP_QUERY_PORT_SM_SL		= 1 << 13,
  	IBV_EXP_QUERY_PORT_SUBNET_TIMEOUT	= 1 << 14,
  	IBV_EXP_QUERY_PORT_INIT_TYPE_REPLY	= 1 << 15,
  	IBV_EXP_QUERY_PORT_ACTIVE_WIDTH		= 1 << 16,
  	IBV_EXP_QUERY_PORT_ACTIVE_SPEED		= 1 << 17,
  	IBV_EXP_QUERY_PORT_PHYS_STATE		= 1 << 18,
  	IBV_EXP_QUERY_PORT_LINK_LAYER		= 1 << 19,
  	/* mask of the fields that exists in the standard query_port_command */
  	IBV_EXP_QUERY_PORT_STD_MASK		= (1 << 20) - 1,
  	/* mask of all supported fields */
  	IBV_EXP_QUERY_PORT_MASK			= IBV_EXP_QUERY_PORT_STD_MASK,
  };
  
  /*
   * Flags for ibv_exp_port_attr struct comp_mask
   * Set flags only when relevant field is valid
   */
  enum ibv_exp_query_port_attr_comp_mask {
  	IBV_EXP_QUERY_PORT_ATTR_MASK1		= 1 << 0,
  	IBV_EXP_QUERY_PORT_ATTR_RESERVED	= 1 << 1,
  
  	IBV_EXP_QUERY_PORT_ATTR_MASKS		= IBV_EXP_QUERY_PORT_ATTR_RESERVED - 1
  };
  
  struct ibv_exp_port_attr {
  	union {
  		struct {
  			enum ibv_port_state	state;
  			enum ibv_mtu		max_mtu;
  			enum ibv_mtu		active_mtu;
  			int			gid_tbl_len;
  			uint32_t		port_cap_flags;
  			uint32_t		max_msg_sz;
  			uint32_t		bad_pkey_cntr;
  			uint32_t		qkey_viol_cntr;
  			uint16_t		pkey_tbl_len;
  			uint16_t		lid;
  			uint16_t		sm_lid;
  			uint8_t			lmc;
  			uint8_t			max_vl_num;
  			uint8_t			sm_sl;
  			uint8_t			subnet_timeout;
  			uint8_t			init_type_reply;
  			uint8_t			active_width;
  			uint8_t			active_speed;
  			uint8_t			phys_state;
  			uint8_t			link_layer;
  			uint8_t			reserved;
  		};
  		struct ibv_port_attr		port_attr;
  	};
  	uint32_t		comp_mask;
  	uint32_t		mask1;
  };
  
  enum ibv_exp_cq_attr_mask {
  	IBV_EXP_CQ_MODERATION                  = 1 << 0,
  	IBV_EXP_CQ_CAP_FLAGS                   = 1 << 1
  };
  
  enum ibv_exp_cq_cap_flags {
  	IBV_EXP_CQ_IGNORE_OVERRUN              = (1 << 0),
  	/* set supported bits for validity check */
  	IBV_EXP_CQ_CAP_MASK                    = (0x00000001)
  };
  
  /*
   * Flags for ibv_exp_cq_attr struct comp_mask
   * Set flags only when relevant field is valid
   */
  enum ibv_exp_cq_attr_comp_mask {
  	IBV_EXP_CQ_ATTR_MODERATION	= (1 << 0),
  	IBV_EXP_CQ_ATTR_CQ_CAP_FLAGS	= (1 << 1),
  	/* set supported bits for validity check */
  	IBV_EXP_CQ_ATTR_RESERVED	= (1 << 2)
  };
  
  struct ibv_exp_cq_attr {
  	uint32_t                comp_mask;
  	struct {
  		uint16_t        cq_count;
  		uint16_t        cq_period;
  	} moderation;
  	uint32_t		cq_cap_flags;
  };
  
  /*
   * Flags for ibv_exp_reg_shared_mr_in struct comp_mask
   */
  enum ibv_exp_reg_shared_mr_comp_mask {
  	IBV_EXP_REG_SHARED_MR_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_reg_shared_mr_in {
  	uint32_t mr_handle;
  	struct ibv_pd *pd;
  	void *addr;
  	uint64_t exp_access; /* use ibv_exp_access_flags */
  	uint32_t comp_mask; /* reserved for future growth (must be 0) */
  };
  
  enum ibv_exp_flow_flags {
  	IBV_EXP_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1,
  };
  
  enum ibv_exp_flow_attr_type {
  	/* steering according to rule specifications */
  	IBV_EXP_FLOW_ATTR_NORMAL		= 0x0,
  	/* default unicast and multicast rule -
  	 * receive all Eth traffic which isn't steered to any QP
  	 */
  	IBV_EXP_FLOW_ATTR_ALL_DEFAULT	= 0x1,
  	/* default multicast rule -
  	 * receive all Eth multicast traffic which isn't steered to any QP
  	 */
  	IBV_EXP_FLOW_ATTR_MC_DEFAULT	= 0x2,
  	/* sniffer rule - receive all port traffic */
  	IBV_EXP_FLOW_ATTR_SNIFFER		= 0x3,
  };
  
  enum ibv_exp_flow_spec_type {
  	IBV_EXP_FLOW_SPEC_ETH		= 0x20,
  	IBV_EXP_FLOW_SPEC_IB		= 0x21,
  	IBV_EXP_FLOW_SPEC_IPV4		= 0x30,
  	IBV_EXP_FLOW_SPEC_IPV6		= 0x31,
  	IBV_EXP_FLOW_SPEC_IPV4_EXT	= 0x32,
  	IBV_EXP_FLOW_SPEC_IPV6_EXT	= 0x33,
  	IBV_EXP_FLOW_SPEC_TCP		= 0x40,
  	IBV_EXP_FLOW_SPEC_UDP		= 0x41,
  	IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL	= 0x50,
  	IBV_EXP_FLOW_SPEC_INNER		= 0x100,
  	IBV_EXP_FLOW_SPEC_ACTION_TAG	= 0x1000,
  };
  
  struct ibv_exp_flow_eth_filter {
  	uint8_t		dst_mac[6];
  	uint8_t		src_mac[6];
  	uint16_t	ether_type;
  	/*
  	 * same layout as 802.1q: prio 3, cfi 1, vlan id 12
  	 */
  	uint16_t	vlan_tag;
  };
  
  struct ibv_exp_flow_spec_eth {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_eth_filter val;
  	struct ibv_exp_flow_eth_filter mask;
  };
  
  struct ibv_exp_flow_ib_filter {
  	uint32_t qpn;
  	uint8_t  dst_gid[16];
  };
  
  struct ibv_exp_flow_spec_ib {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_ib_filter val;
  	struct ibv_exp_flow_ib_filter mask;
  };
  
  struct ibv_exp_flow_ipv4_filter {
  	uint32_t src_ip;
  	uint32_t dst_ip;
  };
  
  struct ibv_exp_flow_spec_ipv4 {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_ipv4_filter val;
  	struct ibv_exp_flow_ipv4_filter mask;
  };
  
  struct ibv_exp_flow_ipv6_filter {
  	uint8_t src_ip[16];
  	uint8_t dst_ip[16];
  };
  
  struct ibv_exp_flow_spec_ipv6 {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_ipv6_filter val;
  	struct ibv_exp_flow_ipv6_filter mask;
  };
  
  struct ibv_exp_flow_spec_action_tag {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	uint32_t  tag_id;
  };
  
  struct ibv_exp_flow_ipv6_ext_filter {
  	uint8_t src_ip[16];
  	uint8_t dst_ip[16];
  	uint32_t flow_label;
  	uint8_t  next_hdr;
  	uint8_t  traffic_class;
  	uint8_t  hop_limit;
  };
  
  struct ibv_exp_flow_spec_ipv6_ext {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_ipv6_ext_filter val;
  	struct ibv_exp_flow_ipv6_ext_filter mask;
  };
  
  struct ibv_exp_flow_ipv4_ext_filter {
  	uint32_t src_ip;
  	uint32_t dst_ip;
  	uint8_t  proto;
  	uint8_t  tos;
  	uint8_t  ttl;
  	uint8_t  flags;
  };
  
  struct ibv_exp_flow_spec_ipv4_ext {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_ipv4_ext_filter val;
  	struct ibv_exp_flow_ipv4_ext_filter mask;
  };
  
  struct ibv_exp_flow_tcp_udp_filter {
  	uint16_t dst_port;
  	uint16_t src_port;
  };
  
  struct ibv_exp_flow_spec_tcp_udp {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_tcp_udp_filter val;
  	struct ibv_exp_flow_tcp_udp_filter mask;
  };
  
  struct ibv_exp_flow_tunnel_filter {
  	uint32_t tunnel_id;
  };
  
  struct ibv_exp_flow_spec_tunnel {
  	enum ibv_exp_flow_spec_type  type;
  	uint16_t  size;
  	struct ibv_exp_flow_tunnel_filter val;
  	struct ibv_exp_flow_tunnel_filter mask;
  };
  
  struct ibv_exp_flow_spec {
  	union {
  		struct {
  			uint32_t type;
  			uint16_t size;
  		} hdr;
  		struct ibv_exp_flow_spec_ib ib;
  		struct ibv_exp_flow_spec_eth eth;
  		struct ibv_exp_flow_spec_ipv4 ipv4;
  		struct ibv_exp_flow_spec_ipv4_ext ipv4_ext;
  		struct ibv_exp_flow_spec_tcp_udp tcp_udp;
  		struct ibv_exp_flow_spec_ipv6 ipv6;
  		struct ibv_exp_flow_spec_ipv6_ext ipv6_ext;
  		struct ibv_exp_flow_spec_tunnel tunnel;
  		struct ibv_exp_flow_spec_action_tag flow_tag;
  	};
  };
  
  struct ibv_exp_flow_attr {
  	enum ibv_exp_flow_attr_type type;
  	uint16_t size;
  	uint16_t priority;
  	uint8_t num_of_specs;
  	uint8_t port;
  	uint32_t flags;
  	/* Following are the optional layers according to user request
  	 * struct ibv_exp_flow_spec_xxx [L2]
  	 * struct ibv_exp_flow_spec_yyy [L3/L4]
  	 */
  	uint64_t reserved; /* reserved for future growth (must be 0) */
  };
  
  struct ibv_exp_flow {
  	struct ibv_context *context;
  	uint32_t	   handle;
  };
  
  struct ibv_exp_dct {
  	struct ibv_context     *context;
  	uint32_t		handle;
  	uint32_t		dct_num;
  	struct ibv_pd	       *pd;
  	struct ibv_srq	       *srq;
  	struct ibv_cq	       *cq;
  	pthread_mutex_t		mutex;
  	pthread_cond_t		cond;
  	uint32_t		events_completed;
  };
  
  enum ibv_exp_wc_opcode {
  	IBV_EXP_WC_SEND,
  	IBV_EXP_WC_RDMA_WRITE,
  	IBV_EXP_WC_RDMA_READ,
  	IBV_EXP_WC_COMP_SWAP,
  	IBV_EXP_WC_FETCH_ADD,
  	IBV_EXP_WC_BIND_MW,
  	IBV_EXP_WC_LOCAL_INV,
  	IBV_EXP_WC_MASKED_COMP_SWAP	=	9,
  	IBV_EXP_WC_MASKED_FETCH_ADD	=	10,
  	IBV_EXP_WC_TSO,
  	IBV_EXP_WC_UMR			=	0x100,
  /*
   * Set value of IBV_EXP_WC_RECV so consumers can test if a completion is a
   * receive by testing (opcode & IBV_EXP_WC_RECV).
   */
  	IBV_EXP_WC_RECV			= 1 << 7,
  	IBV_EXP_WC_RECV_RDMA_WITH_IMM
  };
  
  enum ibv_exp_wc_flags {
  	IBV_EXP_WC_GRH		= IBV_WC_GRH,
  	IBV_EXP_WC_WITH_IMM	= IBV_WC_WITH_IMM,
  	IBV_EXP_WC_WITH_INV	= IBV_WC_WITH_INV,
  
  	IBV_EXP_WC_WITH_SL			= IBV_EXP_START_FLAG << 4,
  	IBV_EXP_WC_WITH_SLID			= IBV_EXP_START_FLAG << 5,
  	IBV_EXP_WC_WITH_TIMESTAMP		= IBV_EXP_START_FLAG << 6,
  	IBV_EXP_WC_QP				= IBV_EXP_START_FLAG << 7,
  	IBV_EXP_WC_SRQ				= IBV_EXP_START_FLAG << 8,
  	IBV_EXP_WC_DCT				= IBV_EXP_START_FLAG << 9,
  	IBV_EXP_WC_RX_IP_CSUM_OK		= IBV_EXP_START_FLAG << 10,
  	IBV_EXP_WC_RX_TCP_UDP_CSUM_OK		= IBV_EXP_START_FLAG << 11,
  	IBV_EXP_WC_RX_IPV4_PACKET		= IBV_EXP_START_FLAG << 12,
  	IBV_EXP_WC_RX_IPV6_PACKET		= IBV_EXP_START_FLAG << 13,
  	IBV_EXP_WC_RX_TUNNEL_PACKET		= IBV_EXP_START_FLAG << 14,
  	IBV_EXP_WC_RX_OUTER_IP_CSUM_OK		= IBV_EXP_START_FLAG << 15,
  	IBV_EXP_WC_RX_OUTER_TCP_UDP_CSUM_OK	= IBV_EXP_START_FLAG << 16,
  	IBV_EXP_WC_RX_OUTER_IPV4_PACKET		= IBV_EXP_START_FLAG << 17,
  	IBV_EXP_WC_RX_OUTER_IPV6_PACKET		= IBV_EXP_START_FLAG << 18,
  };
  
  struct ibv_exp_wc {
  	uint64_t		wr_id;
  	enum ibv_wc_status	status;
  	enum ibv_exp_wc_opcode	exp_opcode;
  	uint32_t		vendor_err;
  	uint32_t		byte_len;
  	uint32_t		imm_data; /* in network byte order */
  	uint32_t		qp_num;
  	uint32_t		src_qp;
  	int			reserved; /* place holder to align with ibv_wc */
  	uint16_t		pkey_index;
  	uint16_t		slid; /* invalid when TS is used */
  	uint8_t			sl; /* invalid when TS is used */
  	uint8_t			dlid_path_bits;
  	uint64_t		timestamp;
  	struct ibv_qp	       *qp;
  	struct ibv_srq	       *srq;
  	struct ibv_exp_dct     *dct;
  	uint64_t		exp_wc_flags; /* use ibv_exp_wc_flags */
  };
  
  /*
   * Flags for ibv_exp_prefetch_mr comp_mask
   */
  enum ibv_exp_prefetch_attr_comp_mask {
  	IBV_EXP_PREFETCH_MR_RESERVED	= (1 << 0),
  };
  
  /*
   * Flags for ibv_exp_prefetch_mr flags
   */
  enum ibv_exp_prefetch_attr_flags {
  	/* request prefetching for write access. Used for both local and remote */
  	IBV_EXP_PREFETCH_WRITE_ACCESS = (1 << 0),
  };
  
  struct ibv_exp_prefetch_attr {
  	/* Use enum ibv_exp_prefetch_attr_flags */
  	uint32_t flags;
  	/* Address of the area to prefetch */
  	void *addr;
  	/* Length of the area to prefetch */
  	size_t length;
  	uint32_t comp_mask;
  };
  
  /*
   * Flags for ibv_exp_reg_mr_in struct comp_mask
   */
  enum ibv_exp_reg_mr_in_comp_mask {
  	/* set supported bits for validity check */
  	IBV_EXP_REG_MR_CREATE_FLAGS	= (1 << 0),
  	IBV_EXP_REG_MR_RESERVED		= (1 << 1)
  };
  
  enum ibv_exp_reg_mr_create_flags {
  	IBV_EXP_REG_MR_CREATE_CONTIG		= (1 << 0)  /* register mr with contiguous pages */
  };
  
  struct ibv_exp_reg_mr_in {
  	struct ibv_pd *pd;
  	void *addr;
  	size_t length;
  	uint64_t exp_access; /* use ibv_exp_access_flags */
  	uint32_t comp_mask; /* reserved for future growth (must be 0) */
  	uint32_t create_flags; /* use ibv_exp_reg_mr_create_flags */
  };
  
  
  enum ibv_exp_task_type {
  	IBV_EXP_TASK_SEND         = 0,
  	IBV_EXP_TASK_RECV         = 1
  };
  
  /*
   * Flags for ibv_exp_task struct comp_mask
   */
  enum ibv_exp_task_comp_mask {
  	IBV_EXP_TASK_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_task {
  	enum ibv_exp_task_type	task_type;
  	struct {
  		struct ibv_qp  *qp;
  		union {
  			struct ibv_exp_send_wr  *send_wr;
  			struct ibv_recv_wr  *recv_wr;
  		};
  	} item;
  	struct ibv_exp_task    *next;
  	uint32_t                comp_mask; /* reserved for future growth (must be 0) */
  };
  
  /*
   * Flags for ibv_exp_arm_attr struct comp_mask
   */
  enum ibv_exp_arm_attr_comp_mask {
  	IBV_EXP_ARM_ATTR_RESERVED	= (1 << 0)
  };
  struct ibv_exp_arm_attr {
  	uint32_t	comp_mask; /* reserved for future growth (must be 0) */
  };
  
  enum ibv_exp_mr_create_flags {
  	IBV_EXP_MR_SIGNATURE_EN		= (1 << 0),
  	IBV_EXP_MR_INDIRECT_KLMS	= (1 << 1)
  };
  
  struct ibv_exp_mr_init_attr {
  	uint32_t max_klm_list_size; /* num of entries */
  	uint32_t create_flags; /* use ibv_exp_mr_create_flags */
  	uint64_t exp_access_flags; /* use ibv_exp_access_flags */
  };
  
  /*
   * Comp_mask for ibv_exp_create_mr_in struct comp_mask
   */
  enum ibv_exp_create_mr_in_comp_mask {
  	IBV_EXP_CREATE_MR_IN_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_create_mr_in {
  	struct ibv_pd *pd;
  	struct ibv_exp_mr_init_attr attr;
  	uint32_t comp_mask; /* use ibv_exp_create_mr_in_comp_mask */
  };
  
  /*
   * Flags for ibv_exp_mkey_attr struct comp_mask
   */
  enum ibv_exp_mkey_attr_comp_mask {
  	IBV_EXP_MKEY_ATTR_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_mkey_attr {
  	uint32_t max_klm_list_size;
  	uint32_t comp_mask; /* use ibv_exp_mkey_attr_comp_mask */
  };
  
  struct ibv_exp_mkey_list_container {
  	uint32_t max_klm_list_size;
  	struct ibv_context *context;
  };
  
  enum ibv_exp_mkey_list_type {
  	IBV_EXP_MKEY_LIST_TYPE_INDIRECT_MR
  };
  
  /*
   * Flags for ibv_exp_mkey_list_container_attr struct comp_mask
   */
  enum ibv_exp_alloc_mkey_list_comp_mask {
  	IBV_EXP_MKEY_LIST_CONTAINER_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_mkey_list_container_attr {
  	struct ibv_pd *pd;
  	uint32_t mkey_list_type;  /* use ibv_exp_mkey_list_type */
  	uint32_t max_klm_list_size;
  	uint32_t comp_mask; /*use ibv_exp_alloc_mkey_list_comp_mask */
  };
  
  /*
   * Flags for ibv_exp_dereg_out struct comp_mask
   */
  enum ibv_exp_dereg_mr_comp_mask {
  	IBV_EXP_DEREG_MR_RESERVED	= (1 << 0)
  };
  
  struct ibv_exp_dereg_out {
  	int need_dofork;
  	uint32_t comp_mask; /* use ibv_exp_dereg_mr_comp_mask */
  };
  
  struct verbs_env_item {
  	char			*name;
  	char			*value;
  	struct verbs_env_item	*next;
  };
  
  struct verbs_environment {
  	struct verbs_env_item  *head;
  	pthread_mutex_t		mtx;
  };
  
  /* RSS stuff */
  
  enum ibv_exp_wq_type {
  	IBV_EXP_WQT_RQ,
  	IBV_EXP_WQT_SRQ
  };
  
  enum ibv_exp_wq_state {
  	IBV_EXP_WQS_RESET,
  	IBV_EXP_WQS_RDY,
  	IBV_EXP_WQS_ERR,
  	IBV_EXP_WQS_UNKNOWN
  };
  
  /* VLAN Offloads */
  enum ibv_exp_vlan_offloads {
  	/* Represents C-VLAN stripping feature */
  	IBV_EXP_RECEIVE_WQ_CVLAN_STRIP 			= (1 << 0),
  	/* Represents C-VLAN insertion feature*/
  	IBV_EXP_RECEIVE_WQ_CVLAN_INSERTION 		= (1 << 1),
  	IBV_EXP_RECEIVE_WQ_VLAN_OFFLOADS_RESERVED 	= (1 << 2),
  };
  
  /*
   * Work Queue. QP can be created without internal WQs "packaged" inside it,
   * this QPs can be configured to use "external" WQ object as its
   * receive/send queue.
   * WQ associated (many to one) with Completion Queue it owns WQ properties
   * (PD, WQ size etc).
   * WQ of type IBV_EXP_WQT_RQ contains receive WQEs, in which case its PD serves
   * scatter as well.
   * WQ of type IBV_EXP_WQT_SRQ is associated (many to one) with regular ibv_srq,
   * in which case it does not hold receive WQEs.
   * QPs can be associated with IBV_EXP_WQT_S/RQ WQs via WQ Indirection Table.
   */
  struct ibv_exp_wq {
  	struct ibv_context     *context;
  	void		       *wq_context; /* Associated Context of the WQ */
  	uint32_t		handle;
  	/* Protection domain WQ should be associated with */
  	struct	ibv_pd	       *pd;
  	/* CQ to be associated with the WQ */
  	struct	ibv_cq	       *cq;
  	/* SRQ handle if WQ is to be associated with an SRQ, otherwise NULL */
  	struct	ibv_srq	       *srq;
  	uint32_t		wq_num;
  	enum ibv_exp_wq_state       state;
  	enum ibv_exp_wq_type	wq_type;
  	uint32_t		comp_mask;
  };
  
  enum ibv_exp_wq_init_attr_mask {
  	IBV_EXP_CREATE_WQ_RES_DOMAIN	= (1 << 0),
  	IBV_EXP_CREATE_WQ_MP_RQ		= (1 << 1),
  	IBV_EXP_CREATE_WQ_VLAN_OFFLOADS = (1 << 2),
  	IBV_EXP_CREATE_WQ_FLAGS		= (1 << 3),
  	IBV_EXP_CREATE_WQ_RESERVED	= (1 << 4)
  };
  
  struct ibv_exp_wq_mp_rq {
  	enum ibv_exp_mp_rq_shifts	use_shift;
  	uint8_t				single_wqe_log_num_of_strides;
  	uint8_t				single_stride_log_num_of_bytes;
  };
  
  enum ibv_exp_wq_init_attr_flags {
  	IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING	= (1ULL << 0),
  	IBV_EXP_CREATE_WQ_FLAG_SCATTER_FCS	= (1ULL << 1),
  	IBV_EXP_CREATE_WQ_FLAG_RESERVED		= (1ULL << 2)
  };
  
  struct ibv_exp_wq_init_attr {
  	/* Associated Context of the WQ */
  	void		       *wq_context;
  	enum ibv_exp_wq_type	wq_type;
  	/* Valid for non IBV_EXP_WQT_SRQ WQ */
  	uint32_t		max_recv_wr;
  	/* Valid for non IBV_EXP_WQT_SRQ WQ */
  	uint32_t		max_recv_sge;
  	/* Protection domain WQ should be associated with */
  	struct	ibv_pd	       *pd;
  	/* CQ to be associated with the WQ */
  	struct	ibv_cq	       *cq;
  	/* SRQ handle if WQ is of type IBV_EXP_WQT_SRQ, otherwise NULL */
  	struct	ibv_srq	       *srq;
  	/* refers to ibv_exp_wq_init_attr_mask */
  	uint32_t		comp_mask;
  	struct ibv_exp_res_domain *res_domain;
  	struct ibv_exp_wq_mp_rq	mp_rq;
  	uint16_t		vlan_offloads; /* use ibv_exp_vlan_offloads enum */
  	uint64_t		flags; /* general wq create flags */
  };
  
  enum ibv_exp_wq_attr_mask {
  	IBV_EXP_WQ_ATTR_STATE		= 1 << 0,
  	IBV_EXP_WQ_ATTR_CURR_STATE	= 1 << 1,
  	IBV_EXP_WQ_ATTR_VLAN_OFFLOADS	= 1 << 2,
  	IBV_EXP_WQ_ATTR_RESERVED	= 1 << 3
  };
  
  struct ibv_exp_wq_attr {
  	/* enum ibv_exp_wq_attr_mask */
  	uint32_t		attr_mask;
  	/* Move the RQ to this state */
  	enum	ibv_exp_wq_state	wq_state;
  	/* Assume this is the current RQ state */
  	enum	ibv_exp_wq_state	curr_wq_state;
  	uint16_t		vlan_offloads; /* use ibv_exp_vlan_offloads enum */
  };
  
  /*
   * Receive Work Queue Indirection Table.
   * It's used in order to distribute incoming packets between different
   * Receive Work Queues. Associating Receive WQs with different CPU cores
   * allows to workload the traffic between different CPU cores.
   * The Indirection Table can contain only WQs of type IBV_EXP_WQT_S/RQ.
  */
  struct ibv_exp_rwq_ind_table {
  	struct ibv_context *context;
  	struct ibv_pd *pd;
  	int ind_tbl_handle;
  	int ind_tbl_num;
  	uint32_t comp_mask;
  };
  
  enum ibv_exp_ind_table_init_attr_mask {
  	IBV_EXP_CREATE_IND_TABLE_RESERVED = (1 << 0)
  };
  
  /*
   * Receive Work Queue Indirection Table attributes
  */
  struct ibv_exp_rwq_ind_table_init_attr {
  	struct ibv_pd *pd;
  	/* Log, base 2, of Indirection table size */
  	uint32_t log_ind_tbl_size;
  	/* Each entry is a pointer to Receive Work Queue */
  	struct ibv_exp_wq **ind_tbl;
  	uint32_t comp_mask;
  };
  
  /* Accelerated verbs */
  enum ibv_exp_thread_model {
  	IBV_EXP_THREAD_SAFE,	/* The lib responsible to protect the object in multithreaded environment */
  	IBV_EXP_THREAD_UNSAFE,	/* The application responsible to protect the object in multithreaded environment */
  	IBV_EXP_THREAD_SINGLE	/* The object is called from only one thread */
  };
  
  enum ibv_exp_msg_model {
  	IBV_EXP_MSG_DEFAULT,		/* Use the provider default message model */
  	IBV_EXP_MSG_LOW_LATENCY,	/* Hint the provider to optimize for low latency */
  	IBV_EXP_MSG_HIGH_BW,		/* Hint the provider to optimize for high bandwidth */
  	IBV_EXP_MSG_FORCE_LOW_LATENCY,	/* Force the provider to optimize for low latency */
  };
  
  /*
   * Resource domains
   */
  enum ibv_exp_res_domain_init_attr_comp_mask {
  	IBV_EXP_RES_DOMAIN_THREAD_MODEL	= (1 << 0),
  	IBV_EXP_RES_DOMAIN_MSG_MODEL	= (1 << 1),
  	IBV_EXP_RES_DOMAIN_RESERVED	= (1 << 2),
  };
  
  struct ibv_exp_res_domain_init_attr {
  	uint32_t			comp_mask; /* use ibv_exp_res_domain_init_attr_comp_mask */
  	enum ibv_exp_thread_model	thread_model;
  	enum ibv_exp_msg_model		msg_model;
  };
  
  enum ibv_exp_destroy_res_domain_comp_mask {
  	IBV_EXP_DESTROY_RES_DOMAIN_RESERVED	= (1 << 0),
  };
  
  struct ibv_exp_destroy_res_domain_attr {
  	uint32_t	comp_mask; /* use ibv_exp_destroy_res_domain_comp_mask */
  };
  
  /*
   * Query interface (specialized Verbs)
   */
  
  enum ibv_exp_query_intf_flags {
  	/* Interface functions includes correctness and validity checks */
  	IBV_EXP_QUERY_INTF_FLAG_ENABLE_CHECKS	= (1 << 0),
  };
  
  enum ibv_exp_intf_family {
  	IBV_EXP_INTF_QP_BURST,
  	IBV_EXP_INTF_CQ,
  	IBV_EXP_INTF_WQ,
  	IBV_EXP_INTF_RESERVED,
  };
  
  enum ibv_exp_experimental_intf_family {
  	IBV_EXP_EXPERIMENTAL_INTF_RESERVED,
  };
  
  enum ibv_exp_intf_scope {
  	IBV_EXP_INTF_GLOBAL, /* Permanent interface, identified by
  	 	 	      * the ibv_exp_intf_family enum
  	 	 	      */
  	IBV_EXP_INTF_EXPERIMENTAL, /* Interface under evaluation, identified by
  	 	 	 	    * the ibv_exp_experimental_intf_family enum
  	 	 	 	    * This interface may change between lib
  	 	 	 	    * versions
  	 			    */
  	IBV_EXP_INTF_VENDOR, /* Vendor specific interface, defined in vendor
  			      * separate header file
  			      */
  	IBV_EXP_INTF_VENDOR_EXPERIMENTAL, /* Vendor interface under evaluation,
  	 	 	 	 	   * defined in vendor separate header
  	 	 	 	 	   * file
  	 	 	 	 	   */
  };
  
  /* Return status from ibv_exp_query_intf */
  enum ibv_exp_query_intf_status {
  	IBV_EXP_INTF_STAT_OK,
  	IBV_EXP_INTF_STAT_VENDOR_NOT_SUPPORTED, /* The provided 'vendor_guid' is not supported */
  	IBV_EXP_INTF_STAT_INTF_NOT_SUPPORTED, /* The provided 'intf' is not supported */
  	IBV_EXP_INTF_STAT_VERSION_NOT_SUPPORTED, /* The provided 'intf_version' is not supported */
  	IBV_EXP_INTF_STAT_INVAL_PARARM, /* General invalid parameter */
  	IBV_EXP_INTF_STAT_INVAL_OBJ_STATE, /* QP is not in INIT, RTR or RTS state */
  	IBV_EXP_INTF_STAT_INVAL_OBJ, /* Mismatch between the provided 'obj'(CQ/QP/WQ) and requested 'intf' */
  	IBV_EXP_INTF_STAT_FLAGS_NOT_SUPPORTED, /* The provided set of 'flags' is not supported */
  	IBV_EXP_INTF_STAT_FAMILY_FLAGS_NOT_SUPPORTED, /* The provided set of 'family_flags' is not supported */
  };
  
  enum ibv_exp_query_intf_comp_mask {
  	IBV_EXP_QUERY_INTF_RESERVED	= (1 << 0),
  };
  
  struct ibv_exp_query_intf_params {
  	uint32_t			flags;		/* use ibv_exp_query_intf_flags */
  	enum ibv_exp_intf_scope		intf_scope;
  	uint64_t			vendor_guid;	/* set in case VENDOR intf_scope selected */
  	uint32_t			intf;		/* for GLOBAL intf_scope use ibv_exp_intf_family enum */
  	uint32_t			intf_version;	/* Version */
  	void				*obj;		/* interface object (CQ/QP/WQ) */
  	void				*family_params;	/* Family-specific params */
  	uint32_t			family_flags;	/* Family-specific flags */
  	uint32_t			comp_mask;	/* use ibv_exp_query_intf_comp_mask */
  };
  
  enum ibv_exp_release_intf_comp_mask {
  	IBV_EXP_RELEASE_INTF_RESERVED =	(1 << 0),
  };
  
  struct ibv_exp_release_intf_params {
  	uint32_t comp_mask; /* use ibv_exp_release_intf_comp_mask */
  };
  
  /*
   * Family interfaces
   */
  
  /* QP burst family */
  
  /* Flags to use in family_flags field of ibv_exp_query_intf_params on family creation */
  enum ibv_exp_qp_burst_family_create_flags {
  	/* To disable loop-back of multi-cast messages in RAW-ETH */
  	IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK		= (1 << 0),
  	/* To enable Multi-Packet send WR when possible */
  	IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR	= (1 << 1),
  };
  
  /* Flags to use on send functions of QP burst family */
  enum ibv_exp_qp_burst_family_flags {
  	IBV_EXP_QP_BURST_SIGNALED	= 1 << 0,
  	IBV_EXP_QP_BURST_SOLICITED	= 1 << 1,
  	IBV_EXP_QP_BURST_IP_CSUM	= 1 << 2,
  	IBV_EXP_QP_BURST_TUNNEL		= 1 << 3,
  	IBV_EXP_QP_BURST_FENCE		= 1 << 4,
  };
  
  /* All functions of QP family included in QP family version 1 */
  struct ibv_exp_qp_burst_family {
  	int (*send_pending)(struct ibv_qp *qp, uint64_t addr, uint32_t length, uint32_t lkey, uint32_t flags);
  	int (*send_pending_inline)(struct ibv_qp *qp, void *addr, uint32_t length, uint32_t flags);
  	int (*send_pending_sg_list)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num, uint32_t flags);
  	int (*send_flush)(struct ibv_qp *qp);
  	int (*send_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num, uint32_t flags);
  	int (*recv_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num);
  };
  
  struct ibv_exp_qp_burst_family_v1 {
  	/*
  	 * send_pending - Put one message in the provider send queue.
  	 *
  	 * Common usage: After calling several times to send_pending
  	 *    the application need to call send_flush to ensure the send
  	 *    of the pending messages.
  	 * Note: Use ibv_exp_qp_burst_family_flags for the flags field
  	 */
  	int (*send_pending)(struct ibv_qp *qp, uint64_t addr, uint32_t length, uint32_t lkey, uint32_t flags);
  	/*
  	 * send_pending_inline - Put one inline message in the provider send queue.
  	 *
  	 * Common usage: Same as send_pending
  	 * Notes:
  	 *  - The message length must fit the max inline size of the QP.
  	 *    Providing bigger messages may lead to data corruption and
  	 *    segmentation fault.
  	 *  - Use ibv_exp_qp_burst_family_flags for the flags field
  	 */
  	int (*send_pending_inline)(struct ibv_qp *qp, void *addr, uint32_t length, uint32_t flags);
  	/*
  	 * send_pending_sg_list - Put one scatter-gather(sg) message in the provider send queue.
  	 *
  	 * Common usage: Same as send_pending
  	 * Notes:
  	 *  - The number of sg entries must fit the max_send_sge of the QP.
  	 *    Providing bigger list of sg entries may lead to data corruption and
  	 *    segmentation fault.
  	 *  - Use ibv_exp_qp_burst_family_flags for the flags field
  	 */
  	int (*send_pending_sg_list)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num, uint32_t flags);
  	/*
  	 * send_flush - To flush the pending messages.
  	 *
  	 * Note: Use ibv_exp_qp_burst_family_flags for the flags field
  	 */
  	int (*send_flush)(struct ibv_qp *qp);
  	/*
  	 * send_burst - Send a list of 'num' messages (no send_flush required in this case)
  	 */
  	int (*send_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num, uint32_t flags);
  	/*
  	 * recv_burst - Post a set of 'num' receive buffers.
  	 *
  	 * Note: One sge per message is supported by this function
  	 */
  	int (*recv_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num);
  	/*
  	 * send_pending_vlan - Put one message in the provider send queue
  	 * and insert vlan_tci to header.
  	 *
  	 * Common usage: Same as send_pending
  	 * Note:
  	 * - Same as send_pending
  	 * - Not supported when MP enable.
  	 */
  	int (*send_pending_vlan)(struct ibv_qp *qp, uint64_t addr, uint32_t length,
  				 uint32_t lkey, uint32_t flags, uint16_t *vlan_tci);
  	/*
  	 * send_pending_inline - Put one inline message in the provider send queue
  	 * and insert vlan_tci to header.
  	 *
  	 * Common usage: Same as send_pending_inline
  	 * Notes:
  	 * - Same as send_pending_inline
  	 * - Not supported when MP enable.
  	 */
  	int (*send_pending_inline_vlan)(struct ibv_qp *qp, void *addr, uint32_t length,
  					uint32_t flags, uint16_t *vlan_tci);
  	/*
  	 * send_pending_sg_list - Put one scatter-gather(sg) message in the provider send queue
  	 * and insert vlan_tci to header.
  	 *
  	 * Common usage: Same as send_pending_sg_list
  	 * Notes:
  	 * - Same as send_pending_sg_list
  	 * - Not supported when MP enable.
  	 */
  	int (*send_pending_sg_list_vlan)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num,
  					 uint32_t flags, uint16_t *vlan_tci);
  };
  
  /* WQ family */
  struct ibv_exp_wq_family {
  	/*
  	 * recv_sg_list - Post one scatter-gather(sg) receive buffer.
  	 *
  	 * Note:
  	 *  - The number of sg entries must fit the max_recv_sge of the WQ.
  	 *    Providing bigger list of sg entries may lead to data corruption and
  	 *    segmentation fault.
  	 */
  	int (*recv_sg_list)(struct ibv_exp_wq *wq, struct ibv_sge *sg_list, uint32_t num_sg);
  	/*
  	 * recv_burst - Post a set of 'num' receive buffers.
  	 *
  	 * Note: One sge per message is supported by this function
  	 */
  	int (*recv_burst)(struct ibv_exp_wq *wq, struct ibv_sge *msg_list, uint32_t num);
  };
  
  /* CQ family */
  enum ibv_exp_cq_family_flags {
  	/* RX offloads flags */
  							/* The cq_family_flags are applicable
  							 * according to the existence of the
  							 * related device capabilities flags */
  	IBV_EXP_CQ_RX_IP_CSUM_OK		= 1 << 0, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */
  	IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK		= 1 << 1, /* IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */
  	IBV_EXP_CQ_RX_IPV4_PACKET		= 1 << 2, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */
  	IBV_EXP_CQ_RX_IPV6_PACKET		= 1 << 3, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */
  	IBV_EXP_CQ_RX_TUNNEL_PACKET		= 1 << 4, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */
  	IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK		= 1 << 5, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */
  	IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK	= 1 << 6, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */
  	IBV_EXP_CQ_RX_OUTER_IPV4_PACKET		= 1 << 7, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */
  	IBV_EXP_CQ_RX_OUTER_IPV6_PACKET		= 1 << 8, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */
  
  	/* Flags supported from CQ family version 1 */
  	/* Multi-Packet RQ flag */
  	IBV_EXP_CQ_RX_MULTI_PACKET_LAST_V1	= 1 << 9, /* Last packet on WR */
  	/* CVLAN stripping RQ flag */
  	IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1		= 1 << 10, /*
  							    * When set, CVLAN is stripped
  							    * from incoming packets.
  							    */
  
  							   /* The RX TCP/UDP packet flags
  							    * applicable according to
  							    * IBV_EXP_DEVICE_RX_TCP_UDP_PKT_TYPE
  							    * device capabilities flag
  							    */
  	IBV_EXP_CQ_RX_TCP_PACKET		= 1 << 11,
  	IBV_EXP_CQ_RX_UDP_PACKET		= 1 << 12,
  };
  
  /* All functions of CQ family included in CQ family version 1 */
  struct ibv_exp_cq_family {
  	int32_t (*poll_cnt)(struct ibv_cq *cq, uint32_t max);
  	int32_t (*poll_length)(struct ibv_cq *cq, void *buf, uint32_t *inl);
  	int32_t (*poll_length_flags)(struct ibv_cq *cq, void *buf, uint32_t *inl, uint32_t *flags);
  };
  
  struct ibv_exp_cq_family_v1 {
  	/*
  	 * poll_cnt - Poll up to 'max' valid completions
  	 *
  	 * The function returns the number of valid completions it
  	 * managed to drain from the CQ.
  	 *
  	 * Usage example: In case a CQ is connected to one send-queue
  	 *                the application may use this function to get
  	 *                the number of the QP send-completions.
  	 *
  	 * Return value (n):
  	 *    n >= 0 : number extracted completions.
  	 *    n == -1 : operation failed. completion is not extracted.
  	 *              To extract this completion, ibv_poll_cq() must be used
  	 *
  	 * Note: The function designed to support TX completion, it may also be
  	 *    used for RX completion but it is not supporting RX inline-scatter.
  	 */
  	int32_t (*poll_cnt)(struct ibv_cq *cq, uint32_t max);
  	/*
  	 * poll_length - Poll one receive completion and provide the related
  	 *               message length.
  	 *
  	 * The function returns only the length of the completed message.
  	 * In case of inline received message the message will be copied
  	 * to the provided buffer ('buf') and the '*inl' status will be set.
  	 * The function extracts only completion of regular receive-messages.
  	 * In case of send-message completion or SRQ receive-message completion
  	 * it returns -1.
  	 *
  	 * Usage example: In case a CQ is connected to one receive-queue
  	 *                the application may use this function to get
  	 *                the size of the next received message.
  	 *
  	 * Return value (n):
  	 *    n > 0 : successful completion with positive length.
  	 *            *inl will be set to 1 if data was copied to buffer.
  	 *
  	 *    0     : Empty.
  	 *    n == -1 : operation failed. completion is not extracted.
  	 *              To extract this completion, ibv_poll_cq() must be used
  	 */
  	int32_t (*poll_length)(struct ibv_cq *cq, void *buf, uint32_t *inl);
  	/*
  	 * poll_length_flags - Poll one receive completion and provide the related
  	 *                     message length and completion flags.
  	 *
  	 * The same as poll_length but also retrieves completion flags as
  	 * defined by the enum ibv_exp_cq_family_flags
  	 */
  	int32_t (*poll_length_flags)(struct ibv_cq *cq, void *buf, uint32_t *inl, uint32_t *flags);
  	/*
  	 * poll_length_flags_mp_rq - Poll one receive completion and provide the related
  	 *                           message length, packet-offset and completion flags.
  	 *
  	 * The same as poll_length_flags but:
  	 *  - Without the inline-receive support.
  	 *  - Also retrieves offset in the WR posted buffer as defined by the WR SG list.
  	 *    The start of the received packet is located in this offset.
  	 */
  	int32_t (*poll_length_flags_mp_rq)(struct ibv_cq *cq, uint32_t *offset, uint32_t *flags);
  	/*
  	 * poll_length_flags_cvlan - Poll one receive completion and provide the
  	 *			     related message length, completion flags
  	 *			     and CVLAN TCI.
  	 *			     The CVLAN TCI value is valid only when
  	 *			     IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1 flag is
  	 *			     set.
  	 *
  	 * The same as poll_length_flags but:
  	 *  - Also retrievs the packet's CVLAN TCI that was stripped by the HW.
  	 */
  	int32_t (*poll_length_flags_cvlan)(struct ibv_cq *cq, void *buf,
  					   uint32_t *inl, uint32_t *flags,
  					   uint16_t *vlan_tci);
  	/*
  	 * poll_length_flags_mp_rq_cvlan - Poll one receive completion and provide
  	 *				   the related message length,
  	 *				   packet-offset, completion flags and
  	 *				   CVLAN TCI
  	 *
  	 * The same as poll_length_flags_cvlan but:
  	 *  - Without the inline-receive support.
  	 *  - Also retrives offset in the WR posted buffer as defined by the
  	 *    WR SG list. The start of the received packet is located in this
  	 *    offset.
  	 */
  	int32_t (*poll_length_flags_mp_rq_cvlan)(struct ibv_cq *cq,
  						 uint32_t *offset,
  						 uint32_t *flags,
  						 uint16_t *vlan_tci);
  };
  
  enum {
  	IBV_EXP_NUM_DC_INFO_LIDS	= 30
  };
  
  struct ibv_exp_dc_info_ent {
  	uint16_t	lid[IBV_EXP_NUM_DC_INFO_LIDS];
  	uint32_t	seqnum;
  };
  
  enum ibv_exp_roce_gid_type {
  	IBV_EXP_IB_ROCE_V1_GID_TYPE,
  	IBV_EXP_ROCE_V2_GID_TYPE,
  	IBV_EXP_ROCE_V1_5_GID_TYPE,
  };
  
  enum ibv_exp_query_gid_attr {
  	IBV_EXP_QUERY_GID_ATTR_TYPE = (1 << 0),
  	IBV_EXP_QUERY_GID_ATTR_GID	= (1 << 1),
  	IBV_EXP_QUERY_GID_ATTR_RESERVED	= (1 << 2),
  };
  
  struct ibv_exp_gid_attr {
  	uint32_t			comp_mask;
  	enum ibv_exp_roce_gid_type	type;
  	union ibv_gid			gid;
  };
  
  /**
   * enum ibv_exp_ec_calc_attr_comp_mask - erasure coding context
   *    init attributes compatibility enumeration
   */
  enum ibv_exp_ec_calc_attr_comp_mask {
  	IBV_EXP_EC_CALC_ATTR_MAX_INFLIGHT	= (1 << 0),
  	IBV_EXP_EC_CALC_ATTR_K			= (1 << 1),
  	IBV_EXP_EC_CALC_ATTR_M			= (1 << 2),
  	IBV_EXP_EC_CALC_ATTR_W			= (1 << 3),
  	IBV_EXP_EC_CALC_ATTR_MAX_DATA_SGE	= (1 << 4),
  	IBV_EXP_EC_CALC_ATTR_MAX_CODE_SGE	= (1 << 5),
  	IBV_EXP_EC_CALC_ATTR_ENCODE_MAT		= (1 << 6),
  	IBV_EXP_EC_CALC_ATTR_AFFINITY		= (1 << 7),
  	IBV_EXP_EC_CALC_ATTR_POLLING		= (1 << 8),
  	IBV_EXP_EC_CALC_INIT_ATTR_RESERVED	= (1 << 9),
  };
  
  /**
   * struct ibv_exp_ec_calc_init_attr - erasure coding engine
   *     initialization attributes
   *
   * @comp_mask:            compatibility bitmask
   * @max_inflight_calcs:   maximum inflight calculations
   * @k:                    number of data blocks
   * @m:                    number of core blocks
   * @w:                    Galois field bits GF(2^w)
   * @max_data_sge:	  maximum data sg elements to be used for encode/decode
   * @max_code_sge:	  maximum code sg elements to be used for encode/decode
   * @encode_matrix:        buffer that contain the encoding matrix
   * @affinity_hint:        affinity hint for asynchronous calcs completion
   *                        steering.
   * @polling:              polling mode (if set no completions will be generated
   *                        by events).
   */
  struct ibv_exp_ec_calc_init_attr {
  	uint32_t		comp_mask;
  	uint32_t		max_inflight_calcs;
  	int			k;
  	int			m;
  	int			w;
  	int			max_data_sge;
  	int			max_code_sge;
  	uint8_t			*encode_matrix;
  	int			affinity_hint;
  	int			polling;
  };
  
  /**
   * enum ibv_exp_ec_status - EX calculation status
   *
   * @IBV_EXP_EC_CALC_SUCCESS:   EC calc operation succeded
   * @IBV_EXP_EC_CALC_FAIL:      EC calc operation failed
   */
  enum ibv_exp_ec_status {
  	IBV_EXP_EC_CALC_SUCCESS,
  	IBV_EXP_EC_CALC_FAIL,
  };
  
  /**
   * struct ibv_exp_ec_comp - completion context of EC calculation
   *
   * @done:      function handle of the EC calculation completion
   * @status:    status of the EC calculation
   *
   * The consumer is expected to embed this structure in his calculation context
   * so that the user context would be acquired back using offsetof()
   */
  struct ibv_exp_ec_comp {
  	void (*done)(struct ibv_exp_ec_comp *comp);
  	enum ibv_exp_ec_status status;
  };
  
  /**
   * struct ibv_exp_ec_calc - erasure coding engine context
   *
   * @pd:                 protection domain
   */
  struct ibv_exp_ec_calc {
  	struct ibv_pd		*pd;
  };
  
  /**
   * struct ibv_exp_ec_mem - erasure coding memory layout context
   *
   * @data_blocks:        array of data sg elements
   * @num_data_sge:       number of data sg elements
   * @code_blocks:        array of code sg elements
   * @num_code_sge:       number of code sg elements
   * @block_size:         logical block size
   */
  struct ibv_exp_ec_mem {
  	struct ibv_sge		*data_blocks;
  	int			num_data_sge;
  	struct ibv_sge		*code_blocks;
  	int			num_code_sge;
  	int			block_size;
  };
  
  /**
   * struct ibv_exp_ec_stripe - erasure coding stripe descriptor
   *
   * @qp:    queue-pair connected to the relevant peer
   * @wr:    send work request, can either be a RDMA wr or a SEND
   */
  struct ibv_exp_ec_stripe {
  	struct ibv_qp		*qp;
  	struct ibv_send_wr	*wr;
  };
  
  struct ibv_exp_peer_commit;
  struct ibv_exp_rollback_ctx;
  
  
  struct ibv_exp_peer_peek;
  struct ibv_exp_peer_abort_peek;
  
  struct verbs_context_exp {
  	/*  "grows up" - new fields go here */
  	struct ibv_ah * (*drv_exp_ibv_create_kah)(struct ibv_pd *pd,
  						  struct ibv_exp_ah_attr *attr_exp);
  	int (*exp_peer_peek_cq)(struct ibv_cq *ibcq,
  				struct ibv_exp_peer_peek *peek_ctx);
  	int (*exp_peer_abort_peek_cq)(struct ibv_cq *ibcq,
  				      struct ibv_exp_peer_abort_peek *ack_ctx);
  	int (*exp_peer_commit_qp)(struct ibv_qp *qp,
  				  struct ibv_exp_peer_commit *peer);
  	int (*exp_rollback_send)(struct ibv_qp *qp,
  				 struct ibv_exp_rollback_ctx *rollback);
  	int (*ec_update_sync)(struct ibv_exp_ec_calc *calc,
  			      struct ibv_exp_ec_mem *ec_mem,
  			      uint8_t *data_updates,
  			      uint8_t *code_updates);
  	int (*ec_update_async)(struct ibv_exp_ec_calc *calc,
  			       struct ibv_exp_ec_mem *ec_mem,
  			       uint8_t *data_updates,
  			       uint8_t *code_updates,
  			       struct ibv_exp_ec_comp *ec_comp);
  	struct ibv_exp_ec_calc *(*alloc_ec_calc)(struct ibv_pd *pd,
  						 struct ibv_exp_ec_calc_init_attr *attr);
  	void (*dealloc_ec_calc)(struct ibv_exp_ec_calc *calc);
  	int (*ec_encode_async)(struct ibv_exp_ec_calc *calc,
  			       struct ibv_exp_ec_mem *ec_mem,
  			       struct ibv_exp_ec_comp *ec_comp);
  	int (*ec_encode_sync)(struct ibv_exp_ec_calc *calc,
  			      struct ibv_exp_ec_mem *ec_mem);
  	int (*ec_decode_async)(struct ibv_exp_ec_calc *calc,
  			       struct ibv_exp_ec_mem *ec_mem,
  			       uint8_t *erasures,
  			       uint8_t *decode_matrix,
  			       struct ibv_exp_ec_comp *ec_comp);
  	int (*ec_decode_sync)(struct ibv_exp_ec_calc *calc,
  			      struct ibv_exp_ec_mem *ec_mem,
  			      uint8_t *erasures,
  			      uint8_t *decode_matrix);
  	int (*ec_poll)(struct ibv_exp_ec_calc *calc, int n);
  	int (*ec_encode_send)(struct ibv_exp_ec_calc *calc,
  			      struct ibv_exp_ec_mem *ec_mem,
  			      struct ibv_exp_ec_stripe *data_stripes,
  			      struct ibv_exp_ec_stripe *code_stripes);
  	int (*exp_query_gid_attr)(struct ibv_context *context, uint8_t port_num,
  				  unsigned int index,
  				  struct ibv_exp_gid_attr *attr);
  	int (*exp_destroy_rwq_ind_table)(struct ibv_exp_rwq_ind_table *rwq_ind_table);
  	struct ibv_exp_rwq_ind_table *(*exp_create_rwq_ind_table)(struct ibv_context *context,
  								  struct ibv_exp_rwq_ind_table_init_attr *init_attr);
  	int (*exp_destroy_wq)(struct ibv_exp_wq *wq);
  	int (*exp_modify_wq)(struct ibv_exp_wq *wq,
  				 struct ibv_exp_wq_attr *wq_attr);
  	struct ibv_exp_wq * (*exp_create_wq)(struct ibv_context *context,
  						 struct ibv_exp_wq_init_attr *wq_init_attr);
  	int (*drv_exp_poll_dc_info)(struct ibv_context *context,
  				    struct ibv_exp_dc_info_ent *ents,
  				    int nent, int port);
  	void *(*exp_query_intf)(struct ibv_context *context, struct ibv_exp_query_intf_params *params,
  				enum ibv_exp_query_intf_status *status);
  	int (*exp_release_intf)(struct ibv_context *context, void *intf,
  				struct ibv_exp_release_intf_params *params);
  	struct ibv_exp_res_domain *(*exp_create_res_domain)(struct ibv_context *context,
  							    struct ibv_exp_res_domain_init_attr *attr);
  	int (*exp_destroy_res_domain)(struct ibv_context *context,
  				      struct ibv_exp_res_domain *res_dom,
  				      struct ibv_exp_destroy_res_domain_attr *attr);
  	int (*lib_exp_use_priv_env)(struct ibv_context *context);
  	int (*lib_exp_setenv)(struct ibv_context *context, const char *name,
  			      const char *value, int overwrite);
  	struct verbs_environment *venv;
  	int (*drv_exp_dereg_mr)(struct ibv_mr *mr, struct ibv_exp_dereg_out *out);
  	void (*ABI_placeholder2)(void);
  	void (*ABI_placeholder1)(void);
  	int (*drv_exp_prefetch_mr)(struct ibv_mr *mr,
  				   struct ibv_exp_prefetch_attr *attr);
  	int (*lib_exp_prefetch_mr)(struct ibv_mr *mr,
  				   struct ibv_exp_prefetch_attr *attr);
  	struct ibv_exp_mkey_list_container * (*drv_exp_alloc_mkey_list_memory)(struct ibv_exp_mkey_list_container_attr *attr);
  	struct ibv_exp_mkey_list_container * (*lib_exp_alloc_mkey_list_memory)(struct ibv_exp_mkey_list_container_attr *attr);
  	int (*drv_exp_dealloc_mkey_list_memory)(struct ibv_exp_mkey_list_container *mem);
  	int (*lib_exp_dealloc_mkey_list_memory)(struct ibv_exp_mkey_list_container *mem);
  	int (*drv_exp_query_mkey)(struct ibv_mr *mr, struct ibv_exp_mkey_attr *mkey_attr);
  	int (*lib_exp_query_mkey)(struct ibv_mr *mr, struct ibv_exp_mkey_attr *mkey_attr);
  	struct ibv_mr * (*drv_exp_create_mr)(struct ibv_exp_create_mr_in *in);
  	struct ibv_mr * (*lib_exp_create_mr)(struct ibv_exp_create_mr_in *in);
  	int (*drv_exp_arm_dct)(struct ibv_exp_dct *dct, struct ibv_exp_arm_attr *attr);
  	int (*lib_exp_arm_dct)(struct ibv_exp_dct *dct, struct ibv_exp_arm_attr *attr);
  	int (*drv_exp_bind_mw)(struct ibv_exp_mw_bind *mw_bind);
  	int (*lib_exp_bind_mw)(struct ibv_exp_mw_bind *mw_bind);
  	int (*drv_exp_post_send)(struct ibv_qp *qp,
  				 struct ibv_exp_send_wr *wr,
  				 struct ibv_exp_send_wr **bad_wr);
  	struct ibv_mr * (*drv_exp_reg_mr)(struct ibv_exp_reg_mr_in *in);
  	struct ibv_mr * (*lib_exp_reg_mr)(struct ibv_exp_reg_mr_in *in);
  	struct ibv_ah * (*drv_exp_ibv_create_ah)(struct ibv_pd *pd,
  						 struct ibv_exp_ah_attr *attr_exp);
  	int (*drv_exp_query_values)(struct ibv_context *context, int q_values,
  				    struct ibv_exp_values *values);
  	struct ibv_cq * (*exp_create_cq)(struct ibv_context *context, int cqe,
  					 struct ibv_comp_channel *channel,
  					 int comp_vector, struct ibv_exp_cq_init_attr *attr);
  	int (*drv_exp_ibv_poll_cq)(struct ibv_cq *ibcq, int num_entries,
  				   struct ibv_exp_wc *wc, uint32_t wc_size);
  	void * (*drv_exp_get_legacy_xrc) (struct ibv_srq *ibv_srq);
  	void (*drv_exp_set_legacy_xrc) (struct ibv_srq *ibv_srq, void *legacy_xrc);
  	struct ibv_mr * (*drv_exp_ibv_reg_shared_mr)(struct ibv_exp_reg_shared_mr_in *in);
  	struct ibv_mr * (*lib_exp_ibv_reg_shared_mr)(struct ibv_exp_reg_shared_mr_in *in);
  	int (*drv_exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr,
  				 uint64_t exp_attr_mask);
  	int (*lib_exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr,
  				 uint64_t exp_attr_mask);
  	int (*drv_exp_post_task)(struct ibv_context *context,
  				 struct ibv_exp_task *task,
  				 struct ibv_exp_task **bad_task);
  	int (*lib_exp_post_task)(struct ibv_context *context,
  				 struct ibv_exp_task *task,
  				 struct ibv_exp_task **bad_task);
  	int (*drv_exp_modify_cq)(struct ibv_cq *cq,
  				 struct ibv_exp_cq_attr *attr, int attr_mask);
  	int (*lib_exp_modify_cq)(struct ibv_cq *cq,
  				 struct ibv_exp_cq_attr *attr, int attr_mask);
  	int (*drv_exp_ibv_destroy_flow) (struct ibv_exp_flow *flow);
  	int (*lib_exp_ibv_destroy_flow) (struct ibv_exp_flow *flow);
  	struct ibv_exp_flow * (*drv_exp_ibv_create_flow) (struct ibv_qp *qp,
  						      struct ibv_exp_flow_attr
  						      *flow_attr);
  	struct ibv_exp_flow * (*lib_exp_ibv_create_flow) (struct ibv_qp *qp,
  							  struct ibv_exp_flow_attr
  							  *flow_attr);
  
  	int (*drv_exp_query_port)(struct ibv_context *context, uint8_t port_num,
  				  struct ibv_exp_port_attr *port_attr);
  	int (*lib_exp_query_port)(struct ibv_context *context, uint8_t port_num,
  				  struct ibv_exp_port_attr *port_attr);
  	struct ibv_exp_dct *(*create_dct)(struct ibv_context *context,
  					  struct ibv_exp_dct_init_attr *attr);
  	int (*destroy_dct)(struct ibv_exp_dct *dct);
  	int (*query_dct)(struct ibv_exp_dct *dct, struct ibv_exp_dct_attr *attr);
  	int (*drv_exp_query_device)(struct ibv_context *context,
  				    struct ibv_exp_device_attr *attr);
  	int (*lib_exp_query_device)(struct ibv_context *context,
  				    struct ibv_exp_device_attr *attr);
  	struct ibv_qp *(*drv_exp_create_qp)(struct ibv_context *context,
  					    struct ibv_exp_qp_init_attr *init_attr);
  	struct ibv_qp *(*lib_exp_create_qp)(struct ibv_context *context,
  					    struct ibv_exp_qp_init_attr *init_attr);
  	size_t sz;	/* Set by library on struct allocation,	*/
  			/* must be located as last field	*/
  };
  
  #define ETHERNET_LL_SIZE 6
  
  static inline struct verbs_context_exp *verbs_get_exp_ctx(struct ibv_context *ctx)
  {
  	struct verbs_context *app_ex_ctx = verbs_get_ctx(ctx);
  	char *actual_ex_ctx;
  
  	if (!app_ex_ctx || !(app_ex_ctx->has_comp_mask & VERBS_CONTEXT_EXP))
  		return NULL;
  
  	actual_ex_ctx = ((char *)ctx) - (app_ex_ctx->sz - sizeof(struct ibv_context));
  
  	return (struct verbs_context_exp *)(actual_ex_ctx - sizeof(struct verbs_context_exp));
  }
  
  #define verbs_get_exp_ctx_op(ctx, op) ({ \
  	struct verbs_context_exp *_vctx = verbs_get_exp_ctx(ctx); \
  	(!_vctx || (_vctx->sz < sizeof(*_vctx) - offsetof(struct verbs_context_exp, op)) || \
  	!_vctx->op) ? NULL : _vctx; })
  
  #define verbs_set_exp_ctx_op(_vctx, op, ptr) ({ \
  	struct verbs_context_exp *vctx = _vctx; \
  	if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context_exp, op))) \
  		vctx->op = ptr; })
  
  /**
   * ibv_exp_get_device_list - Get a refreshed list of IB devices available
   * @num_devices: optional.  if non-NULL, set to the number of devices
   * returned in the array.
   *
   * Return a NULL-terminated array of IB devices.  The array can be
   * released with ibv_free_device_list().
   */
  struct ibv_device **ibv_exp_get_device_list(int *num_devices);
  
  /*
   * ibv_exp_alloc_ec_calc() - allocate an erasure coding
   *     calculation offload context
   * @pd:          user allocated protection domain
   * @attrs:       initialization attributes
   *
   * Returns handle handle to the EC calculation APIs
   */
  static inline struct ibv_exp_ec_calc *
  ibv_exp_alloc_ec_calc(struct ibv_pd *pd,
  		      struct ibv_exp_ec_calc_init_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(pd->context, alloc_ec_calc);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask,
  					      IBV_EXP_EC_CALC_INIT_ATTR_RESERVED - 1);
  
  	return vctx->alloc_ec_calc(pd, attr);
  }
  
  /*
   * ibv_exp_dealloc_ec_calc() - free an erasure coding
   *     calculation offload context
   * @ec_calc:       ec context
   */
  static inline void ibv_exp_dealloc_ec_calc(struct ibv_exp_ec_calc *calc)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, dealloc_ec_calc);
  	if (!vctx) {
  		errno = ENOSYS;
  		return;
  	}
  
  	vctx->dealloc_ec_calc(calc);
  }
  
  /**
   * ibv_exp_ec_encode_async() - asynchronous encode of given data blocks
   *    and place in code_blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   * @ec_comp:          EC calculation completion context
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout, the total length of the sg elements must satisfy
   *   k * ec_mem.block_size.
   * - ec_mem.num_data_sg must not exceed the calc max_data_sge
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   m * ec_mem.block_size.
   * - ec_mem.num_code_sg must not exceed the calc max_code_sge
   *
   * Notes:
   * The ec_calc will perform the erasure coding calc operation,
   * once it completes, it will call ec_comp->done() handle.
   * The caller will take it from there.
   */
  static inline int
  ibv_exp_ec_encode_async(struct ibv_exp_ec_calc *calc,
  			struct ibv_exp_ec_mem *ec_mem,
  			struct ibv_exp_ec_comp *ec_comp)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_async);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->ec_encode_async(calc, ec_mem, ec_comp);
  }
  
  /**
   * ibv_exp_ec_encode_sync() - synchronous encode of given data blocks
   *    and place in code_blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout, the total length of the sg elements must satisfy
   *   k * ec_mem.block_size.
   * - ec_mem.num_data_sg must not exceed the calc max_data_sge
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   m * ec_mem.block_size.
   * - ec_mem.num_code_sg must not exceed the calc max_code_sge
   *
   * Returns 0 on success, non-zero on failure.
   *
   */
  static inline int
  ibv_exp_ec_encode_sync(struct ibv_exp_ec_calc *calc,
  		       struct ibv_exp_ec_mem *ec_mem)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_sync);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->ec_encode_sync(calc, ec_mem);
  }
  
  /**
   * ibv_exp_ec_decode_async() - decode a given set of data blocks
   *    and code_blocks and place into output recovery blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   * @erasures:         pointer to byte-map of which blocks were erased
   * 		      and needs to be recovered
   * @decode_matrix:    buffer that contains the decode matrix
   * @ec_comp:          EC calculation completion context
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout, the total length of the sg elements must satisfy
   *   k * ec_mem.block_size.
   * - ec_mem.num_data_sg must not exceed the calc max_data_sge
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   number of missing blocks * ec_mem.block_size.
   * - ec_mem.num_code_sg must not exceed the calc max_code_sge
   * - erasures byte-mask consists of the survived and erased blocks.
   *   The first k bytes stand for the k data blocks followed by
   *   m bytes that stand for the code blocks.
   *
   * Returns 0 on success, or non-zero on failure with a corresponding
   * errno.
   *
   * Notes:
   * The ec_calc will perform the erasure coding calc operation,
   * once it completes, it will call ec_comp->done() handle.
   * The caller will take it from there
   */
  static inline int
  ibv_exp_ec_decode_async(struct ibv_exp_ec_calc *calc,
  			struct ibv_exp_ec_mem *ec_mem,
  			uint8_t *erasures,
  			uint8_t *decode_matrix,
  			struct ibv_exp_ec_comp *ec_comp)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_decode_async);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->ec_decode_async(calc, ec_mem, erasures,
  				     decode_matrix, ec_comp);
  }
  
  /**
   * ibv_exp_ec_decode_sync() - decode a given set of data blocks
   *    and code_blocks and place into output recovery blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   * @erasures:         pointer to byte-map of which blocks were erased
   * 		      and needs to be recovered
   * @decode_matrix:    registered buffer of the decode matrix
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout, the total length of the sg elements must satisfy
   *   k * ec_mem.block_size.
   * - ec_mem.num_data_sg must not exceed the calc max_data_sge
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   number of missing blocks * ec_mem.block_size.
   * - ec_mem.num_code_sg must not exceed the calc max_code_sge
   * - erasures byte-map consists of the survived and erased blocks.
   *   The first k bytes stand for the k data blocks followed by
   *   m bytes that stand for the code blocks.
   *
   * Returns 0 on success, or non-zero on failure with a corresponding
   * errno.
   */
  static inline int
  ibv_exp_ec_decode_sync(struct ibv_exp_ec_calc *calc,
  		       struct ibv_exp_ec_mem *ec_mem,
  		       uint8_t *erasures,
  		       uint8_t *decode_matrix)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_decode_sync);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->ec_decode_sync(calc, ec_mem, erasures, decode_matrix);
  }
  
  /**
   * ibv_exp_ec_update_async() - copmutes redundancies based on updated blocks,
   *    their replacements and old redundancies and place into output code blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   * @data_updates:     array which is a map of data blocks that are updated
   * @code_updates:     array which is a map of code blocks to be computed
   * @ec_comp:          EC calculation completion context
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout in the following way:
   *   assume we want to update blocks d_i and d_j with i<j,
   *   then sg enries should be as follows:
   *   c_0 ... c_m d_i d'_i d_j d'_j
   *   were c_0 ... c_m are all previous redundancies,
   *   d_i is an original i-th block, d'_i is new i-th block
   * - ec_mem.num_data_sg should be equal to the number of sg entries,
   *   i.e. to num of code blocks to be updated + 2*num of updates
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   number of overall code blocks to be updated.
   * - ec_mem.num_code_sg must be equal to the number of code blocks
   *   to be updated and not to exceed the calc max_code_sge
   * - data_updates is an array of size k (=number of data blocks)
   *   and is a byte-map for blocks to be updated, i.e
   *   if we want to update i-th block and do not want to update j-th block,
   *   then data_updates[i]=1 and data_updates[j]=0.
   * - code_updates is an array of size m(=number of code blocks)
   *   and is a byte-map of code blocks that should be computed, i.e
   *   if we want to compute i-th block and do not want to compute j-th block,
   *   then code_updates[i]=1 and code_updates[j]=0.
   *
   * Returns 0 on success, or non-zero on failure with a corresponding
   * errno.
   */
  
  static inline int
  ibv_exp_ec_update_async(struct ibv_exp_ec_calc *calc,
  			struct ibv_exp_ec_mem *ec_mem,
  			uint8_t *data_updates,
  			uint8_t *code_updates,
  			struct ibv_exp_ec_comp *ec_comp)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_update_async);
  	if (!vctx)
  		return -ENOSYS;
  
  	return vctx->ec_update_async(calc, ec_mem, data_updates,
  				     code_updates, ec_comp);
  }
  
  /**
   * ibv_exp_ec_update_sync() - copmutes redundancies based on updated blocks,
   *    their replacements and old redundancies and place into output code blocks
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout
   * @data_updates:     array which is a map of data blocks that are updated
   * @code_updates:     array which is a map of code blocks to be computed
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout in the following way:
   *   assume we want to update blocks d_i and d_j with i<j,
   *   then enries of sg should be as follows:
   *   c_0..c_m d_i d'_i d_j d'_j
   *   were c_0 .. c_m are previous redundancies,
   *   d_i is an original i-th block, d'_i is new i-th block
   * - ec_mem.num_data_sg should be equal to the number of sg entries,
   *   i.e. to num of code blocks to be updated + 2*num of updates
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   number of overall code blocks to be updated.
   * - ec_mem.num_code_sg must be equal to the number of code blocks
   *   to be updated and not to exceed the calc max_code_sge
   * - data_updates is an array of size k (=number of data blocks)
   *   and is a byte-map for blocks to be updated, i.e
   *   if we want to update i-th block and do not want to update j-th block,
   *   then data_updates[i]=1 and data_updates[j]=0.
   * - code_updates is an array of size m(=number of code blocks)
   *   and is a bytemap of code blocks that should be computed, i.e
   *   if we want to compute i-th block and do not want to compute j-th block,
   *   then code_updates[i]=1 and code_updates[j]=0.
   *
   * Returns 0 on success, or non-zero on failure with a corresponding
   * errno.
   */
  
  static inline int
  ibv_exp_ec_update_sync(struct ibv_exp_ec_calc *calc,
  		       struct ibv_exp_ec_mem *ec_mem,
  		       uint8_t *data_updates,
  		       uint8_t *code_updates)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_update_sync);
  	if (!vctx)
  		return -ENOSYS;
  
  	return vctx->ec_update_sync(calc, ec_mem, data_updates, code_updates);
  }
  /**
   * ibv_exp_ec_calc_poll() - poll for EC calculation
   *
   * @calc: EC calc context
   * @n: number of calculations to poll
   *
   * Returns the number of calc completions processed which
   * is lower or equal to n. Relevant only when EC calc context
   * was allocated in polling mode.
   */
  static inline int
  ibv_exp_ec_poll(struct ibv_exp_ec_calc *calc, int n)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_poll);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->ec_poll(calc, n);
  }
  
  /**
   * ibv_exp_ec_encode_send() - encode a given data blocks
   *    initiate the data and code blocks transfers to the wire with the qps array.
   * @ec_calc:          erasure coding calculation engine
   * @ec_mem:           erasure coding memory layout context
   * @data_stripes:     array of stripe handles, each represents a data block channel
   * @code_stripes:     array of qp handles, each represents a code block channel
   *
   * Restrictions:
   * - ec_calc is an initialized erasure coding calc engine structure
   * - ec_mem.data_blocks sg array must describe the data memory
   *   layout, the total length of the sg elements must satisfy
   *   k * ec_mem.block_size.
   * - ec_mem.num_data_sg must not exceed the calc max_data_sge
   * - ec_mem.code_blocks sg array must describe the code memory
   *   layout, the total length of the sg elements must satisfy
   *   m * ec_mem.block_size.
   * - ec_mem.num_code_sg must not exceed the calc max_code_sge
   *
   * Returns 0 on success, or non-zero on failure with a corresponding
   * errno.
   */
  static inline int
  ibv_exp_ec_encode_send(struct ibv_exp_ec_calc *calc,
  		       struct ibv_exp_ec_mem *ec_mem,
  		       struct ibv_exp_ec_stripe *data_stripes,
  		       struct ibv_exp_ec_stripe *code_stripes)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_send);
  	if (!vctx)
  		return -ENOSYS;
  
  	return vctx->ec_encode_send(calc, ec_mem, data_stripes, code_stripes);
  }
  
  static inline struct ibv_qp *
  ibv_exp_create_qp(struct ibv_context *context, struct ibv_exp_qp_init_attr *qp_init_attr)
  {
  	struct verbs_context_exp *vctx;
  	uint32_t mask = qp_init_attr->comp_mask;
  
  	if (mask == IBV_EXP_QP_INIT_ATTR_PD)
  		return ibv_create_qp(qp_init_attr->pd,
  				     (struct ibv_qp_init_attr *) qp_init_attr);
  
  	vctx = verbs_get_exp_ctx_op(context, lib_exp_create_qp);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(qp_init_attr->comp_mask,
  					      IBV_EXP_QP_INIT_ATTR_RESERVED1 - 1);
  
  	return vctx->lib_exp_create_qp(context, qp_init_attr);
  }
  
  /*
   * ibv_exp_use_priv_env
   *
   * switch to use private environment
   */
  static inline int ibv_exp_use_priv_env(struct ibv_context *context)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, lib_exp_use_priv_env);
  	if (!vctx) {
  		errno = ENOSYS;
  		return -1;
  	}
  
  	return vctx->lib_exp_use_priv_env(context);
  }
  
  /*
   * ibv_exp_poll_dc_info
   *
   * The function is not thread safe. Any locking must be done by the user.
   *
   * Return: >= 0 number of returned entries
   *	   < 0 error
   *
   */
  static inline int ibv_exp_poll_dc_info(struct ibv_context *context,
  				       struct ibv_exp_dc_info_ent *ents,
  				       int nent, int port)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, drv_exp_poll_dc_info);
  	if (!vctx) {
  		errno = ENOSYS;
  		return -1;
  	}
  
  	return vctx->drv_exp_poll_dc_info(context, ents, nent, port);
  }
  
  /*
   * ibv_exp_setenv
   *
   * see man setenv for parameter description
   */
  static inline int ibv_exp_setenv(struct ibv_context *context,
  				 const char *name,
  				 const char *value,
  				 int overwrite)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, lib_exp_setenv);
  	if (!vctx)
  		return setenv(name, value, overwrite);
  
  	return vctx->lib_exp_setenv(context, name, value, overwrite);
  }
  
  static inline int ibv_exp_query_device(struct ibv_context *context,
  				       struct ibv_exp_device_attr *attr)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context,
  							      lib_exp_query_device);
  	if (!vctx)
  		return ENOSYS;
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_DEVICE_ATTR_RESERVED - 1);
  	return vctx->lib_exp_query_device(context, attr);
  }
  
  static inline struct ibv_exp_dct *
  ibv_exp_create_dct(struct ibv_context *context,
  		   struct ibv_exp_dct_init_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_exp_dct *dct;
  
  	vctx = verbs_get_exp_ctx_op(context, create_dct);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask,
  					      IBV_EXP_DCT_INIT_ATTR_RESERVED - 1);
  	pthread_mutex_lock(&context->mutex);
  	dct = vctx->create_dct(context, attr);
  	if (dct)
  		dct->context = context;
  
  	pthread_mutex_unlock(&context->mutex);
  
  	return dct;
  }
  
  static inline int ibv_exp_destroy_dct(struct ibv_exp_dct *dct)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_context *context = dct->context;
  	int err;
  
  	vctx = verbs_get_exp_ctx_op(context, destroy_dct);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  
  	pthread_mutex_lock(&context->mutex);
  	err = vctx->destroy_dct(dct);
  	pthread_mutex_unlock(&context->mutex);
  
  	return err;
  }
  
  static inline int ibv_exp_query_dct(struct ibv_exp_dct *dct,
  				    struct ibv_exp_dct_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_context *context = dct->context;
  	int err;
  
  	vctx = verbs_get_exp_ctx_op(context, query_dct);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_DCT_ATTR_RESERVED - 1);
  	pthread_mutex_lock(&context->mutex);
  	err = vctx->query_dct(dct, attr);
  	pthread_mutex_unlock(&context->mutex);
  
  	return err;
  }
  
  static inline int ibv_exp_arm_dct(struct ibv_exp_dct *dct,
  				  struct ibv_exp_arm_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_context *context = dct->context;
  	int err;
  
  	vctx = verbs_get_exp_ctx_op(context, lib_exp_arm_dct);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_ARM_ATTR_RESERVED - 1);
  	pthread_mutex_lock(&context->mutex);
  	err = vctx->lib_exp_arm_dct(dct, attr);
  	pthread_mutex_unlock(&context->mutex);
  
  	return err;
  }
  
  static inline int ibv_exp_query_port(struct ibv_context *context,
  				     uint8_t port_num,
  				     struct ibv_exp_port_attr *port_attr)
  {
  	struct verbs_context_exp *vctx;
  
  	if (0 == port_attr->comp_mask)
  		return ibv_query_port(context, port_num,
  				      &port_attr->port_attr);
  
  	/* Check that only valid flags were given */
  	if (!(port_attr->comp_mask & IBV_EXP_QUERY_PORT_ATTR_MASK1) ||
  	    (port_attr->comp_mask & ~IBV_EXP_QUERY_PORT_ATTR_MASKS) ||
  	    (port_attr->mask1 & ~IBV_EXP_QUERY_PORT_MASK)) {
  		errno = EINVAL;
  		return -errno;
  	}
  
  	vctx = verbs_get_exp_ctx_op(context, lib_exp_query_port);
  
  	if (!vctx) {
  		/* Fallback to legacy mode */
  		if (port_attr->comp_mask == IBV_EXP_QUERY_PORT_ATTR_MASK1 &&
  		    !(port_attr->mask1 & ~IBV_EXP_QUERY_PORT_STD_MASK))
  			return ibv_query_port(context, port_num,
  					      &port_attr->port_attr);
  
  		/* Unsupported field was requested */
  		errno = ENOSYS;
  		return -errno;
  	}
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(port_attr->comp_mask,
  						IBV_EXP_QUERY_PORT_ATTR_RESERVED - 1);
  
  	return vctx->lib_exp_query_port(context, port_num, port_attr);
  }
  
  /**
   * ibv_exp_post_task - Post a list of tasks to different QPs.
   */
  static inline int ibv_exp_post_task(struct ibv_context *context,
  				    struct ibv_exp_task *task,
  				    struct ibv_exp_task **bad_task)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context,
  							      lib_exp_post_task);
  	if (!vctx)
  		return ENOSYS;
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(task->comp_mask,
  						IBV_EXP_TASK_RESERVED - 1);
  
  	return vctx->lib_exp_post_task(context, task, bad_task);
  }
  
  static inline int ibv_exp_query_values(struct ibv_context *context, int q_values,
  				       struct ibv_exp_values *values)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context,
  							      drv_exp_query_values);
  	if (!vctx) {
  		errno = ENOSYS;
  		return -errno;
  	}
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(values->comp_mask,
  						IBV_EXP_VALUES_RESERVED - 1);
  
  	return vctx->drv_exp_query_values(context, q_values, values);
  }
  
  static inline struct ibv_exp_flow *ibv_exp_create_flow(struct ibv_qp *qp,
  						       struct ibv_exp_flow_attr *flow)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(qp->context,
  							      lib_exp_ibv_create_flow);
  	if (!vctx || !vctx->lib_exp_ibv_create_flow)
  		return NULL;
  
  	if (flow->reserved != 0L) {
  		fprintf(stderr, "%s:%d: flow->reserved must be 0\n", __FUNCTION__, __LINE__);
  		flow->reserved = 0L;
  	}
  
  	return vctx->lib_exp_ibv_create_flow(qp, flow);
  }
  
  static inline int ibv_exp_destroy_flow(struct ibv_exp_flow *flow_id)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(flow_id->context,
  							      lib_exp_ibv_destroy_flow);
  	if (!vctx || !vctx->lib_exp_ibv_destroy_flow)
  		return -ENOSYS;
  
  	return vctx->lib_exp_ibv_destroy_flow(flow_id);
  }
  
  static inline int ibv_exp_poll_cq(struct ibv_cq *ibcq, int num_entries,
  				  struct ibv_exp_wc *wc, uint32_t wc_size)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(ibcq->context,
  							      drv_exp_ibv_poll_cq);
  	if (!vctx)
  		return -ENOSYS;
  
  	return vctx->drv_exp_ibv_poll_cq(ibcq, num_entries, wc, wc_size);
  }
  
  /**
   * ibv_exp_post_send - Post a list of work requests to a send queue.
   */
  static inline int ibv_exp_post_send(struct ibv_qp *qp,
  				    struct ibv_exp_send_wr *wr,
  				    struct ibv_exp_send_wr **bad_wr)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(qp->context,
  							      drv_exp_post_send);
  	if (!vctx)
  		return -ENOSYS;
  
  	return vctx->drv_exp_post_send(qp, wr, bad_wr);
  }
  
  /**
   * ibv_exp_reg_shared_mr - Register to an existing shared memory region
   * @in - Experimental register shared MR input data.
   */
  static inline struct ibv_mr *ibv_exp_reg_shared_mr(struct ibv_exp_reg_shared_mr_in *mr_in)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(mr_in->pd->context,
  							      lib_exp_ibv_reg_shared_mr);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(mr_in->comp_mask,
  					      IBV_EXP_REG_SHARED_MR_RESERVED - 1);
  
  	return vctx->lib_exp_ibv_reg_shared_mr(mr_in);
  }
  
  /**
   * ibv_exp_modify_cq - Modifies the attributes for the specified CQ.
   * @cq: The CQ to modify.
   * @cq_attr: Specifies the CQ attributes to modify.
   * @cq_attr_mask: A bit-mask used to specify which attributes of the CQ
   *   are being modified.
   */
  static inline int ibv_exp_modify_cq(struct ibv_cq *cq,
  				    struct ibv_exp_cq_attr *cq_attr,
  				    int cq_attr_mask)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(cq->context,
  							      lib_exp_modify_cq);
  	if (!vctx)
  		return ENOSYS;
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(cq_attr->comp_mask,
  						IBV_EXP_CQ_ATTR_RESERVED - 1);
  
  	return vctx->lib_exp_modify_cq(cq, cq_attr, cq_attr_mask);
  }
  
  static inline struct ibv_cq *ibv_exp_create_cq(struct ibv_context *context,
  					       int cqe,
  					       void *cq_context,
  					       struct ibv_comp_channel *channel,
  					       int comp_vector,
  					       struct ibv_exp_cq_init_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_cq *cq;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_create_cq);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask,
  					      IBV_EXP_CQ_INIT_ATTR_RESERVED1 - 1);
  	pthread_mutex_lock(&context->mutex);
  	cq = vctx->exp_create_cq(context, cqe, channel, comp_vector, attr);
  	if (cq) {
  		cq->context		   = context;
  		cq->channel		   = channel;
  		if (channel)
  			++channel->refcnt;
  		cq->cq_context		   = cq_context;
  		cq->comp_events_completed  = 0;
  		cq->async_events_completed = 0;
  		pthread_mutex_init(&cq->mutex, NULL);
  		pthread_cond_init(&cq->cond, NULL);
  	}
  
  	pthread_mutex_unlock(&context->mutex);
  
  	return cq;
  }
  
  /**
   * ibv_exp_modify_qp - Modify a queue pair.
   * The argument exp_attr_mask specifies the QP attributes to be modified.
   * Use ibv_exp_qp_attr_mask for this argument.
   */
  static inline int
  ibv_exp_modify_qp(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, uint64_t exp_attr_mask)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(qp->context, lib_exp_modify_qp);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_QP_ATTR_RESERVED - 1);
  
  	return vctx->lib_exp_modify_qp(qp, attr, exp_attr_mask);
  }
  
  /**
   * ibv_exp_reg_mr - Register a memory region
   */
  static inline struct ibv_mr *ibv_exp_reg_mr(struct ibv_exp_reg_mr_in *in)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(in->pd->context, lib_exp_reg_mr);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(in->comp_mask,
  					      IBV_EXP_REG_MR_RESERVED - 1);
  
  	return vctx->lib_exp_reg_mr(in);
  }
  
  
  /**
   * ibv_exp_bind_mw - Bind a memory window to a region
   */
  static inline int ibv_exp_bind_mw(struct ibv_exp_mw_bind *mw_bind)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(mw_bind->mw->context, lib_exp_bind_mw);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(mw_bind->comp_mask,
  						IBV_EXP_BIND_MW_RESERVED - 1);
  
  	return vctx->lib_exp_bind_mw(mw_bind);
  }
  
  /**
   * ibv_exp_prefetch_mr - Prefetch part of a memory region.
   *
   * Can be used only with MRs registered with IBV_EXP_ACCESS_ON_DEMAND
   *
   * Returns 0 on success,
   * - ENOSYS libibverbs or provider driver doesn't support the prefetching verb.
   * - EFAULT when the range requested is out of the memory region bounds, or when
   *   parts of it are not part of the process address space.
   * - EINVAL when the MR is invalid.
   */
  static inline int ibv_exp_prefetch_mr(
  		struct ibv_mr *mr,
  		struct ibv_exp_prefetch_attr *attr)
  {
  	struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(mr->context,
  			lib_exp_prefetch_mr);
  
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_PREFETCH_MR_RESERVED - 1);
  
  	return vctx->lib_exp_prefetch_mr(mr, attr);
  }
  
  typedef int (*drv_exp_post_send_func)(struct ibv_qp *qp,
  				 struct ibv_exp_send_wr *wr,
  				 struct ibv_exp_send_wr **bad_wr);
  typedef int (*drv_post_send_func)(struct ibv_qp *qp, struct ibv_send_wr *wr,
  				struct ibv_send_wr **bad_wr);
  typedef int (*drv_exp_poll_cq_func)(struct ibv_cq *ibcq, int num_entries,
  				   struct ibv_exp_wc *wc, uint32_t wc_size);
  typedef int (*drv_poll_cq_func)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc);
  typedef int (*drv_post_recv_func)(struct ibv_qp *qp, struct ibv_recv_wr *wr,
  			 struct ibv_recv_wr **bad_wr);
  
  static inline void *ibv_exp_get_provider_func(struct ibv_context *context,
  						enum ibv_exp_func_name name)
  {
  	struct verbs_context_exp *vctx;
  
  	switch (name) {
  	case IBV_EXP_POST_SEND_FUNC:
  		vctx = verbs_get_exp_ctx_op(context, drv_exp_post_send);
  		if (!vctx)
  			goto error;
  
  		return (void *)vctx->drv_exp_post_send;
  
  	case IBV_EXP_POLL_CQ_FUNC:
  		vctx = verbs_get_exp_ctx_op(context, drv_exp_ibv_poll_cq);
  		if (!vctx)
  			goto error;
  
  		return (void *)vctx->drv_exp_ibv_poll_cq;
  
  	case IBV_POST_SEND_FUNC:
  		if (!context->ops.post_send)
  			goto error;
  
  		return (void *)context->ops.post_send;
  
  	case IBV_POLL_CQ_FUNC:
  		if (!context->ops.poll_cq)
  			goto error;
  
  		return (void *)context->ops.poll_cq;
  
  	case IBV_POST_RECV_FUNC:
  		if (!context->ops.post_recv)
  			goto error;
  
  		return (void *)context->ops.post_recv;
  
  	default:
  		break;
  	}
  
  error:
  	errno = ENOSYS;
  	return NULL;
  }
  
  static inline struct ibv_mr *ibv_exp_create_mr(struct ibv_exp_create_mr_in *in)
  {
  	struct verbs_context_exp *vctx;
  	struct ibv_mr *mr;
  
  	vctx = verbs_get_exp_ctx_op(in->pd->context, lib_exp_create_mr);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(in->comp_mask,
  					      IBV_EXP_CREATE_MR_IN_RESERVED - 1);
  	mr = vctx->lib_exp_create_mr(in);
  	if (mr)
  		mr->pd = in->pd;
  
  	return mr;
  }
  
  static inline int ibv_exp_query_mkey(struct ibv_mr *mr,
  				     struct ibv_exp_mkey_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(mr->context, lib_exp_query_mkey);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_MKEY_ATTR_RESERVED - 1);
  
  	return vctx->lib_exp_query_mkey(mr, attr);
  }
  
  static inline int ibv_exp_dealloc_mkey_list_memory(struct ibv_exp_mkey_list_container *mem)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(mem->context,
  				    lib_exp_dealloc_mkey_list_memory);
  	if (!vctx) {
  		errno = ENOSYS;
  		return errno;
  	}
  
  	return vctx->lib_exp_dealloc_mkey_list_memory(mem);
  }
  
  static inline struct ibv_exp_mkey_list_container *
  ibv_exp_alloc_mkey_list_memory(struct ibv_exp_mkey_list_container_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(attr->pd->context,
  				    lib_exp_alloc_mkey_list_memory);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask,
  					      IBV_EXP_MKEY_LIST_CONTAINER_RESERVED - 1);
  
  	return vctx->lib_exp_alloc_mkey_list_memory(attr);
  }
  
  /**
   * ibv_exp_create_res_domain - create resource domain
   */
  static inline struct ibv_exp_res_domain *ibv_exp_create_res_domain(struct ibv_context *context,
  								   struct ibv_exp_res_domain_init_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_create_res_domain);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask,
  					      IBV_EXP_RES_DOMAIN_RESERVED - 1);
  
  	return vctx->exp_create_res_domain(context, attr);
  }
  
  /**
   * ibv_exp_destroy_res_domain - destroy resource domain
   */
  static inline int ibv_exp_destroy_res_domain(struct ibv_context *context,
  					     struct ibv_exp_res_domain *res_dom,
  					     struct ibv_exp_destroy_res_domain_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_destroy_res_domain);
  	if (!vctx)
  		return errno = ENOSYS;
  
  	if (attr)
  		IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  							IBV_EXP_DESTROY_RES_DOMAIN_RESERVED - 1);
  
  	return vctx->exp_destroy_res_domain(context, res_dom, attr);
  }
  
  /**
   * ibv_exp_query_intf - query for family of verbs interface for specific QP/CQ
   *
   * Usually family of data-path verbs.
   * Application may call ibv_exp_query_intf for QPs in the following states:
   *    IBV_QPS_INIT, IBV_QPS_RTR and IBV_QPS_RTS
   *
   * Returns the family of verbs.
   * On failure returns NULL. The failure reason provided by the 'status'
   * output variable.
   */
  static inline void *ibv_exp_query_intf(struct ibv_context *context,
  				       struct ibv_exp_query_intf_params *params,
  				       enum ibv_exp_query_intf_status *status)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_query_intf);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(params->comp_mask,
  					      IBV_EXP_QUERY_INTF_RESERVED - 1);
  
  	return vctx->exp_query_intf(context, params, status);
  }
  
  /**
   * ibv_exp_release_intf - release the queried interface
   */
  static inline int ibv_exp_release_intf(struct ibv_context *context, void *intf,
  				       struct ibv_exp_release_intf_params *params)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_release_intf);
  	if (!vctx)
  		return errno = ENOSYS;
  
  	if (params)
  		IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(params->comp_mask,
  							IBV_EXP_RELEASE_INTF_RESERVED - 1);
  
  	return vctx->exp_release_intf(context, intf, params);
  }
  
  static inline struct ibv_exp_wq *ibv_exp_create_wq(struct ibv_context *context,
  						   struct ibv_exp_wq_init_attr *wq_init_attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_create_wq);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(wq_init_attr->comp_mask,
  							IBV_EXP_CREATE_WQ_RESERVED - 1);
  
  	return vctx->exp_create_wq(context, wq_init_attr);
  }
  
  static inline int ibv_exp_modify_wq(struct ibv_exp_wq *wq, struct ibv_exp_wq_attr *wq_attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(wq->context, exp_modify_wq);
  	if (!vctx)
  		return ENOSYS;
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(wq_attr->attr_mask,
  							IBV_EXP_WQ_ATTR_RESERVED - 1);
  	return vctx->exp_modify_wq(wq, wq_attr);
  }
  
  static inline int ibv_exp_destroy_wq(struct ibv_exp_wq *wq)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(wq->context, exp_destroy_wq);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->exp_destroy_wq(wq);
  }
  
  /*
   * ibv_exp_create_rwq_ind_table - Creates a RQ Indirection Table associated
   * with the specified protection domain.
   * @pd: The protection domain associated with the Indirection Table.
   * @ibv_exp_rwq_ind_table_init_attr: A list of initial attributes required to
   * create the Indirection Table.
   * Return Value
   * ibv_exp_create_rwq_ind_table returns a pointer to the created
   * Indirection Table, or NULL if the request fails.
   */
  static inline struct ibv_exp_rwq_ind_table *ibv_exp_create_rwq_ind_table(struct ibv_context *context,
  									 struct ibv_exp_rwq_ind_table_init_attr *init_attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_create_rwq_ind_table);
  	if (!vctx) {
  		errno = ENOSYS;
  		return NULL;
  	}
  
  	IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(init_attr->comp_mask,
  					      IBV_EXP_CREATE_IND_TABLE_RESERVED - 1);
  	return vctx->exp_create_rwq_ind_table(context, init_attr);
  }
  
  /*
   * ibv_exp_destroy_rwq_ind_table - Destroys the specified Indirection Table.
   * @rwq_ind_table: The Indirection Table to destroy.
   * Return Value
   * ibv_destroy_rwq_ind_table() returns 0 on success, or the value of errno
   * on failure (which indicates the failure reason).
  */
  static inline int ibv_exp_destroy_rwq_ind_table(struct ibv_exp_rwq_ind_table *rwq_ind_table)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(rwq_ind_table->context, exp_destroy_rwq_ind_table);
  	if (!vctx)
  		return ENOSYS;
  
  	return vctx->exp_destroy_rwq_ind_table(rwq_ind_table);
  }
  
  /*
   * ibv_exp_query_gid_attr - query a GID attributes
   * @context: ib context
   * @port_num: port number
   * @index: gid index in the gids table
   * @attr: the gid attributes of index in the gids table
   * Return value
   * ibv_exp_query_gid_attr return 0 on success, or the value of errno on failure.
   */
  static inline int ibv_exp_query_gid_attr(struct ibv_context *context,
  					 uint8_t port_num,
  					 unsigned int index,
  					 struct ibv_exp_gid_attr *attr)
  {
  	struct verbs_context_exp *vctx;
  
  	vctx = verbs_get_exp_ctx_op(context, exp_query_gid_attr);
  	if (!vctx)
  		return ENOSYS;
  
  	IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask,
  						IBV_EXP_QUERY_GID_ATTR_RESERVED - 1);
  	return vctx->exp_query_gid_attr(context, port_num, index, attr);
  }
  END_C_DECLS
  
  #define VERBS_MAX_ENV_VAL 4096
  
  #  undef __attribute_const
  
  
  #endif /* INFINIBAND_VERBS_EXP_H */
  