text: /*
   * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   *
   * This software is available to you under a choice of one of two
   * licenses.  You may choose to be licensed under the terms of the GNU
   * General Public License (GPL) Version 2, available from the file
   * COPYING in the main directory of this source tree, or the
   * OpenIB.org BSD license below:
   *
   *     Redistribution and use in source and binary forms, with or
   *     without modification, are permitted provided that the following
   *     conditions are met:
   *
   *      - Redistributions of source code must retain the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer.
   *
   *      - Redistributions in binary form must reproduce the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer in the documentation and/or other materials
   *        provided with the distribution.
   *
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   * SOFTWARE.
   */
  
  #ifndef MTHCA_H
  #define MTHCA_H
  
  #include <stddef.h>
  
  #include <infiniband/driver.h>
  #include <util/udma_barrier.h>
  
  #include <valgrind/memcheck.h>
  
  #define PFX		"mthca: "
  
  enum mthca_hca_type {
  	MTHCA_TAVOR,
  	MTHCA_ARBEL
  };
  
  enum {
  	MTHCA_CQ_ENTRY_SIZE = 0x20
  };
  
  enum {
  	MTHCA_QP_TABLE_BITS = 8,
  	MTHCA_QP_TABLE_SIZE = 1 << MTHCA_QP_TABLE_BITS,
  	MTHCA_QP_TABLE_MASK = MTHCA_QP_TABLE_SIZE - 1
  };
  
  enum {
  	MTHCA_DB_REC_PAGE_SIZE = 4096,
  	MTHCA_DB_REC_PER_PAGE  = MTHCA_DB_REC_PAGE_SIZE / 8
  };
  
  enum mthca_db_type {
  	MTHCA_DB_TYPE_INVALID   = 0x0,
  	MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
  	MTHCA_DB_TYPE_CQ_ARM    = 0x2,
  	MTHCA_DB_TYPE_SQ        = 0x3,
  	MTHCA_DB_TYPE_RQ        = 0x4,
  	MTHCA_DB_TYPE_SRQ       = 0x5,
  	MTHCA_DB_TYPE_GROUP_SEP = 0x7
  };
  
  enum {
  	MTHCA_OPCODE_NOP            = 0x00,
  	MTHCA_OPCODE_RDMA_WRITE     = 0x08,
  	MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09,
  	MTHCA_OPCODE_SEND           = 0x0a,
  	MTHCA_OPCODE_SEND_IMM       = 0x0b,
  	MTHCA_OPCODE_RDMA_READ      = 0x10,
  	MTHCA_OPCODE_ATOMIC_CS      = 0x11,
  	MTHCA_OPCODE_ATOMIC_FA      = 0x12,
  	MTHCA_OPCODE_BIND_MW        = 0x18,
  	MTHCA_OPCODE_INVALID        = 0xff
  };
  
  struct mthca_ah_page;
  
  struct mthca_device {
  	struct verbs_device ibv_dev;
  	enum mthca_hca_type hca_type;
  	int                 page_size;
  };
  
  struct mthca_db_table;
  
  struct mthca_context {
  	struct verbs_context     ibv_ctx;
  	void                  *uar;
  	pthread_spinlock_t     uar_lock;
  	struct mthca_db_table *db_tab;
  	struct ibv_pd         *pd;
  	struct {
  		struct mthca_qp	**table;
  		int		  refcnt;
  	}		       qp_table[MTHCA_QP_TABLE_SIZE];
  	pthread_mutex_t        qp_table_mutex;
  	int                    num_qps;
  	int		       qp_table_shift;
  	int		       qp_table_mask;
  };
  
  struct mthca_buf {
  	void		       *buf;
  	size_t			length;
  };
  
  struct mthca_pd {
  	struct ibv_pd         ibv_pd;
  	struct mthca_ah_page *ah_list;
  	pthread_mutex_t       ah_mutex;
  	uint32_t              pdn;
  };
  
  struct mthca_cq {
  	struct ibv_cq  	   ibv_cq;
  	struct mthca_buf   buf;
  	pthread_spinlock_t lock;
  	struct ibv_mr  	  *mr;
  	uint32_t       	   cqn;
  	uint32_t       	   cons_index;
  
  	/* Next fields are mem-free only */
  	int                set_ci_db_index;
  	__be32            *set_ci_db;
  	int                arm_db_index;
  	__be32            *arm_db;
  	int                arm_sn;
  };
  
  struct mthca_srq {
  	struct ibv_srq     ibv_srq;
  	struct mthca_buf   buf;
  	void           	  *last;
  	pthread_spinlock_t lock;
  	struct ibv_mr 	  *mr;
  	uint64_t      	  *wrid;
  	uint32_t       	   srqn;
  	int            	   max;
  	int            	   max_gs;
  	int            	   wqe_shift;
  	int            	   first_free;
  	int            	   last_free;
  	int                buf_size;
  
  	/* Next fields are mem-free only */
  	int           	   db_index;
  	__be32		  *db;
  	uint16_t      	   counter;
  };
  
  struct mthca_wq {
  	pthread_spinlock_t lock;
  	int            	   max;
  	unsigned       	   next_ind;
  	unsigned       	   last_comp;
  	unsigned       	   head;
  	unsigned       	   tail;
  	void           	  *last;
  	int            	   max_gs;
  	int            	   wqe_shift;
  
  	/* Next fields are mem-free only */
  	int                db_index;
  	__be32		  *db;
  };
  
  struct mthca_qp {
  	struct ibv_qp    ibv_qp;
  	struct mthca_buf buf;
  	uint64_t        *wrid;
  	int              send_wqe_offset;
  	int              max_inline_data;
  	int              buf_size;
  	struct mthca_wq  sq;
  	struct mthca_wq  rq;
  	struct ibv_mr   *mr;
  	int              sq_sig_all;
  };
  
  struct mthca_av {
  	__be32   port_pd;
  	uint8_t  reserved1;
  	uint8_t  g_slid;
  	__be16   dlid;
  	uint8_t  reserved2;
  	uint8_t  gid_index;
  	uint8_t  msg_sr;
  	uint8_t  hop_limit;
  	__be32   sl_tclass_flowlabel;
  	__be32   dgid[4];
  };
  
  struct mthca_ah {
  	struct ibv_ah         ibv_ah;
  	struct mthca_av      *av;
  	struct mthca_ah_page *page;
  	uint32_t              key;
  };
  
  static inline unsigned long align(unsigned long val, unsigned long align)
  {
  	return (val + align - 1) & ~(align - 1);
  }
  
  static inline uintptr_t db_align(__be32 *db)
  {
  	return (uintptr_t) db & ~((uintptr_t) MTHCA_DB_REC_PAGE_SIZE - 1);
  }
  
  #define to_mxxx(xxx, type) container_of(ib##xxx, struct mthca_##type, ibv_##xxx)
  
  static inline struct mthca_device *to_mdev(struct ibv_device *ibdev)
  {
  	return container_of(ibdev, struct mthca_device, ibv_dev.device);
  }
  
  static inline struct mthca_context *to_mctx(struct ibv_context *ibctx)
  {
  	return container_of(ibctx, struct mthca_context, ibv_ctx.context);
  }
  
  static inline struct mthca_pd *to_mpd(struct ibv_pd *ibpd)
  {
  	return to_mxxx(pd, pd);
  }
  
  static inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq)
  {
  	return to_mxxx(cq, cq);
  }
  
  static inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq)
  {
  	return to_mxxx(srq, srq);
  }
  
  static inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp)
  {
  	return to_mxxx(qp, qp);
  }
  
  static inline struct mthca_ah *to_mah(struct ibv_ah *ibah)
  {
  	return to_mxxx(ah, ah);
  }
  
  static inline int mthca_is_memfree(struct ibv_context *ibctx)
  {
  	return to_mdev(ibctx->device)->hca_type == MTHCA_ARBEL;
  }
  
  int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size);
  void mthca_free_buf(struct mthca_buf *buf);
  
  int mthca_alloc_db(struct mthca_db_table *db_tab, enum mthca_db_type type,
  		   __be32 **db);
  void mthca_set_db_qn(__be32 *db, enum mthca_db_type type, uint32_t qn);
  void mthca_free_db(struct mthca_db_table *db_tab, enum mthca_db_type type, int db_index);
  struct mthca_db_table *mthca_alloc_db_tab(int uarc_size);
  void mthca_free_db_tab(struct mthca_db_table *db_tab);
  
  int mthca_query_device(struct ibv_context *context,
  		       struct ibv_device_attr *attr);
  int mthca_query_port(struct ibv_context *context, uint8_t port,
  		     struct ibv_port_attr *attr);
  
  struct ibv_pd *mthca_alloc_pd(struct ibv_context *context);
  int mthca_free_pd(struct ibv_pd *pd);
  
  struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
  			    uint64_t hca_va, int access);
  int mthca_dereg_mr(struct verbs_mr *mr);
  
  struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe,
  			       struct ibv_comp_channel *channel,
  			       int comp_vector);
  int mthca_resize_cq(struct ibv_cq *cq, int cqe);
  int mthca_destroy_cq(struct ibv_cq *cq);
  int mthca_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
  int mthca_tavor_arm_cq(struct ibv_cq *cq, int solicited);
  int mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited);
  void mthca_arbel_cq_event(struct ibv_cq *cq);
  void __mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
  void mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
  void mthca_cq_resize_copy_cqes(struct mthca_cq *cq, void *buf, int new_cqe);
  int mthca_alloc_cq_buf(struct mthca_device *dev, struct mthca_buf *buf, int nent);
  
  struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
  				 struct ibv_srq_init_attr *attr);
  int mthca_modify_srq(struct ibv_srq *srq,
  		     struct ibv_srq_attr *attr,
  		     int mask);
  int mthca_query_srq(struct ibv_srq *srq,
  			   struct ibv_srq_attr *attr);
  int mthca_destroy_srq(struct ibv_srq *srq);
  int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
  			struct mthca_srq *srq);
  void mthca_free_srq_wqe(struct mthca_srq *srq, int ind);
  int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
  			      struct ibv_recv_wr *wr,
  			      struct ibv_recv_wr **bad_wr);
  int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
  			      struct ibv_recv_wr *wr,
  			      struct ibv_recv_wr **bad_wr);
  
  struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
  int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  		   int attr_mask,
  		   struct ibv_qp_init_attr *init_attr);
  int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  		    int attr_mask);
  int mthca_destroy_qp(struct ibv_qp *qp);
  void mthca_init_qp_indices(struct mthca_qp *qp);
  int mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
  			  struct ibv_send_wr **bad_wr);
  int mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
  			  struct ibv_recv_wr **bad_wr);
  int mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
  			  struct ibv_send_wr **bad_wr);
  int mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
  			  struct ibv_recv_wr **bad_wr);
  int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
  		       enum ibv_qp_type type, struct mthca_qp *qp);
  struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn);
  int mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp);
  void mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn);
  int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
  		       int index, int *dbd, __be32 *new_wqe);
  struct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
  int mthca_destroy_ah(struct ibv_ah *ah);
  int mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr,
  		   struct mthca_ah *ah);
  void mthca_free_av(struct mthca_ah *ah);
  
  #endif /* MTHCA_H */
  