text: // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
  /*
   * Copyright(c) 2020 Intel Corporation.
   *
   */
  
  /*
   * This file contains HFI1 support for IPOIB SDMA functionality
   */
  
  #include <linux/log2.h>
  #include <linux/circ_buf.h>
  
  #include "sdma.h"
  #include "verbs.h"
  #include "trace_ibhdrs.h"
  #include "ipoib.h"
  
  /* Add a convenience helper */
  #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
  #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
  #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
  
  /**
   * struct ipoib_txreq - IPOIB transmit descriptor
   * @txreq: sdma transmit request
   * @sdma_hdr: 9b ib headers
   * @sdma_status: status returned by sdma engine
   * @priv: ipoib netdev private data
   * @txq: txq on which skb was output
   * @skb: skb to send
   */
  struct ipoib_txreq {
  	struct sdma_txreq           txreq;
  	struct hfi1_sdma_header     sdma_hdr;
  	int                         sdma_status;
  	struct hfi1_ipoib_dev_priv *priv;
  	struct hfi1_ipoib_txq      *txq;
  	struct sk_buff             *skb;
  };
  
  struct ipoib_txparms {
  	struct hfi1_devdata        *dd;
  	struct rdma_ah_attr        *ah_attr;
  	struct hfi1_ibport         *ibp;
  	struct hfi1_ipoib_txq      *txq;
  	union hfi1_ipoib_flow       flow;
  	u32                         dqpn;
  	u8                          hdr_dwords;
  	u8                          entropy;
  };
  
  static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
  {
  	return sent - completed;
  }
  
  static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
  {
  	return hfi1_ipoib_txreqs(txq->sent_txreqs,
  				 atomic64_read(&txq->complete_txreqs));
  }
  
  static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
  {
  	if (atomic_inc_return(&txq->stops) == 1)
  		netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
  }
  
  static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
  {
  	if (atomic_dec_and_test(&txq->stops))
  		netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
  }
  
  static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
  {
  	return min_t(uint, txq->priv->netdev->tx_queue_len,
  		     txq->tx_ring.max_items - 1);
  }
  
  static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
  {
  	return min_t(uint, txq->priv->netdev->tx_queue_len,
  		     txq->tx_ring.max_items) >> 1;
  }
  
  static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
  {
  	++txq->sent_txreqs;
  	if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
  	    !atomic_xchg(&txq->ring_full, 1))
  		hfi1_ipoib_stop_txq(txq);
  }
  
  static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
  {
  	struct net_device *dev = txq->priv->netdev;
  
  	/* If shutting down just return as queue state is irrelevant */
  	if (unlikely(dev->reg_state != NETREG_REGISTERED))
  		return;
  
  	/*
  	 * When the queue has been drained to less than half full it will be
  	 * restarted.
  	 * The size of the txreq ring is fixed at initialization.
  	 * The tx queue len can be adjusted upward while the interface is
  	 * running.
  	 * The tx queue len can be large enough to overflow the txreq_ring.
  	 * Use the minimum of the current tx_queue_len or the rings max txreqs
  	 * to protect against ring overflow.
  	 */
  	if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
  	    atomic_xchg(&txq->ring_full, 0))
  		hfi1_ipoib_wake_txq(txq);
  }
  
  static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
  {
  	struct hfi1_ipoib_dev_priv *priv = tx->priv;
  
  	if (likely(!tx->sdma_status)) {
  		dev_sw_netstats_tx_add(priv->netdev, 1, tx->skb->len);
  	} else {
  		++priv->netdev->stats.tx_errors;
  		dd_dev_warn(priv->dd,
  			    "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
  			    __func__, tx->sdma_status,
  			    le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx,
  			    tx->txq->sde->this_idx);
  	}
  
  	napi_consume_skb(tx->skb, budget);
  	sdma_txclean(priv->dd, &tx->txreq);
  	kmem_cache_free(priv->txreq_cache, tx);
  }
  
  static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget)
  {
  	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
  	unsigned long head;
  	unsigned long tail;
  	unsigned int max_tx;
  	int work_done;
  	int tx_count;
  
  	spin_lock_bh(&tx_ring->consumer_lock);
  
  	/* Read index before reading contents at that index. */
  	head = smp_load_acquire(&tx_ring->head);
  	tail = tx_ring->tail;
  	max_tx = tx_ring->max_items;
  
  	work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget);
  
  	for (tx_count = work_done; tx_count; tx_count--) {
  		hfi1_ipoib_free_tx(tx_ring->items[tail], budget);
  		tail = CIRC_NEXT(tail, max_tx);
  	}
  
  	atomic64_add(work_done, &txq->complete_txreqs);
  
  	/* Finished freeing tx items so store the tail value. */
  	smp_store_release(&tx_ring->tail, tail);
  
  	spin_unlock_bh(&tx_ring->consumer_lock);
  
  	hfi1_ipoib_check_queue_stopped(txq);
  
  	return work_done;
  }
  
  static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev);
  	struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis];
  
  	int work_done = hfi1_ipoib_drain_tx_ring(txq, budget);
  
  	if (work_done < budget)
  		napi_complete_done(napi, work_done);
  
  	return work_done;
  }
  
  static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
  {
  	struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring;
  	unsigned long head;
  	unsigned long tail;
  	size_t max_tx;
  
  	spin_lock(&tx_ring->producer_lock);
  
  	head = tx_ring->head;
  	tail = READ_ONCE(tx_ring->tail);
  	max_tx = tx_ring->max_items;
  
  	if (likely(CIRC_SPACE(head, tail, max_tx))) {
  		tx_ring->items[head] = tx;
  
  		/* Finish storing txreq before incrementing head. */
  		smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
  		napi_schedule(tx->txq->napi);
  	} else {
  		struct hfi1_ipoib_txq *txq = tx->txq;
  		struct hfi1_ipoib_dev_priv *priv = tx->priv;
  
  		/* Ring was full */
  		hfi1_ipoib_free_tx(tx, 0);
  		atomic64_inc(&txq->complete_txreqs);
  		dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx);
  	}
  
  	spin_unlock(&tx_ring->producer_lock);
  }
  
  static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
  {
  	struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
  
  	tx->sdma_status = status;
  
  	hfi1_ipoib_add_tx(tx);
  }
  
  static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
  					struct ipoib_txparms *txp)
  {
  	struct hfi1_devdata *dd = txp->dd;
  	struct sdma_txreq *txreq = &tx->txreq;
  	struct sk_buff *skb = tx->skb;
  	int ret = 0;
  	int i;
  
  	if (skb_headlen(skb)) {
  		ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
  		if (unlikely(ret))
  			return ret;
  	}
  
  	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  
  		ret = sdma_txadd_page(dd,
  				      txreq,
  				      skb_frag_page(frag),
  				      frag->bv_offset,
  				      skb_frag_size(frag));
  		if (unlikely(ret))
  			break;
  	}
  
  	return ret;
  }
  
  static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
  				    struct ipoib_txparms *txp)
  {
  	struct hfi1_devdata *dd = txp->dd;
  	struct sdma_txreq *txreq = &tx->txreq;
  	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
  	u16 pkt_bytes =
  		sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
  	int ret;
  
  	ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
  	if (unlikely(ret))
  		return ret;
  
  	/* add pbc + headers */
  	ret = sdma_txadd_kvaddr(dd,
  				txreq,
  				sdma_hdr,
  				sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
  	if (unlikely(ret))
  		return ret;
  
  	/* add the ulp payload */
  	return hfi1_ipoib_build_ulp_payload(tx, txp);
  }
  
  static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
  					   struct ipoib_txparms *txp)
  {
  	struct hfi1_ipoib_dev_priv *priv = tx->priv;
  	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
  	struct sk_buff *skb = tx->skb;
  	struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
  	struct rdma_ah_attr *ah_attr = txp->ah_attr;
  	struct ib_other_headers *ohdr;
  	struct ib_grh *grh;
  	u16 dwords;
  	u16 slid;
  	u16 dlid;
  	u16 lrh0;
  	u32 bth0;
  	u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
  			 priv->netdev->dev_addr[2] << 8 |
  			 priv->netdev->dev_addr[3]);
  	u16 payload_dwords;
  	u8 pad_cnt;
  
  	pad_cnt = -skb->len & 3;
  
  	/* Includes ICRC */
  	payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
  
  	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
  	txp->hdr_dwords = 7;
  
  	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
  		grh = &sdma_hdr->hdr.ibh.u.l.grh;
  		txp->hdr_dwords +=
  			hfi1_make_grh(txp->ibp,
  				      grh,
  				      rdma_ah_read_grh(ah_attr),
  				      txp->hdr_dwords - LRH_9B_DWORDS,
  				      payload_dwords);
  		lrh0 = HFI1_LRH_GRH;
  		ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
  	} else {
  		lrh0 = HFI1_LRH_BTH;
  		ohdr = &sdma_hdr->hdr.ibh.u.oth;
  	}
  
  	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
  	lrh0 |= (txp->flow.sc5 & 0xf) << 12;
  
  	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
  	if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
  		slid = be16_to_cpu(IB_LID_PERMISSIVE);
  	} else {
  		u16 lid = (u16)ppd->lid;
  
  		if (lid) {
  			lid |= rdma_ah_get_path_bits(ah_attr) &
  				((1 << ppd->lmc) - 1);
  			slid = lid;
  		} else {
  			slid = be16_to_cpu(IB_LID_PERMISSIVE);
  		}
  	}
  
  	/* Includes ICRC */
  	dwords = txp->hdr_dwords + payload_dwords;
  
  	/* Build the lrh */
  	sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
  	hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
  
  	/* Build the bth */
  	bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
  
  	ohdr->bth[0] = cpu_to_be32(bth0);
  	ohdr->bth[1] = cpu_to_be32(txp->dqpn);
  	ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs));
  
  	/* Build the deth */
  	ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
  	ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
  					  HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
  
  	/* Construct the pbc. */
  	sdma_hdr->pbc =
  		cpu_to_le64(create_pbc(ppd,
  				       ib_is_sc5(txp->flow.sc5) <<
  							      PBC_DC_INFO_SHIFT,
  				       0,
  				       sc_to_vlt(priv->dd, txp->flow.sc5),
  				       dwords - SIZE_OF_CRC +
  						(sizeof(sdma_hdr->pbc) >> 2)));
  }
  
  static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
  						      struct sk_buff *skb,
  						      struct ipoib_txparms *txp)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
  	struct ipoib_txreq *tx;
  	int ret;
  
  	tx = kmem_cache_alloc_node(priv->txreq_cache,
  				   GFP_ATOMIC,
  				   priv->dd->node);
  	if (unlikely(!tx))
  		return ERR_PTR(-ENOMEM);
  
  	/* so that we can test if the sdma descriptors are there */
  	tx->txreq.num_desc = 0;
  	tx->priv = priv;
  	tx->txq = txp->txq;
  	tx->skb = skb;
  	INIT_LIST_HEAD(&tx->txreq.list);
  
  	hfi1_ipoib_build_ib_tx_headers(tx, txp);
  
  	ret = hfi1_ipoib_build_tx_desc(tx, txp);
  	if (likely(!ret)) {
  		if (txp->txq->flow.as_int != txp->flow.as_int) {
  			txp->txq->flow.tx_queue = txp->flow.tx_queue;
  			txp->txq->flow.sc5 = txp->flow.sc5;
  			txp->txq->sde =
  				sdma_select_engine_sc(priv->dd,
  						      txp->flow.tx_queue,
  						      txp->flow.sc5);
  		}
  
  		return tx;
  	}
  
  	sdma_txclean(priv->dd, &tx->txreq);
  	kmem_cache_free(priv->txreq_cache, tx);
  
  	return ERR_PTR(ret);
  }
  
  static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
  				     struct hfi1_ipoib_txq *txq)
  {
  	int ret;
  	u16 count_out;
  
  	ret = sdma_send_txlist(txq->sde,
  			       iowait_get_ib_work(&txq->wait),
  			       &txq->tx_list,
  			       &count_out);
  	if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
  		return ret;
  
  	dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
  
  	return ret;
  }
  
  static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
  				    struct hfi1_ipoib_txq *txq)
  {
  	int ret = 0;
  
  	if (!list_empty(&txq->tx_list)) {
  		/* Flush the current list */
  		ret = hfi1_ipoib_submit_tx_list(dev, txq);
  
  		if (unlikely(ret))
  			if (ret != -EBUSY)
  				++dev->stats.tx_carrier_errors;
  	}
  
  	return ret;
  }
  
  static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
  				struct ipoib_txreq *tx)
  {
  	int ret;
  
  	ret = sdma_send_txreq(txq->sde,
  			      iowait_get_ib_work(&txq->wait),
  			      &tx->txreq,
  			      txq->pkts_sent);
  	if (likely(!ret)) {
  		txq->pkts_sent = true;
  		iowait_starve_clear(txq->pkts_sent, &txq->wait);
  	}
  
  	return ret;
  }
  
  static int hfi1_ipoib_send_dma_single(struct net_device *dev,
  				      struct sk_buff *skb,
  				      struct ipoib_txparms *txp)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
  	struct hfi1_ipoib_txq *txq = txp->txq;
  	struct ipoib_txreq *tx;
  	int ret;
  
  	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
  	if (IS_ERR(tx)) {
  		int ret = PTR_ERR(tx);
  
  		dev_kfree_skb_any(skb);
  
  		if (ret == -ENOMEM)
  			++dev->stats.tx_errors;
  		else
  			++dev->stats.tx_carrier_errors;
  
  		return NETDEV_TX_OK;
  	}
  
  	ret = hfi1_ipoib_submit_tx(txq, tx);
  	if (likely(!ret)) {
  tx_ok:
  		trace_sdma_output_ibhdr(tx->priv->dd,
  					&tx->sdma_hdr.hdr,
  					ib_is_sc5(txp->flow.sc5));
  		hfi1_ipoib_check_queue_depth(txq);
  		return NETDEV_TX_OK;
  	}
  
  	txq->pkts_sent = false;
  
  	if (ret == -EBUSY || ret == -ECOMM)
  		goto tx_ok;
  
  	sdma_txclean(priv->dd, &tx->txreq);
  	dev_kfree_skb_any(skb);
  	kmem_cache_free(priv->txreq_cache, tx);
  	++dev->stats.tx_carrier_errors;
  
  	return NETDEV_TX_OK;
  }
  
  static int hfi1_ipoib_send_dma_list(struct net_device *dev,
  				    struct sk_buff *skb,
  				    struct ipoib_txparms *txp)
  {
  	struct hfi1_ipoib_txq *txq = txp->txq;
  	struct ipoib_txreq *tx;
  
  	/* Has the flow change ? */
  	if (txq->flow.as_int != txp->flow.as_int) {
  		int ret;
  
  		ret = hfi1_ipoib_flush_tx_list(dev, txq);
  		if (unlikely(ret)) {
  			if (ret == -EBUSY)
  				++dev->stats.tx_dropped;
  			dev_kfree_skb_any(skb);
  			return NETDEV_TX_OK;
  		}
  	}
  	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
  	if (IS_ERR(tx)) {
  		int ret = PTR_ERR(tx);
  
  		dev_kfree_skb_any(skb);
  
  		if (ret == -ENOMEM)
  			++dev->stats.tx_errors;
  		else
  			++dev->stats.tx_carrier_errors;
  
  		return NETDEV_TX_OK;
  	}
  
  	list_add_tail(&tx->txreq.list, &txq->tx_list);
  
  	hfi1_ipoib_check_queue_depth(txq);
  
  	trace_sdma_output_ibhdr(tx->priv->dd,
  				&tx->sdma_hdr.hdr,
  				ib_is_sc5(txp->flow.sc5));
  
  	if (!netdev_xmit_more())
  		(void)hfi1_ipoib_flush_tx_list(dev, txq);
  
  	return NETDEV_TX_OK;
  }
  
  static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
  {
  	if (skb_transport_header_was_set(skb)) {
  		u8 *hdr = (u8 *)skb_transport_header(skb);
  
  		return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
  	}
  
  	return (u8)skb_get_queue_mapping(skb);
  }
  
  int hfi1_ipoib_send_dma(struct net_device *dev,
  			struct sk_buff *skb,
  			struct ib_ah *address,
  			u32 dqpn)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
  	struct ipoib_txparms txp;
  	struct rdma_netdev *rn = netdev_priv(dev);
  
  	if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
  		dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
  			    skb->len,
  			    rn->mtu + HFI1_IPOIB_ENCAP_LEN);
  		++dev->stats.tx_dropped;
  		++dev->stats.tx_errors;
  		dev_kfree_skb_any(skb);
  		return NETDEV_TX_OK;
  	}
  
  	txp.dd = priv->dd;
  	txp.ah_attr = &ibah_to_rvtah(address)->attr;
  	txp.ibp = to_iport(priv->device, priv->port_num);
  	txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
  	txp.dqpn = dqpn;
  	txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
  	txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
  	txp.entropy = hfi1_ipoib_calc_entropy(skb);
  
  	if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
  		return hfi1_ipoib_send_dma_list(dev, skb, &txp);
  
  	return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
  }
  
  /*
   * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
   *
   * This function gets called from sdma_send_txreq() when there are not enough
   * sdma descriptors available to send the packet. It adds Tx queue's wait
   * structure to sdma engine's dmawait list to be woken up when descriptors
   * become available.
   */
  static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
  				 struct iowait_work *wait,
  				 struct sdma_txreq *txreq,
  				 uint seq,
  				 bool pkts_sent)
  {
  	struct hfi1_ipoib_txq *txq =
  		container_of(wait->iow, struct hfi1_ipoib_txq, wait);
  
  	write_seqlock(&sde->waitlock);
  
  	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
  		if (sdma_progress(sde, seq, txreq)) {
  			write_sequnlock(&sde->waitlock);
  			return -EAGAIN;
  		}
  
  		if (list_empty(&txreq->list))
  			/* came from non-list submit */
  			list_add_tail(&txreq->list, &txq->tx_list);
  		if (list_empty(&txq->wait.list)) {
  			if (!atomic_xchg(&txq->no_desc, 1))
  				hfi1_ipoib_stop_txq(txq);
  			iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
  		}
  
  		write_sequnlock(&sde->waitlock);
  		return -EBUSY;
  	}
  
  	write_sequnlock(&sde->waitlock);
  	return -EINVAL;
  }
  
  /*
   * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
   *
   * This function gets called when SDMA descriptors becomes available and Tx
   * queue's wait structure was previously added to sdma engine's dmawait list.
   */
  static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
  {
  	struct hfi1_ipoib_txq *txq =
  		container_of(wait, struct hfi1_ipoib_txq, wait);
  
  	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
  		iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
  }
  
  static void hfi1_ipoib_flush_txq(struct work_struct *work)
  {
  	struct iowait_work *ioww =
  		container_of(work, struct iowait_work, iowork);
  	struct iowait *wait = iowait_ioww_to_iow(ioww);
  	struct hfi1_ipoib_txq *txq =
  		container_of(wait, struct hfi1_ipoib_txq, wait);
  	struct net_device *dev = txq->priv->netdev;
  
  	if (likely(dev->reg_state == NETREG_REGISTERED) &&
  	    likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
  		if (atomic_xchg(&txq->no_desc, 0))
  			hfi1_ipoib_wake_txq(txq);
  }
  
  int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
  {
  	struct net_device *dev = priv->netdev;
  	char buf[HFI1_IPOIB_TXREQ_NAME_LEN];
  	unsigned long tx_ring_size;
  	int i;
  
  	/*
  	 * Ring holds 1 less than tx_ring_size
  	 * Round up to next power of 2 in order to hold at least tx_queue_len
  	 */
  	tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1);
  
  	snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit);
  	priv->txreq_cache = kmem_cache_create(buf,
  					      sizeof(struct ipoib_txreq),
  					      0,
  					      0,
  					      NULL);
  	if (!priv->txreq_cache)
  		return -ENOMEM;
  
  	priv->tx_napis = kcalloc_node(dev->num_tx_queues,
  				      sizeof(struct napi_struct),
  				      GFP_KERNEL,
  				      priv->dd->node);
  	if (!priv->tx_napis)
  		goto free_txreq_cache;
  
  	priv->txqs = kcalloc_node(dev->num_tx_queues,
  				  sizeof(struct hfi1_ipoib_txq),
  				  GFP_KERNEL,
  				  priv->dd->node);
  	if (!priv->txqs)
  		goto free_tx_napis;
  
  	for (i = 0; i < dev->num_tx_queues; i++) {
  		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
  
  		iowait_init(&txq->wait,
  			    0,
  			    hfi1_ipoib_flush_txq,
  			    NULL,
  			    hfi1_ipoib_sdma_sleep,
  			    hfi1_ipoib_sdma_wakeup,
  			    NULL,
  			    NULL);
  		txq->priv = priv;
  		txq->sde = NULL;
  		INIT_LIST_HEAD(&txq->tx_list);
  		atomic64_set(&txq->complete_txreqs, 0);
  		atomic_set(&txq->stops, 0);
  		atomic_set(&txq->ring_full, 0);
  		atomic_set(&txq->no_desc, 0);
  		txq->q_idx = i;
  		txq->flow.tx_queue = 0xff;
  		txq->flow.sc5 = 0xff;
  		txq->pkts_sent = false;
  
  		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
  					     priv->dd->node);
  
  		txq->tx_ring.items =
  			kcalloc_node(tx_ring_size,
  				     sizeof(struct ipoib_txreq *),
  				     GFP_KERNEL, priv->dd->node);
  		if (!txq->tx_ring.items)
  			goto free_txqs;
  
  		spin_lock_init(&txq->tx_ring.producer_lock);
  		spin_lock_init(&txq->tx_ring.consumer_lock);
  		txq->tx_ring.max_items = tx_ring_size;
  
  		txq->napi = &priv->tx_napis[i];
  		netif_tx_napi_add(dev, txq->napi,
  				  hfi1_ipoib_process_tx_ring,
  				  NAPI_POLL_WEIGHT);
  	}
  
  	return 0;
  
  free_txqs:
  	for (i--; i >= 0; i--) {
  		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
  
  		netif_napi_del(txq->napi);
  		kfree(txq->tx_ring.items);
  	}
  
  	kfree(priv->txqs);
  	priv->txqs = NULL;
  
  free_tx_napis:
  	kfree(priv->tx_napis);
  	priv->tx_napis = NULL;
  
  free_txreq_cache:
  	kmem_cache_destroy(priv->txreq_cache);
  	priv->txreq_cache = NULL;
  	return -ENOMEM;
  }
  
  static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
  {
  	struct sdma_txreq *txreq;
  	struct sdma_txreq *txreq_tmp;
  	atomic64_t *complete_txreqs = &txq->complete_txreqs;
  
  	list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
  		struct ipoib_txreq *tx =
  			container_of(txreq, struct ipoib_txreq, txreq);
  
  		list_del(&txreq->list);
  		sdma_txclean(txq->priv->dd, &tx->txreq);
  		dev_kfree_skb_any(tx->skb);
  		kmem_cache_free(txq->priv->txreq_cache, tx);
  		atomic64_inc(complete_txreqs);
  	}
  
  	if (hfi1_ipoib_used(txq))
  		dd_dev_warn(txq->priv->dd,
  			    "txq %d not empty found %llu requests\n",
  			    txq->q_idx,
  			    hfi1_ipoib_txreqs(txq->sent_txreqs,
  					      atomic64_read(complete_txreqs)));
  }
  
  void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
  {
  	int i;
  
  	for (i = 0; i < priv->netdev->num_tx_queues; i++) {
  		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
  
  		iowait_cancel_work(&txq->wait);
  		iowait_sdma_drain(&txq->wait);
  		hfi1_ipoib_drain_tx_list(txq);
  		netif_napi_del(txq->napi);
  		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
  		kfree(txq->tx_ring.items);
  	}
  
  	kfree(priv->txqs);
  	priv->txqs = NULL;
  
  	kfree(priv->tx_napis);
  	priv->tx_napis = NULL;
  
  	kmem_cache_destroy(priv->txreq_cache);
  	priv->txreq_cache = NULL;
  }
  
  void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
  	int i;
  
  	for (i = 0; i < dev->num_tx_queues; i++) {
  		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
  
  		napi_enable(txq->napi);
  	}
  }
  
  void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
  {
  	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
  	int i;
  
  	for (i = 0; i < dev->num_tx_queues; i++) {
  		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
  
  		napi_disable(txq->napi);
  		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
  	}
  }
  