text: // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
  /*
   * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
   * Copyright 2016-2019 NXP
   *
   */
  
  #include <asm/cacheflush.h>
  #include <linux/io.h>
  #include <linux/slab.h>
  #include <linux/spinlock.h>
  #include <soc/fsl/dpaa2-global.h>
  
  #include "qbman-portal.h"
  
  /* All QBMan command and result structures use this "valid bit" encoding */
  #define QB_VALID_BIT ((u32)0x80)
  
  /* QBMan portal management command codes */
  #define QBMAN_MC_ACQUIRE       0x30
  #define QBMAN_WQCHAN_CONFIGURE 0x46
  
  /* CINH register offsets */
  #define QBMAN_CINH_SWP_EQCR_PI      0x800
  #define QBMAN_CINH_SWP_EQCR_CI	    0x840
  #define QBMAN_CINH_SWP_EQAR    0x8c0
  #define QBMAN_CINH_SWP_CR_RT        0x900
  #define QBMAN_CINH_SWP_VDQCR_RT     0x940
  #define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
  #define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
  #define QBMAN_CINH_SWP_DQPI    0xa00
  #define QBMAN_CINH_SWP_DCAP    0xac0
  #define QBMAN_CINH_SWP_SDQCR   0xb00
  #define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
  #define QBMAN_CINH_SWP_RCR_PI       0xc00
  #define QBMAN_CINH_SWP_RAR     0xcc0
  #define QBMAN_CINH_SWP_ISR     0xe00
  #define QBMAN_CINH_SWP_IER     0xe40
  #define QBMAN_CINH_SWP_ISDR    0xe80
  #define QBMAN_CINH_SWP_IIR     0xec0
  
  /* CENA register offsets */
  #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6))
  #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6))
  #define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((u32)(n) << 6))
  #define QBMAN_CENA_SWP_CR      0x600
  #define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((u32)(vb) >> 1))
  #define QBMAN_CENA_SWP_VDQCR   0x780
  #define QBMAN_CENA_SWP_EQCR_CI 0x840
  #define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
  
  /* CENA register offsets in memory-backed mode */
  #define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((u32)(n) << 6))
  #define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((u32)(n) << 6))
  #define QBMAN_CENA_SWP_CR_MEM       0x1600
  #define QBMAN_CENA_SWP_RR_MEM       0x1680
  #define QBMAN_CENA_SWP_VDQCR_MEM    0x1780
  
  /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
  #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6)
  
  /* Define token used to determine if response written to memory is valid */
  #define QMAN_DQ_TOKEN_VALID 1
  
  /* SDQCR attribute codes */
  #define QB_SDQCR_FC_SHIFT   29
  #define QB_SDQCR_FC_MASK    0x1
  #define QB_SDQCR_DCT_SHIFT  24
  #define QB_SDQCR_DCT_MASK   0x3
  #define QB_SDQCR_TOK_SHIFT  16
  #define QB_SDQCR_TOK_MASK   0xff
  #define QB_SDQCR_SRC_SHIFT  0
  #define QB_SDQCR_SRC_MASK   0xffff
  
  /* opaque token for static dequeues */
  #define QMAN_SDQCR_TOKEN    0xbb
  
  #define QBMAN_EQCR_DCA_IDXMASK          0x0f
  #define QBMAN_ENQUEUE_FLAG_DCA          (1ULL << 31)
  
  #define EQ_DESC_SIZE_WITHOUT_FD 29
  #define EQ_DESC_SIZE_FD_START 32
  
  enum qbman_sdqcr_dct {
  	qbman_sdqcr_dct_null = 0,
  	qbman_sdqcr_dct_prio_ics,
  	qbman_sdqcr_dct_active_ics,
  	qbman_sdqcr_dct_active
  };
  
  enum qbman_sdqcr_fc {
  	qbman_sdqcr_fc_one = 0,
  	qbman_sdqcr_fc_up_to_3 = 1
  };
  
  /* Internal Function declaration */
  static int qbman_swp_enqueue_direct(struct qbman_swp *s,
  				    const struct qbman_eq_desc *d,
  				    const struct dpaa2_fd *fd);
  static int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
  				      const struct qbman_eq_desc *d,
  				      const struct dpaa2_fd *fd);
  static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
  					     const struct qbman_eq_desc *d,
  					     const struct dpaa2_fd *fd,
  					     uint32_t *flags,
  					     int num_frames);
  static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
  					       const struct qbman_eq_desc *d,
  					       const struct dpaa2_fd *fd,
  					       uint32_t *flags,
  					       int num_frames);
  static int
  qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
  				       const struct qbman_eq_desc *d,
  				       const struct dpaa2_fd *fd,
  				       int num_frames);
  static
  int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
  					     const struct qbman_eq_desc *d,
  					     const struct dpaa2_fd *fd,
  					     int num_frames);
  static int qbman_swp_pull_direct(struct qbman_swp *s,
  				 struct qbman_pull_desc *d);
  static int qbman_swp_pull_mem_back(struct qbman_swp *s,
  				   struct qbman_pull_desc *d);
  
  const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
  const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
  
  static int qbman_swp_release_direct(struct qbman_swp *s,
  				    const struct qbman_release_desc *d,
  				    const u64 *buffers,
  				    unsigned int num_buffers);
  static int qbman_swp_release_mem_back(struct qbman_swp *s,
  				      const struct qbman_release_desc *d,
  				      const u64 *buffers,
  				      unsigned int num_buffers);
  
  /* Function pointers */
  int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s,
  			     const struct qbman_eq_desc *d,
  			     const struct dpaa2_fd *fd)
  	= qbman_swp_enqueue_direct;
  
  int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
  				      const struct qbman_eq_desc *d,
  				      const struct dpaa2_fd *fd,
  				      uint32_t *flags,
  					     int num_frames)
  	= qbman_swp_enqueue_multiple_direct;
  
  int
  (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
  				       const struct qbman_eq_desc *d,
  				       const struct dpaa2_fd *fd,
  				       int num_frames)
  	= qbman_swp_enqueue_multiple_desc_direct;
  
  int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d)
  			= qbman_swp_pull_direct;
  
  const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
  			= qbman_swp_dqrr_next_direct;
  
  int (*qbman_swp_release_ptr)(struct qbman_swp *s,
  			     const struct qbman_release_desc *d,
  			     const u64 *buffers,
  			     unsigned int num_buffers)
  			= qbman_swp_release_direct;
  
  /* Portal Access */
  
  static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
  {
  	return readl_relaxed(p->addr_cinh + offset);
  }
  
  static inline void qbman_write_register(struct qbman_swp *p, u32 offset,
  					u32 value)
  {
  	writel_relaxed(value, p->addr_cinh + offset);
  }
  
  static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset)
  {
  	return p->addr_cena + offset;
  }
  
  #define QBMAN_CINH_SWP_CFG   0xd00
  
  #define SWP_CFG_DQRR_MF_SHIFT 20
  #define SWP_CFG_EST_SHIFT     16
  #define SWP_CFG_CPBS_SHIFT    15
  #define SWP_CFG_WN_SHIFT      14
  #define SWP_CFG_RPM_SHIFT     12
  #define SWP_CFG_DCM_SHIFT     10
  #define SWP_CFG_EPM_SHIFT     8
  #define SWP_CFG_VPM_SHIFT     7
  #define SWP_CFG_CPM_SHIFT     6
  #define SWP_CFG_SD_SHIFT      5
  #define SWP_CFG_SP_SHIFT      4
  #define SWP_CFG_SE_SHIFT      3
  #define SWP_CFG_DP_SHIFT      2
  #define SWP_CFG_DE_SHIFT      1
  #define SWP_CFG_EP_SHIFT      0
  
  static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn,	u8 est, u8 rpm, u8 dcm,
  				    u8 epm, int sd, int sp, int se,
  				    int dp, int de, int ep)
  {
  	return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
  		est << SWP_CFG_EST_SHIFT |
  		wn << SWP_CFG_WN_SHIFT |
  		rpm << SWP_CFG_RPM_SHIFT |
  		dcm << SWP_CFG_DCM_SHIFT |
  		epm << SWP_CFG_EPM_SHIFT |
  		sd << SWP_CFG_SD_SHIFT |
  		sp << SWP_CFG_SP_SHIFT |
  		se << SWP_CFG_SE_SHIFT |
  		dp << SWP_CFG_DP_SHIFT |
  		de << SWP_CFG_DE_SHIFT |
  		ep << SWP_CFG_EP_SHIFT);
  }
  
  #define QMAN_RT_MODE	   0x00000100
  
  static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
  {
  	/* 'first' is included, 'last' is excluded */
  	if (first <= last)
  		return last - first;
  	else
  		return (2 * ringsize) - (first - last);
  }
  
  /**
   * qbman_swp_init() - Create a functional object representing the given
   *                    QBMan portal descriptor.
   * @d: the given qbman swp descriptor
   *
   * Return qbman_swp portal for success, NULL if the object cannot
   * be created.
   */
  struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
  {
  	struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL);
  	u32 reg;
  	u32 mask_size;
  	u32 eqcr_pi;
  
  	if (!p)
  		return NULL;
  
  	spin_lock_init(&p->access_spinlock);
  
  	p->desc = d;
  	p->mc.valid_bit = QB_VALID_BIT;
  	p->sdq = 0;
  	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
  	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
  	p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
  	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
  		p->mr.valid_bit = QB_VALID_BIT;
  
  	atomic_set(&p->vdq.available, 1);
  	p->vdq.valid_bit = QB_VALID_BIT;
  	p->dqrr.next_idx = 0;
  	p->dqrr.valid_bit = QB_VALID_BIT;
  
  	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
  		p->dqrr.dqrr_size = 4;
  		p->dqrr.reset_bug = 1;
  	} else {
  		p->dqrr.dqrr_size = 8;
  		p->dqrr.reset_bug = 0;
  	}
  
  	p->addr_cena = d->cena_bar;
  	p->addr_cinh = d->cinh_bar;
  
  	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
  
  		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
  			1, /* Writes Non-cacheable */
  			0, /* EQCR_CI stashing threshold */
  			3, /* RPM: RCR in array mode */
  			2, /* DCM: Discrete consumption ack */
  			2, /* EPM: EQCR in ring mode */
  			1, /* mem stashing drop enable enable */
  			1, /* mem stashing priority enable */
  			1, /* mem stashing enable */
  			1, /* dequeue stashing priority enable */
  			0, /* dequeue stashing enable enable */
  			0); /* EQCR_CI stashing priority enable */
  	} else {
  		memset(p->addr_cena, 0, 64 * 1024);
  		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
  			1, /* Writes Non-cacheable */
  			1, /* EQCR_CI stashing threshold */
  			3, /* RPM: RCR in array mode */
  			2, /* DCM: Discrete consumption ack */
  			0, /* EPM: EQCR in ring mode */
  			1, /* mem stashing drop enable */
  			1, /* mem stashing priority enable */
  			1, /* mem stashing enable */
  			1, /* dequeue stashing priority enable */
  			0, /* dequeue stashing enable */
  			0); /* EQCR_CI stashing priority enable */
  		reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
  		       1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
  		       1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */
  	}
  
  	qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
  	reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
  	if (!reg) {
  		pr_err("qbman: the portal is not enabled!\n");
  		kfree(p);
  		return NULL;
  	}
  
  	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
  		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
  		qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
  	}
  	/*
  	 * SDQCR needs to be initialized to 0 when no channels are
  	 * being dequeued from or else the QMan HW will indicate an
  	 * error.  The values that were calculated above will be
  	 * applied when dequeues from a specific channel are enabled.
  	 */
  	qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
  
  	p->eqcr.pi_ring_size = 8;
  	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
  		p->eqcr.pi_ring_size = 32;
  		qbman_swp_enqueue_ptr =
  			qbman_swp_enqueue_mem_back;
  		qbman_swp_enqueue_multiple_ptr =
  			qbman_swp_enqueue_multiple_mem_back;
  		qbman_swp_enqueue_multiple_desc_ptr =
  			qbman_swp_enqueue_multiple_desc_mem_back;
  		qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
  		qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
  		qbman_swp_release_ptr = qbman_swp_release_mem_back;
  	}
  
  	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
  		p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
  	eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI);
  	p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
  	p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
  	p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI)
  			& p->eqcr.pi_ci_mask;
  	p->eqcr.available = p->eqcr.pi_ring_size;
  
  	return p;
  }
  
  /**
   * qbman_swp_finish() - Create and destroy a functional object representing
   *                      the given QBMan portal descriptor.
   * @p: the qbman_swp object to be destroyed
   */
  void qbman_swp_finish(struct qbman_swp *p)
  {
  	kfree(p);
  }
  
  /**
   * qbman_swp_interrupt_read_status()
   * @p: the given software portal
   *
   * Return the value in the SWP_ISR register.
   */
  u32 qbman_swp_interrupt_read_status(struct qbman_swp *p)
  {
  	return qbman_read_register(p, QBMAN_CINH_SWP_ISR);
  }
  
  /**
   * qbman_swp_interrupt_clear_status()
   * @p: the given software portal
   * @mask: The mask to clear in SWP_ISR register
   */
  void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask)
  {
  	qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask);
  }
  
  /**
   * qbman_swp_interrupt_get_trigger() - read interrupt enable register
   * @p: the given software portal
   *
   * Return the value in the SWP_IER register.
   */
  u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
  {
  	return qbman_read_register(p, QBMAN_CINH_SWP_IER);
  }
  
  /**
   * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
   * @p: the given software portal
   * @mask: The mask of bits to enable in SWP_IER
   */
  void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask)
  {
  	qbman_write_register(p, QBMAN_CINH_SWP_IER, mask);
  }
  
  /**
   * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
   * @p: the given software portal object
   *
   * Return the value in the SWP_IIR register.
   */
  int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
  {
  	return qbman_read_register(p, QBMAN_CINH_SWP_IIR);
  }
  
  /**
   * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
   * @p: the given software portal object
   * @mask: The mask to set in SWP_IIR register
   */
  void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
  {
  	qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
  }
  
  /*
   * Different management commands all use this common base layer of code to issue
   * commands and poll for results.
   */
  
  /*
   * Returns a pointer to where the caller should fill in their management command
   * (caller should ignore the verb byte)
   */
  void *qbman_swp_mc_start(struct qbman_swp *p)
  {
  	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
  		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR);
  	else
  		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM);
  }
  
  /*
   * Commits merges in the caller-supplied command verb (which should not include
   * the valid-bit) and submits the command to hardware
   */
  void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb)
  {
  	u8 *v = cmd;
  
  	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
  		dma_wmb();
  		*v = cmd_verb | p->mc.valid_bit;
  	} else {
  		*v = cmd_verb | p->mc.valid_bit;
  		dma_wmb();
  		qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
  	}
  }
  
  /*
   * Checks for a completed response (returns non-NULL if only if the response
   * is complete).
   */
  void *qbman_swp_mc_result(struct qbman_swp *p)
  {
  	u32 *ret, verb;
  
  	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
  		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
  		/* Remove the valid-bit - command completed if the rest
  		 * is non-zero.
  		 */
  		verb = ret[0] & ~QB_VALID_BIT;
  		if (!verb)
  			return NULL;
  		p->mc.valid_bit ^= QB_VALID_BIT;
  	} else {
  		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM);
  		/* Command completed if the valid bit is toggled */
  		if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
  			return NULL;
  		/* Command completed if the rest is non-zero */
  		verb = ret[0] & ~QB_VALID_BIT;
  		if (!verb)
  			return NULL;
  		p->mr.valid_bit ^= QB_VALID_BIT;
  	}
  
  	return ret;
  }
  
  #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
  enum qb_enqueue_commands {
  	enqueue_empty = 0,
  	enqueue_response_always = 1,
  	enqueue_rejects_to_fq = 2
  };
  
  #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
  #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
  #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
  #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
  
  /**
   * qbman_eq_desc_clear() - Clear the contents of a descriptor to
   *                         default/starting state.
   */
  void qbman_eq_desc_clear(struct qbman_eq_desc *d)
  {
  	memset(d, 0, sizeof(*d));
  }
  
  /**
   * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
   * @d:                the enqueue descriptor.
   * @response_success: 1 = enqueue with response always; 0 = enqueue with
   *                    rejections returned on a FQ.
   */
  void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
  {
  	d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
  	if (respond_success)
  		d->verb |= enqueue_response_always;
  	else
  		d->verb |= enqueue_rejects_to_fq;
  }
  
  /*
   * Exactly one of the following descriptor "targets" should be set. (Calling any
   * one of these will replace the effect of any prior call to one of these.)
   *   -enqueue to a frame queue
   *   -enqueue to a queuing destination
   */
  
  /**
   * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
   * @d:    the enqueue descriptor
   * @fqid: the id of the frame queue to be enqueued
   */
  void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
  {
  	d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
  	d->tgtid = cpu_to_le32(fqid);
  }
  
  /**
   * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
   * @d:       the enqueue descriptor
   * @qdid:    the id of the queuing destination to be enqueued
   * @qd_bin:  the queuing destination bin
   * @qd_prio: the queuing destination priority
   */
  void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
  			  u32 qd_bin, u32 qd_prio)
  {
  	d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
  	d->tgtid = cpu_to_le32(qdid);
  	d->qdbin = cpu_to_le16(qd_bin);
  	d->qpri = qd_prio;
  }
  
  #define EQAR_IDX(eqar)     ((eqar) & 0x7)
  #define EQAR_VB(eqar)      ((eqar) & 0x80)
  #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
  
  static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
  						   u8 idx)
  {
  	if (idx < 16)
  		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
  				     QMAN_RT_MODE);
  	else
  		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT2 +
  				     (idx - 16) * 4,
  				     QMAN_RT_MODE);
  }
  
  #define QB_RT_BIT ((u32)0x100)
  /**
   * qbman_swp_enqueue_direct() - Issue an enqueue command
   * @s:  the software portal used for enqueue
   * @d:  the enqueue descriptor
   * @fd: the frame descriptor to be enqueued
   *
   * Please note that 'fd' should only be NULL if the "action" of the
   * descriptor is "orp_hole" or "orp_nesn".
   *
   * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
   */
  static
  int qbman_swp_enqueue_direct(struct qbman_swp *s,
  			     const struct qbman_eq_desc *d,
  			     const struct dpaa2_fd *fd)
  {
  	int flags = 0;
  	int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1);
  
  	if (ret >= 0)
  		ret = 0;
  	else
  		ret = -EBUSY;
  	return  ret;
  }
  
  /**
   * qbman_swp_enqueue_mem_back() - Issue an enqueue command
   * @s:  the software portal used for enqueue
   * @d:  the enqueue descriptor
   * @fd: the frame descriptor to be enqueued
   *
   * Please note that 'fd' should only be NULL if the "action" of the
   * descriptor is "orp_hole" or "orp_nesn".
   *
   * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
   */
  static
  int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
  			       const struct qbman_eq_desc *d,
  			       const struct dpaa2_fd *fd)
  {
  	int flags = 0;
  	int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1);
  
  	if (ret >= 0)
  		ret = 0;
  	else
  		ret = -EBUSY;
  	return  ret;
  }
  
  /**
   * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
   * using one enqueue descriptor
   * @s:  the software portal used for enqueue
   * @d:  the enqueue descriptor
   * @fd: table pointer of frame descriptor table to be enqueued
   * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
   * @num_frames: number of fd to be enqueued
   *
   * Return the number of fd enqueued, or a negative error number.
   */
  static
  int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
  				      const struct qbman_eq_desc *d,
  				      const struct dpaa2_fd *fd,
  				      uint32_t *flags,
  				      int num_frames)
  {
  	uint32_t *p = NULL;
  	const uint32_t *cl = (uint32_t *)d;
  	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
  	int i, num_enqueued = 0;
  	uint64_t addr_cena;
  
  	spin_lock(&s->access_spinlock);
  	half_mask = (s->eqcr.pi_ci_mask>>1);
  	full_mask = s->eqcr.pi_ci_mask;
  
  	if (!s->eqcr.available) {
  		eqcr_ci = s->eqcr.ci;
  		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
  		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
  
  		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
  					eqcr_ci, s->eqcr.ci);
  		if (!s->eqcr.available) {
  			spin_unlock(&s->access_spinlock);
  			return 0;
  		}
  	}
  
  	eqcr_pi = s->eqcr.pi;
  	num_enqueued = (s->eqcr.available < num_frames) ?
  			s->eqcr.available : num_frames;
  	s->eqcr.available -= num_enqueued;
  	/* Fill in the EQCR ring */
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		/* Skip copying the verb */
  		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
  		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
  		       &fd[i], sizeof(*fd));
  		eqcr_pi++;
  	}
  
  	dma_wmb();
  
  	/* Set the verb byte, have to substitute in the valid-bit */
  	eqcr_pi = s->eqcr.pi;
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		p[0] = cl[0] | s->eqcr.pi_vb;
  		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
  			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
  
  			d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
  				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
  		}
  		eqcr_pi++;
  		if (!(eqcr_pi & half_mask))
  			s->eqcr.pi_vb ^= QB_VALID_BIT;
  	}
  
  	/* Flush all the cacheline without load/store in between */
  	eqcr_pi = s->eqcr.pi;
  	addr_cena = (size_t)s->addr_cena;
  	for (i = 0; i < num_enqueued; i++)
  		eqcr_pi++;
  	s->eqcr.pi = eqcr_pi & full_mask;
  	spin_unlock(&s->access_spinlock);
  
  	return num_enqueued;
  }
  
  /**
   * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
   * using one enqueue descriptor
   * @s:  the software portal used for enqueue
   * @d:  the enqueue descriptor
   * @fd: table pointer of frame descriptor table to be enqueued
   * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
   * @num_frames: number of fd to be enqueued
   *
   * Return the number of fd enqueued, or a negative error number.
   */
  static
  int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
  					const struct qbman_eq_desc *d,
  					const struct dpaa2_fd *fd,
  					uint32_t *flags,
  					int num_frames)
  {
  	uint32_t *p = NULL;
  	const uint32_t *cl = (uint32_t *)(d);
  	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
  	int i, num_enqueued = 0;
  	unsigned long irq_flags;
  
  	spin_lock(&s->access_spinlock);
  	local_irq_save(irq_flags);
  
  	half_mask = (s->eqcr.pi_ci_mask>>1);
  	full_mask = s->eqcr.pi_ci_mask;
  	if (!s->eqcr.available) {
  		eqcr_ci = s->eqcr.ci;
  		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
  		s->eqcr.ci = __raw_readl(p) & full_mask;
  		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
  					eqcr_ci, s->eqcr.ci);
  		if (!s->eqcr.available) {
  			local_irq_restore(irq_flags);
  			spin_unlock(&s->access_spinlock);
  			return 0;
  		}
  	}
  
  	eqcr_pi = s->eqcr.pi;
  	num_enqueued = (s->eqcr.available < num_frames) ?
  			s->eqcr.available : num_frames;
  	s->eqcr.available -= num_enqueued;
  	/* Fill in the EQCR ring */
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		/* Skip copying the verb */
  		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
  		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
  		       &fd[i], sizeof(*fd));
  		eqcr_pi++;
  	}
  
  	/* Set the verb byte, have to substitute in the valid-bit */
  	eqcr_pi = s->eqcr.pi;
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		p[0] = cl[0] | s->eqcr.pi_vb;
  		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
  			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
  
  			d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
  				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
  		}
  		eqcr_pi++;
  		if (!(eqcr_pi & half_mask))
  			s->eqcr.pi_vb ^= QB_VALID_BIT;
  	}
  	s->eqcr.pi = eqcr_pi & full_mask;
  
  	dma_wmb();
  	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
  				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
  	local_irq_restore(irq_flags);
  	spin_unlock(&s->access_spinlock);
  
  	return num_enqueued;
  }
  
  /**
   * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
   * using multiple enqueue descriptor
   * @s:  the software portal used for enqueue
   * @d:  table of minimal enqueue descriptor
   * @fd: table pointer of frame descriptor table to be enqueued
   * @num_frames: number of fd to be enqueued
   *
   * Return the number of fd enqueued, or a negative error number.
   */
  static
  int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
  					   const struct qbman_eq_desc *d,
  					   const struct dpaa2_fd *fd,
  					   int num_frames)
  {
  	uint32_t *p;
  	const uint32_t *cl;
  	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
  	int i, num_enqueued = 0;
  	uint64_t addr_cena;
  
  	half_mask = (s->eqcr.pi_ci_mask>>1);
  	full_mask = s->eqcr.pi_ci_mask;
  	if (!s->eqcr.available) {
  		eqcr_ci = s->eqcr.ci;
  		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
  		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
  		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
  					eqcr_ci, s->eqcr.ci);
  		if (!s->eqcr.available)
  			return 0;
  	}
  
  	eqcr_pi = s->eqcr.pi;
  	num_enqueued = (s->eqcr.available < num_frames) ?
  			s->eqcr.available : num_frames;
  	s->eqcr.available -= num_enqueued;
  	/* Fill in the EQCR ring */
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		cl = (uint32_t *)(&d[i]);
  		/* Skip copying the verb */
  		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
  		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
  		       &fd[i], sizeof(*fd));
  		eqcr_pi++;
  	}
  
  	dma_wmb();
  
  	/* Set the verb byte, have to substitute in the valid-bit */
  	eqcr_pi = s->eqcr.pi;
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		cl = (uint32_t *)(&d[i]);
  		p[0] = cl[0] | s->eqcr.pi_vb;
  		eqcr_pi++;
  		if (!(eqcr_pi & half_mask))
  			s->eqcr.pi_vb ^= QB_VALID_BIT;
  	}
  
  	/* Flush all the cacheline without load/store in between */
  	eqcr_pi = s->eqcr.pi;
  	addr_cena = (uint64_t)s->addr_cena;
  	for (i = 0; i < num_enqueued; i++)
  		eqcr_pi++;
  	s->eqcr.pi = eqcr_pi & full_mask;
  
  	return num_enqueued;
  }
  
  /**
   * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
   * using multiple enqueue descriptor
   * @s:  the software portal used for enqueue
   * @d:  table of minimal enqueue descriptor
   * @fd: table pointer of frame descriptor table to be enqueued
   * @num_frames: number of fd to be enqueued
   *
   * Return the number of fd enqueued, or a negative error number.
   */
  static
  int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
  					   const struct qbman_eq_desc *d,
  					   const struct dpaa2_fd *fd,
  					   int num_frames)
  {
  	uint32_t *p;
  	const uint32_t *cl;
  	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
  	int i, num_enqueued = 0;
  
  	half_mask = (s->eqcr.pi_ci_mask>>1);
  	full_mask = s->eqcr.pi_ci_mask;
  	if (!s->eqcr.available) {
  		eqcr_ci = s->eqcr.ci;
  		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
  		s->eqcr.ci = __raw_readl(p) & full_mask;
  		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
  					eqcr_ci, s->eqcr.ci);
  		if (!s->eqcr.available)
  			return 0;
  	}
  
  	eqcr_pi = s->eqcr.pi;
  	num_enqueued = (s->eqcr.available < num_frames) ?
  			s->eqcr.available : num_frames;
  	s->eqcr.available -= num_enqueued;
  	/* Fill in the EQCR ring */
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		cl = (uint32_t *)(&d[i]);
  		/* Skip copying the verb */
  		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
  		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
  		       &fd[i], sizeof(*fd));
  		eqcr_pi++;
  	}
  
  	/* Set the verb byte, have to substitute in the valid-bit */
  	eqcr_pi = s->eqcr.pi;
  	for (i = 0; i < num_enqueued; i++) {
  		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
  		cl = (uint32_t *)(&d[i]);
  		p[0] = cl[0] | s->eqcr.pi_vb;
  		eqcr_pi++;
  		if (!(eqcr_pi & half_mask))
  			s->eqcr.pi_vb ^= QB_VALID_BIT;
  	}
  
  	s->eqcr.pi = eqcr_pi & full_mask;
  
  	dma_wmb();
  	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
  				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
  
  	return num_enqueued;
  }
  
  /* Static (push) dequeue */
  
  /**
   * qbman_swp_push_get() - Get the push dequeue setup
   * @p:           the software portal object
   * @channel_idx: the channel index to query
   * @enabled:     returned boolean to show whether the push dequeue is enabled
   *               for the given channel
   */
  void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled)
  {
  	u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
  
  	WARN_ON(channel_idx > 15);
  	*enabled = src | (1 << channel_idx);
  }
  
  /**
   * qbman_swp_push_set() - Enable or disable push dequeue
   * @p:           the software portal object
   * @channel_idx: the channel index (0 to 15)
   * @enable:      enable or disable push dequeue
   */
  void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable)
  {
  	u16 dqsrc;
  
  	WARN_ON(channel_idx > 15);
  	if (enable)
  		s->sdq |= 1 << channel_idx;
  	else
  		s->sdq &= ~(1 << channel_idx);
  
  	/* Read make the complete src map.  If no channels are enabled
  	 * the SDQCR must be 0 or else QMan will assert errors
  	 */
  	dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
  	if (dqsrc != 0)
  		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq);
  	else
  		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0);
  }
  
  #define QB_VDQCR_VERB_DCT_SHIFT    0
  #define QB_VDQCR_VERB_DT_SHIFT     2
  #define QB_VDQCR_VERB_RLS_SHIFT    4
  #define QB_VDQCR_VERB_WAE_SHIFT    5
  
  enum qb_pull_dt_e {
  	qb_pull_dt_channel,
  	qb_pull_dt_workqueue,
  	qb_pull_dt_framequeue
  };
  
  /**
   * qbman_pull_desc_clear() - Clear the contents of a descriptor to
   *                           default/starting state
   * @d: the pull dequeue descriptor to be cleared
   */
  void qbman_pull_desc_clear(struct qbman_pull_desc *d)
  {
  	memset(d, 0, sizeof(*d));
  }
  
  /**
   * qbman_pull_desc_set_storage()- Set the pull dequeue storage
   * @d:            the pull dequeue descriptor to be set
   * @storage:      the pointer of the memory to store the dequeue result
   * @storage_phys: the physical address of the storage memory
   * @stash:        to indicate whether write allocate is enabled
   *
   * If not called, or if called with 'storage' as NULL, the result pull dequeues
   * will produce results to DQRR. If 'storage' is non-NULL, then results are
   * produced to the given memory location (using the DMA address which
   * the caller provides in 'storage_phys'), and 'stash' controls whether or not
   * those writes to main-memory express a cache-warming attribute.
   */
  void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
  				 struct dpaa2_dq *storage,
  				 dma_addr_t storage_phys,
  				 int stash)
  {
  	/* save the virtual address */
  	d->rsp_addr_virt = (u64)(uintptr_t)storage;
  
  	if (!storage) {
  		d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
  		return;
  	}
  	d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
  	if (stash)
  		d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
  	else
  		d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
  
  	d->rsp_addr = cpu_to_le64(storage_phys);
  }
  
  /**
   * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
   * @d:         the pull dequeue descriptor to be set
   * @numframes: number of frames to be set, must be between 1 and 16, inclusive
   */
  void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes)
  {
  	d->numf = numframes - 1;
  }
  
  /*
   * Exactly one of the following descriptor "actions" should be set. (Calling any
   * one of these will replace the effect of any prior call to one of these.)
   * - pull dequeue from the given frame queue (FQ)
   * - pull dequeue from any FQ in the given work queue (WQ)
   * - pull dequeue from any FQ in any WQ in the given channel
   */
  
  /**
   * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
   * @fqid: the frame queue index of the given FQ
   */
  void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid)
  {
  	d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
  	d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
  	d->dq_src = cpu_to_le32(fqid);
  }
  
  /**
   * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
   * @wqid: composed of channel id and wqid within the channel
   * @dct:  the dequeue command type
   */
  void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
  			    enum qbman_pull_type_e dct)
  {
  	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
  	d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
  	d->dq_src = cpu_to_le32(wqid);
  }
  
  /**
   * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
   *                                 dequeues
   * @chid: the channel id to be dequeued
   * @dct:  the dequeue command type
   */
  void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
  				 enum qbman_pull_type_e dct)
  {
  	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
  	d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
  	d->dq_src = cpu_to_le32(chid);
  }
  
  /**
   * qbman_swp_pull_direct() - Issue the pull dequeue command
   * @s: the software portal object
   * @d: the software portal descriptor which has been configured with
   *     the set of qbman_pull_desc_set_*() calls
   *
   * Return 0 for success, and -EBUSY if the software portal is not ready
   * to do pull dequeue.
   */
  static
  int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d)
  {
  	struct qbman_pull_desc *p;
  
  	if (!atomic_dec_and_test(&s->vdq.available)) {
  		atomic_inc(&s->vdq.available);
  		return -EBUSY;
  	}
  	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
  	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
  		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
  	else
  		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
  	p->numf = d->numf;
  	p->tok = QMAN_DQ_TOKEN_VALID;
  	p->dq_src = d->dq_src;
  	p->rsp_addr = d->rsp_addr;
  	p->rsp_addr_virt = d->rsp_addr_virt;
  	dma_wmb();
  	/* Set the verb byte, have to substitute in the valid-bit */
  	p->verb = d->verb | s->vdq.valid_bit;
  	s->vdq.valid_bit ^= QB_VALID_BIT;
  
  	return 0;
  }
  
  /**
   * qbman_swp_pull_mem_back() - Issue the pull dequeue command
   * @s: the software portal object
   * @d: the software portal descriptor which has been configured with
   *     the set of qbman_pull_desc_set_*() calls
   *
   * Return 0 for success, and -EBUSY if the software portal is not ready
   * to do pull dequeue.
   */
  static
  int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d)
  {
  	struct qbman_pull_desc *p;
  
  	if (!atomic_dec_and_test(&s->vdq.available)) {
  		atomic_inc(&s->vdq.available);
  		return -EBUSY;
  	}
  	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
  	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
  		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
  	else
  		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
  	p->numf = d->numf;
  	p->tok = QMAN_DQ_TOKEN_VALID;
  	p->dq_src = d->dq_src;
  	p->rsp_addr = d->rsp_addr;
  	p->rsp_addr_virt = d->rsp_addr_virt;
  
  	/* Set the verb byte, have to substitute in the valid-bit */
  	p->verb = d->verb | s->vdq.valid_bit;
  	s->vdq.valid_bit ^= QB_VALID_BIT;
  	dma_wmb();
  	qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
  
  	return 0;
  }
  
  #define QMAN_DQRR_PI_MASK   0xf
  
  /**
   * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
   * @s: the software portal object
   *
   * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
   * only once, so repeated calls can return a sequence of DQRR entries, without
   * requiring they be consumed immediately or in any particular order.
   */
  const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
  {
  	u32 verb;
  	u32 response_verb;
  	u32 flags;
  	struct dpaa2_dq *p;
  
  	/* Before using valid-bit to detect if something is there, we have to
  	 * handle the case of the DQRR reset bug...
  	 */
  	if (unlikely(s->dqrr.reset_bug)) {
  		/*
  		 * We pick up new entries by cache-inhibited producer index,
  		 * which means that a non-coherent mapping would require us to
  		 * invalidate and read *only* once that PI has indicated that
  		 * there's an entry here. The first trip around the DQRR ring
  		 * will be much less efficient than all subsequent trips around
  		 * it...
  		 */
  		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
  			QMAN_DQRR_PI_MASK;
  
  		/* there are new entries if pi != next_idx */
  		if (pi == s->dqrr.next_idx)
  			return NULL;
  
  		/*
  		 * if next_idx is/was the last ring index, and 'pi' is
  		 * different, we can disable the workaround as all the ring
  		 * entries have now been DMA'd to so valid-bit checking is
  		 * repaired. Note: this logic needs to be based on next_idx
  		 * (which increments one at a time), rather than on pi (which
  		 * can burst and wrap-around between our snapshots of it).
  		 */
  		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
  			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
  				 s->dqrr.next_idx, pi);
  			s->dqrr.reset_bug = 0;
  		}
  		prefetch(qbman_get_cmd(s,
  				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  	}
  
  	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
  	verb = p->dq.verb;
  
  	/*
  	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
  	 * in the DQRR reset bug workaround, we shouldn't need to skip these
  	 * check, because we've already determined that a new entry is available
  	 * and we've invalidated the cacheline before reading it, so the
  	 * valid-bit behaviour is repaired and should tell us what we already
  	 * knew from reading PI.
  	 */
  	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
  		prefetch(qbman_get_cmd(s,
  				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  		return NULL;
  	}
  	/*
  	 * There's something there. Move "next_idx" attention to the next ring
  	 * entry (and prefetch it) before returning what we found.
  	 */
  	s->dqrr.next_idx++;
  	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
  	if (!s->dqrr.next_idx)
  		s->dqrr.valid_bit ^= QB_VALID_BIT;
  
  	/*
  	 * If this is the final response to a volatile dequeue command
  	 * indicate that the vdq is available
  	 */
  	flags = p->dq.stat;
  	response_verb = verb & QBMAN_RESULT_MASK;
  	if ((response_verb == QBMAN_RESULT_DQ) &&
  	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
  	    (flags & DPAA2_DQ_STAT_EXPIRED))
  		atomic_inc(&s->vdq.available);
  
  	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  
  	return p;
  }
  
  /**
   * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
   * @s: the software portal object
   *
   * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
   * only once, so repeated calls can return a sequence of DQRR entries, without
   * requiring they be consumed immediately or in any particular order.
   */
  const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
  {
  	u32 verb;
  	u32 response_verb;
  	u32 flags;
  	struct dpaa2_dq *p;
  
  	/* Before using valid-bit to detect if something is there, we have to
  	 * handle the case of the DQRR reset bug...
  	 */
  	if (unlikely(s->dqrr.reset_bug)) {
  		/*
  		 * We pick up new entries by cache-inhibited producer index,
  		 * which means that a non-coherent mapping would require us to
  		 * invalidate and read *only* once that PI has indicated that
  		 * there's an entry here. The first trip around the DQRR ring
  		 * will be much less efficient than all subsequent trips around
  		 * it...
  		 */
  		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
  			QMAN_DQRR_PI_MASK;
  
  		/* there are new entries if pi != next_idx */
  		if (pi == s->dqrr.next_idx)
  			return NULL;
  
  		/*
  		 * if next_idx is/was the last ring index, and 'pi' is
  		 * different, we can disable the workaround as all the ring
  		 * entries have now been DMA'd to so valid-bit checking is
  		 * repaired. Note: this logic needs to be based on next_idx
  		 * (which increments one at a time), rather than on pi (which
  		 * can burst and wrap-around between our snapshots of it).
  		 */
  		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
  			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
  				 s->dqrr.next_idx, pi);
  			s->dqrr.reset_bug = 0;
  		}
  		prefetch(qbman_get_cmd(s,
  				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  	}
  
  	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
  	verb = p->dq.verb;
  
  	/*
  	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
  	 * in the DQRR reset bug workaround, we shouldn't need to skip these
  	 * check, because we've already determined that a new entry is available
  	 * and we've invalidated the cacheline before reading it, so the
  	 * valid-bit behaviour is repaired and should tell us what we already
  	 * knew from reading PI.
  	 */
  	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
  		prefetch(qbman_get_cmd(s,
  				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  		return NULL;
  	}
  	/*
  	 * There's something there. Move "next_idx" attention to the next ring
  	 * entry (and prefetch it) before returning what we found.
  	 */
  	s->dqrr.next_idx++;
  	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
  	if (!s->dqrr.next_idx)
  		s->dqrr.valid_bit ^= QB_VALID_BIT;
  
  	/*
  	 * If this is the final response to a volatile dequeue command
  	 * indicate that the vdq is available
  	 */
  	flags = p->dq.stat;
  	response_verb = verb & QBMAN_RESULT_MASK;
  	if ((response_verb == QBMAN_RESULT_DQ) &&
  	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
  	    (flags & DPAA2_DQ_STAT_EXPIRED))
  		atomic_inc(&s->vdq.available);
  
  	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
  
  	return p;
  }
  
  /**
   * qbman_swp_dqrr_consume() -  Consume DQRR entries previously returned from
   *                             qbman_swp_dqrr_next().
   * @s: the software portal object
   * @dq: the DQRR entry to be consumed
   */
  void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq)
  {
  	qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
  }
  
  /**
   * qbman_result_has_new_result() - Check and get the dequeue response from the
   *                                 dq storage memory set in pull dequeue command
   * @s: the software portal object
   * @dq: the dequeue result read from the memory
   *
   * Return 1 for getting a valid dequeue result, or 0 for not getting a valid
   * dequeue result.
   *
   * Only used for user-provided storage of dequeue results, not DQRR. For
   * efficiency purposes, the driver will perform any required endianness
   * conversion to ensure that the user's dequeue result storage is in host-endian
   * format. As such, once the user has called qbman_result_has_new_result() and
   * been returned a valid dequeue result, they should not call it again on
   * the same memory location (except of course if another dequeue command has
   * been executed to produce a new result to that location).
   */
  int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq)
  {
  	if (dq->dq.tok != QMAN_DQ_TOKEN_VALID)
  		return 0;
  
  	/*
  	 * Set token to be 0 so we will detect change back to 1
  	 * next time the looping is traversed. Const is cast away here
  	 * as we want users to treat the dequeue responses as read only.
  	 */
  	((struct dpaa2_dq *)dq)->dq.tok = 0;
  
  	/*
  	 * Determine whether VDQCR is available based on whether the
  	 * current result is sitting in the first storage location of
  	 * the busy command.
  	 */
  	if (s->vdq.storage == dq) {
  		s->vdq.storage = NULL;
  		atomic_inc(&s->vdq.available);
  	}
  
  	return 1;
  }
  
  /**
   * qbman_release_desc_clear() - Clear the contents of a descriptor to
   *                              default/starting state.
   */
  void qbman_release_desc_clear(struct qbman_release_desc *d)
  {
  	memset(d, 0, sizeof(*d));
  	d->verb = 1 << 5; /* Release Command Valid */
  }
  
  /**
   * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
   */
  void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid)
  {
  	d->bpid = cpu_to_le16(bpid);
  }
  
  /**
   * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
   * interrupt source should be asserted after the release command is completed.
   */
  void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
  {
  	if (enable)
  		d->verb |= 1 << 6;
  	else
  		d->verb &= ~(1 << 6);
  }
  
  #define RAR_IDX(rar)     ((rar) & 0x7)
  #define RAR_VB(rar)      ((rar) & 0x80)
  #define RAR_SUCCESS(rar) ((rar) & 0x100)
  
  /**
   * qbman_swp_release_direct() - Issue a buffer release command
   * @s:           the software portal object
   * @d:           the release descriptor
   * @buffers:     a pointer pointing to the buffer address to be released
   * @num_buffers: number of buffers to be released,  must be less than 8
   *
   * Return 0 for success, -EBUSY if the release command ring is not ready.
   */
  int qbman_swp_release_direct(struct qbman_swp *s,
  			     const struct qbman_release_desc *d,
  			     const u64 *buffers, unsigned int num_buffers)
  {
  	int i;
  	struct qbman_release_desc *p;
  	u32 rar;
  
  	if (!num_buffers || (num_buffers > 7))
  		return -EINVAL;
  
  	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
  	if (!RAR_SUCCESS(rar))
  		return -EBUSY;
  
  	/* Start the release command */
  	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
  
  	/* Copy the caller's buffer pointers to the command */
  	for (i = 0; i < num_buffers; i++)
  		p->buf[i] = cpu_to_le64(buffers[i]);
  	p->bpid = d->bpid;
  
  	/*
  	 * Set the verb byte, have to substitute in the valid-bit
  	 * and the number of buffers.
  	 */
  	dma_wmb();
  	p->verb = d->verb | RAR_VB(rar) | num_buffers;
  
  	return 0;
  }
  
  /**
   * qbman_swp_release_mem_back() - Issue a buffer release command
   * @s:           the software portal object
   * @d:           the release descriptor
   * @buffers:     a pointer pointing to the buffer address to be released
   * @num_buffers: number of buffers to be released,  must be less than 8
   *
   * Return 0 for success, -EBUSY if the release command ring is not ready.
   */
  int qbman_swp_release_mem_back(struct qbman_swp *s,
  			       const struct qbman_release_desc *d,
  			       const u64 *buffers, unsigned int num_buffers)
  {
  	int i;
  	struct qbman_release_desc *p;
  	u32 rar;
  
  	if (!num_buffers || (num_buffers > 7))
  		return -EINVAL;
  
  	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
  	if (!RAR_SUCCESS(rar))
  		return -EBUSY;
  
  	/* Start the release command */
  	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
  
  	/* Copy the caller's buffer pointers to the command */
  	for (i = 0; i < num_buffers; i++)
  		p->buf[i] = cpu_to_le64(buffers[i]);
  	p->bpid = d->bpid;
  
  	p->verb = d->verb | RAR_VB(rar) | num_buffers;
  	dma_wmb();
  	qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
  			     RAR_IDX(rar)  * 4, QMAN_RT_MODE);
  
  	return 0;
  }
  
  struct qbman_acquire_desc {
  	u8 verb;
  	u8 reserved;
  	__le16 bpid;
  	u8 num;
  	u8 reserved2[59];
  };
  
  struct qbman_acquire_rslt {
  	u8 verb;
  	u8 rslt;
  	__le16 reserved;
  	u8 num;
  	u8 reserved2[3];
  	__le64 buf[7];
  };
  
  /**
   * qbman_swp_acquire() - Issue a buffer acquire command
   * @s:           the software portal object
   * @bpid:        the buffer pool index
   * @buffers:     a pointer pointing to the acquired buffer addresses
   * @num_buffers: number of buffers to be acquired, must be less than 8
   *
   * Return 0 for success, or negative error code if the acquire command
   * fails.
   */
  int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
  		      unsigned int num_buffers)
  {
  	struct qbman_acquire_desc *p;
  	struct qbman_acquire_rslt *r;
  	int i;
  
  	if (!num_buffers || (num_buffers > 7))
  		return -EINVAL;
  
  	/* Start the management command */
  	p = qbman_swp_mc_start(s);
  
  	if (!p)
  		return -EBUSY;
  
  	/* Encode the caller-provided attributes */
  	p->bpid = cpu_to_le16(bpid);
  	p->num = num_buffers;
  
  	/* Complete the management command */
  	r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
  	if (unlikely(!r)) {
  		pr_err("qbman: acquire from BPID %d failed, no response\n",
  		       bpid);
  		return -EIO;
  	}
  
  	/* Decode the outcome */
  	WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE);
  
  	/* Determine success or failure */
  	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
  		pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n",
  		       bpid, r->rslt);
  		return -EIO;
  	}
  
  	WARN_ON(r->num > num_buffers);
  
  	/* Copy the acquired buffers to the caller's array */
  	for (i = 0; i < r->num; i++)
  		buffers[i] = le64_to_cpu(r->buf[i]);
  
  	return (int)r->num;
  }
  
  struct qbman_alt_fq_state_desc {
  	u8 verb;
  	u8 reserved[3];
  	__le32 fqid;
  	u8 reserved2[56];
  };
  
  struct qbman_alt_fq_state_rslt {
  	u8 verb;
  	u8 rslt;
  	u8 reserved[62];
  };
  
  #define ALT_FQ_FQID_MASK 0x00FFFFFF
  
  int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
  			   u8 alt_fq_verb)
  {
  	struct qbman_alt_fq_state_desc *p;
  	struct qbman_alt_fq_state_rslt *r;
  
  	/* Start the management command */
  	p = qbman_swp_mc_start(s);
  	if (!p)
  		return -EBUSY;
  
  	p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK);
  
  	/* Complete the management command */
  	r = qbman_swp_mc_complete(s, p, alt_fq_verb);
  	if (unlikely(!r)) {
  		pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
  		       alt_fq_verb);
  		return -EIO;
  	}
  
  	/* Decode the outcome */
  	WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb);
  
  	/* Determine success or failure */
  	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
  		pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n",
  		       fqid, r->verb, r->rslt);
  		return -EIO;
  	}
  
  	return 0;
  }
  
  struct qbman_cdan_ctrl_desc {
  	u8 verb;
  	u8 reserved;
  	__le16 ch;
  	u8 we;
  	u8 ctrl;
  	__le16 reserved2;
  	__le64 cdan_ctx;
  	u8 reserved3[48];
  
  };
  
  struct qbman_cdan_ctrl_rslt {
  	u8 verb;
  	u8 rslt;
  	__le16 ch;
  	u8 reserved[60];
  };
  
  int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid,
  		       u8 we_mask, u8 cdan_en,
  		       u64 ctx)
  {
  	struct qbman_cdan_ctrl_desc *p = NULL;
  	struct qbman_cdan_ctrl_rslt *r = NULL;
  
  	/* Start the management command */
  	p = qbman_swp_mc_start(s);
  	if (!p)
  		return -EBUSY;
  
  	/* Encode the caller-provided attributes */
  	p->ch = cpu_to_le16(channelid);
  	p->we = we_mask;
  	if (cdan_en)
  		p->ctrl = 1;
  	else
  		p->ctrl = 0;
  	p->cdan_ctx = cpu_to_le64(ctx);
  
  	/* Complete the management command */
  	r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
  	if (unlikely(!r)) {
  		pr_err("qbman: wqchan config failed, no response\n");
  		return -EIO;
  	}
  
  	WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE);
  
  	/* Determine success or failure */
  	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
  		pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n",
  		       channelid, r->rslt);
  		return -EIO;
  	}
  
  	return 0;
  }
  
  #define QBMAN_RESPONSE_VERB_MASK	0x7f
  #define QBMAN_FQ_QUERY_NP		0x45
  #define QBMAN_BP_QUERY			0x32
  
  struct qbman_fq_query_desc {
  	u8 verb;
  	u8 reserved[3];
  	__le32 fqid;
  	u8 reserved2[56];
  };
  
  int qbman_fq_query_state(struct qbman_swp *s, u32 fqid,
  			 struct qbman_fq_query_np_rslt *r)
  {
  	struct qbman_fq_query_desc *p;
  	void *resp;
  
  	p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s);
  	if (!p)
  		return -EBUSY;
  
  	/* FQID is a 24 bit value */
  	p->fqid = cpu_to_le32(fqid & 0x00FFFFFF);
  	resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP);
  	if (!resp) {
  		pr_err("qbman: Query FQID %d NP fields failed, no response\n",
  		       fqid);
  		return -EIO;
  	}
  	*r = *(struct qbman_fq_query_np_rslt *)resp;
  	/* Decode the outcome */
  	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP);
  
  	/* Determine success or failure */
  	if (r->rslt != QBMAN_MC_RSLT_OK) {
  		pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n",
  		       p->fqid, r->rslt);
  		return -EIO;
  	}
  
  	return 0;
  }
  
  u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r)
  {
  	return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF);
  }
  
  u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r)
  {
  	return le32_to_cpu(r->byte_cnt);
  }
  
  struct qbman_bp_query_desc {
  	u8 verb;
  	u8 reserved;
  	__le16 bpid;
  	u8 reserved2[60];
  };
  
  int qbman_bp_query(struct qbman_swp *s, u16 bpid,
  		   struct qbman_bp_query_rslt *r)
  {
  	struct qbman_bp_query_desc *p;
  	void *resp;
  
  	p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s);
  	if (!p)
  		return -EBUSY;
  
  	p->bpid = cpu_to_le16(bpid);
  	resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY);
  	if (!resp) {
  		pr_err("qbman: Query BPID %d fields failed, no response\n",
  		       bpid);
  		return -EIO;
  	}
  	*r = *(struct qbman_bp_query_rslt *)resp;
  	/* Decode the outcome */
  	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY);
  
  	/* Determine success or failure */
  	if (r->rslt != QBMAN_MC_RSLT_OK) {
  		pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n",
  		       bpid, r->rslt);
  		return -EIO;
  	}
  
  	return 0;
  }
  
  u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a)
  {
  	return le32_to_cpu(a->fill);
  }
  