text: /*
   * S3C24XX DMA handling
   *
   * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
   *
   * based on amba-pl08x.c
   *
   * Copyright (c) 2006 ARM Ltd.
   * Copyright (c) 2010 ST-Ericsson SA
   *
   * Author: Peter Pearse <peter.pearse@arm.com>
   * Author: Linus Walleij <linus.walleij@stericsson.com>
   *
   * This program is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License as published by the Free
   * Software Foundation; either version 2 of the License, or (at your option)
   * any later version.
   *
   * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals
   * that can be routed to any of the 4 to 8 hardware-channels.
   *
   * Therefore on these DMA controllers the number of channels
   * and the number of incoming DMA signals are two totally different things.
   * It is usually not possible to theoretically handle all physical signals,
   * so a multiplexing scheme with possible denial of use is necessary.
   *
   * Open items:
   * - bursts
   */
  
  #include <linux/platform_device.h>
  #include <linux/types.h>
  #include <linux/dmaengine.h>
  #include <linux/dma-mapping.h>
  #include <linux/interrupt.h>
  #include <linux/clk.h>
  #include <linux/module.h>
  #include <linux/slab.h>
  #include <linux/platform_data/dma-s3c24xx.h>
  
  #include "dmaengine.h"
  #include "virt-dma.h"
  
  #define MAX_DMA_CHANNELS	8
  
  #define S3C24XX_DISRC			0x00
  #define S3C24XX_DISRCC			0x04
  #define S3C24XX_DISRCC_INC_INCREMENT	0
  #define S3C24XX_DISRCC_INC_FIXED	BIT(0)
  #define S3C24XX_DISRCC_LOC_AHB		0
  #define S3C24XX_DISRCC_LOC_APB		BIT(1)
  
  #define S3C24XX_DIDST			0x08
  #define S3C24XX_DIDSTC			0x0c
  #define S3C24XX_DIDSTC_INC_INCREMENT	0
  #define S3C24XX_DIDSTC_INC_FIXED	BIT(0)
  #define S3C24XX_DIDSTC_LOC_AHB		0
  #define S3C24XX_DIDSTC_LOC_APB		BIT(1)
  #define S3C24XX_DIDSTC_INT_TC0		0
  #define S3C24XX_DIDSTC_INT_RELOAD	BIT(2)
  
  #define S3C24XX_DCON			0x10
  
  #define S3C24XX_DCON_TC_MASK		0xfffff
  #define S3C24XX_DCON_DSZ_BYTE		(0 << 20)
  #define S3C24XX_DCON_DSZ_HALFWORD	(1 << 20)
  #define S3C24XX_DCON_DSZ_WORD		(2 << 20)
  #define S3C24XX_DCON_DSZ_MASK		(3 << 20)
  #define S3C24XX_DCON_DSZ_SHIFT		20
  #define S3C24XX_DCON_AUTORELOAD		0
  #define S3C24XX_DCON_NORELOAD		BIT(22)
  #define S3C24XX_DCON_HWTRIG		BIT(23)
  #define S3C24XX_DCON_HWSRC_SHIFT	24
  #define S3C24XX_DCON_SERV_SINGLE	0
  #define S3C24XX_DCON_SERV_WHOLE		BIT(27)
  #define S3C24XX_DCON_TSZ_UNIT		0
  #define S3C24XX_DCON_TSZ_BURST4		BIT(28)
  #define S3C24XX_DCON_INT		BIT(29)
  #define S3C24XX_DCON_SYNC_PCLK		0
  #define S3C24XX_DCON_SYNC_HCLK		BIT(30)
  #define S3C24XX_DCON_DEMAND		0
  #define S3C24XX_DCON_HANDSHAKE		BIT(31)
  
  #define S3C24XX_DSTAT			0x14
  #define S3C24XX_DSTAT_STAT_BUSY		BIT(20)
  #define S3C24XX_DSTAT_CURRTC_MASK	0xfffff
  
  #define S3C24XX_DMASKTRIG		0x20
  #define S3C24XX_DMASKTRIG_SWTRIG	BIT(0)
  #define S3C24XX_DMASKTRIG_ON		BIT(1)
  #define S3C24XX_DMASKTRIG_STOP		BIT(2)
  
  #define S3C24XX_DMAREQSEL		0x24
  #define S3C24XX_DMAREQSEL_HW		BIT(0)
  
  /*
   * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel
   * for a DMA source. Instead only specific channels are valid.
   * All of these SoCs have 4 physical channels and the number of request
   * source bits is 3. Additionally we also need 1 bit to mark the channel
   * as valid.
   * Therefore we separate the chansel element of the channel data into 4
   * parts of 4 bits each, to hold the information if the channel is valid
   * and the hw request source to use.
   *
   * Example:
   * SDI is valid on channels 0, 2 and 3 - with varying hw request sources.
   * For it the chansel field would look like
   *
   * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1
   * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2
   * ((BIT(3) | 2) << 0 * 4)   // channel 0, with request source 2
   */
  #define S3C24XX_CHANSEL_WIDTH		4
  #define S3C24XX_CHANSEL_VALID		BIT(3)
  #define S3C24XX_CHANSEL_REQ_MASK	7
  
  /*
   * struct soc_data - vendor-specific config parameters for individual SoCs
   * @stride: spacing between the registers of each channel
   * @has_reqsel: does the controller use the newer requestselection mechanism
   * @has_clocks: are controllable dma-clocks present
   */
  struct soc_data {
  	int stride;
  	bool has_reqsel;
  	bool has_clocks;
  };
  
  /*
   * enum s3c24xx_dma_chan_state - holds the virtual channel states
   * @S3C24XX_DMA_CHAN_IDLE: the channel is idle
   * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport
   * channel and is running a transfer on it
   * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport
   * channel to become available (only pertains to memcpy channels)
   */
  enum s3c24xx_dma_chan_state {
  	S3C24XX_DMA_CHAN_IDLE,
  	S3C24XX_DMA_CHAN_RUNNING,
  	S3C24XX_DMA_CHAN_WAITING,
  };
  
  /*
   * struct s3c24xx_sg - structure containing data per sg
   * @src_addr: src address of sg
   * @dst_addr: dst address of sg
   * @len: transfer len in bytes
   * @node: node for txd's dsg_list
   */
  struct s3c24xx_sg {
  	dma_addr_t src_addr;
  	dma_addr_t dst_addr;
  	size_t len;
  	struct list_head node;
  };
  
  /*
   * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor
   * @vd: virtual DMA descriptor
   * @dsg_list: list of children sg's
   * @at: sg currently being transfered
   * @width: transfer width
   * @disrcc: value for source control register
   * @didstc: value for destination control register
   * @dcon: base value for dcon register
   * @cyclic: indicate cyclic transfer
   */
  struct s3c24xx_txd {
  	struct virt_dma_desc vd;
  	struct list_head dsg_list;
  	struct list_head *at;
  	u8 width;
  	u32 disrcc;
  	u32 didstc;
  	u32 dcon;
  	bool cyclic;
  };
  
  struct s3c24xx_dma_chan;
  
  /*
   * struct s3c24xx_dma_phy - holder for the physical channels
   * @id: physical index to this channel
   * @valid: does the channel have all required elements
   * @base: virtual memory base (remapped) for the this channel
   * @irq: interrupt for this channel
   * @clk: clock for this channel
   * @lock: a lock to use when altering an instance of this struct
   * @serving: virtual channel currently being served by this physicalchannel
   * @host: a pointer to the host (internal use)
   */
  struct s3c24xx_dma_phy {
  	unsigned int			id;
  	bool				valid;
  	void __iomem			*base;
  	int				irq;
  	struct clk			*clk;
  	spinlock_t			lock;
  	struct s3c24xx_dma_chan		*serving;
  	struct s3c24xx_dma_engine	*host;
  };
  
  /*
   * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel
   * @id: the id of the channel
   * @name: name of the channel
   * @vc: wrappped virtual channel
   * @phy: the physical channel utilized by this channel, if there is one
   * @runtime_addr: address for RX/TX according to the runtime config
   * @at: active transaction on this channel
   * @lock: a lock for this channel data
   * @host: a pointer to the host (internal use)
   * @state: whether the channel is idle, running etc
   * @slave: whether this channel is a device (slave) or for memcpy
   */
  struct s3c24xx_dma_chan {
  	int id;
  	const char *name;
  	struct virt_dma_chan vc;
  	struct s3c24xx_dma_phy *phy;
  	struct dma_slave_config cfg;
  	struct s3c24xx_txd *at;
  	struct s3c24xx_dma_engine *host;
  	enum s3c24xx_dma_chan_state state;
  	bool slave;
  };
  
  /*
   * struct s3c24xx_dma_engine - the local state holder for the S3C24XX
   * @pdev: the corresponding platform device
   * @pdata: platform data passed in from the platform/machine
   * @base: virtual memory base (remapped)
   * @slave: slave engine for this instance
   * @memcpy: memcpy engine for this instance
   * @phy_chans: array of data for the physical channels
   */
  struct s3c24xx_dma_engine {
  	struct platform_device			*pdev;
  	const struct s3c24xx_dma_platdata	*pdata;
  	struct soc_data				*sdata;
  	void __iomem				*base;
  	struct dma_device			slave;
  	struct dma_device			memcpy;
  	struct s3c24xx_dma_phy			*phy_chans;
  };
  
  /*
   * Physical channel handling
   */
  
  /*
   * Check whether a certain channel is busy or not.
   */
  static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
  {
  	unsigned int val = readl(phy->base + S3C24XX_DSTAT);
  	return val & S3C24XX_DSTAT_STAT_BUSY;
  }
  
  static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
  				  struct s3c24xx_dma_phy *phy)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  	int phyvalid;
  
  	/* every phy is valid for memcopy channels */
  	if (!s3cchan->slave)
  		return true;
  
  	/* On newer variants all phys can be used for all virtual channels */
  	if (s3cdma->sdata->has_reqsel)
  		return true;
  
  	phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
  	return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
  }
  
  /*
   * Allocate a physical channel for a virtual channel
   *
   * Try to locate a physical channel to be used for this transfer. If all
   * are taken return NULL and the requester will have to cope by using
   * some fallback PIO mode or retrying later.
   */
  static
  struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	struct s3c24xx_dma_phy *phy = NULL;
  	unsigned long flags;
  	int i;
  	int ret;
  
  	for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
  		phy = &s3cdma->phy_chans[i];
  
  		if (!phy->valid)
  			continue;
  
  		if (!s3c24xx_dma_phy_valid(s3cchan, phy))
  			continue;
  
  		spin_lock_irqsave(&phy->lock, flags);
  
  		if (!phy->serving) {
  			phy->serving = s3cchan;
  			spin_unlock_irqrestore(&phy->lock, flags);
  			break;
  		}
  
  		spin_unlock_irqrestore(&phy->lock, flags);
  	}
  
  	/* No physical channel available, cope with it */
  	if (i == s3cdma->pdata->num_phy_channels) {
  		dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
  		return NULL;
  	}
  
  	/* start the phy clock */
  	if (s3cdma->sdata->has_clocks) {
  		ret = clk_enable(phy->clk);
  		if (ret) {
  			dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
  				phy->id, ret);
  			phy->serving = NULL;
  			return NULL;
  		}
  	}
  
  	return phy;
  }
  
  /*
   * Mark the physical channel as free.
   *
   * This drops the link between the physical and virtual channel.
   */
  static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
  {
  	struct s3c24xx_dma_engine *s3cdma = phy->host;
  
  	if (s3cdma->sdata->has_clocks)
  		clk_disable(phy->clk);
  
  	phy->serving = NULL;
  }
  
  /*
   * Stops the channel by writing the stop bit.
   * This should not be used for an on-going transfer, but as a method of
   * shutting down a channel (eg, when it's no longer used) or terminating a
   * transfer.
   */
  static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
  {
  	writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
  }
  
  /*
   * Virtual channel handling
   */
  
  static inline
  struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
  {
  	return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
  }
  
  static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_phy *phy = s3cchan->phy;
  	struct s3c24xx_txd *txd = s3cchan->at;
  	u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
  
  	return tc * txd->width;
  }
  
  static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
  				  struct dma_slave_config *config)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	unsigned long flags;
  	int ret = 0;
  
  	/* Reject definitely invalid configurations */
  	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
  	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
  		return -EINVAL;
  
  	spin_lock_irqsave(&s3cchan->vc.lock, flags);
  
  	if (!s3cchan->slave) {
  		ret = -EINVAL;
  		goto out;
  	}
  
  	s3cchan->cfg = *config;
  
  out:
  	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  	return ret;
  }
  
  /*
   * Transfer handling
   */
  
  static inline
  struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
  {
  	return container_of(tx, struct s3c24xx_txd, vd.tx);
  }
  
  static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
  {
  	struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  
  	if (txd) {
  		INIT_LIST_HEAD(&txd->dsg_list);
  		txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
  	}
  
  	return txd;
  }
  
  static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
  {
  	struct s3c24xx_sg *dsg, *_dsg;
  
  	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
  		list_del(&dsg->node);
  		kfree(dsg);
  	}
  
  	kfree(txd);
  }
  
  static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
  				       struct s3c24xx_txd *txd)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	struct s3c24xx_dma_phy *phy = s3cchan->phy;
  	const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  	struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
  	u32 dcon = txd->dcon;
  	u32 val;
  
  	/* transfer-size and -count from len and width */
  	switch (txd->width) {
  	case 1:
  		dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
  		break;
  	case 2:
  		dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
  		break;
  	case 4:
  		dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
  		break;
  	}
  
  	if (s3cchan->slave) {
  		struct s3c24xx_dma_channel *cdata =
  					&pdata->channels[s3cchan->id];
  
  		if (s3cdma->sdata->has_reqsel) {
  			writel_relaxed((cdata->chansel << 1) |
  							S3C24XX_DMAREQSEL_HW,
  					phy->base + S3C24XX_DMAREQSEL);
  		} else {
  			int csel = cdata->chansel >> (phy->id *
  							S3C24XX_CHANSEL_WIDTH);
  
  			csel &= S3C24XX_CHANSEL_REQ_MASK;
  			dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
  			dcon |= S3C24XX_DCON_HWTRIG;
  		}
  	} else {
  		if (s3cdma->sdata->has_reqsel)
  			writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
  	}
  
  	writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
  	writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
  	writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
  	writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
  	writel_relaxed(dcon, phy->base + S3C24XX_DCON);
  
  	val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
  	val &= ~S3C24XX_DMASKTRIG_STOP;
  	val |= S3C24XX_DMASKTRIG_ON;
  
  	/* trigger the dma operation for memcpy transfers */
  	if (!s3cchan->slave)
  		val |= S3C24XX_DMASKTRIG_SWTRIG;
  
  	writel(val, phy->base + S3C24XX_DMASKTRIG);
  }
  
  /*
   * Set the initial DMA register values and start first sg.
   */
  static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_phy *phy = s3cchan->phy;
  	struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
  	struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
  
  	list_del(&txd->vd.node);
  
  	s3cchan->at = txd;
  
  	/* Wait for channel inactive */
  	while (s3c24xx_dma_phy_busy(phy))
  		cpu_relax();
  
  	/* point to the first element of the sg list */
  	txd->at = txd->dsg_list.next;
  	s3c24xx_dma_start_next_sg(s3cchan, txd);
  }
  
  static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
  				struct s3c24xx_dma_chan *s3cchan)
  {
  	LIST_HEAD(head);
  
  	vchan_get_all_descriptors(&s3cchan->vc, &head);
  	vchan_dma_desc_free_list(&s3cchan->vc, &head);
  }
  
  /*
   * Try to allocate a physical channel.  When successful, assign it to
   * this virtual channel, and initiate the next descriptor.  The
   * virtual channel lock must be held at this point.
   */
  static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	struct s3c24xx_dma_phy *phy;
  
  	phy = s3c24xx_dma_get_phy(s3cchan);
  	if (!phy) {
  		dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
  			s3cchan->name);
  		s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
  		return;
  	}
  
  	dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
  		phy->id, s3cchan->name);
  
  	s3cchan->phy = phy;
  	s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
  
  	s3c24xx_dma_start_next_txd(s3cchan);
  }
  
  static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
  	struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  
  	dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
  		phy->id, s3cchan->name);
  
  	/*
  	 * We do this without taking the lock; we're really only concerned
  	 * about whether this pointer is NULL or not, and we're guaranteed
  	 * that this will only be called when it _already_ is non-NULL.
  	 */
  	phy->serving = s3cchan;
  	s3cchan->phy = phy;
  	s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
  	s3c24xx_dma_start_next_txd(s3cchan);
  }
  
  /*
   * Free a physical DMA channel, potentially reallocating it to another
   * virtual channel if we have any pending.
   */
  static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
  {
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	struct s3c24xx_dma_chan *p, *next;
  
  retry:
  	next = NULL;
  
  	/* Find a waiting virtual channel for the next transfer. */
  	list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
  		if (p->state == S3C24XX_DMA_CHAN_WAITING) {
  			next = p;
  			break;
  		}
  
  	if (!next) {
  		list_for_each_entry(p, &s3cdma->slave.channels,
  				    vc.chan.device_node)
  			if (p->state == S3C24XX_DMA_CHAN_WAITING &&
  				      s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
  				next = p;
  				break;
  			}
  	}
  
  	/* Ensure that the physical channel is stopped */
  	s3c24xx_dma_terminate_phy(s3cchan->phy);
  
  	if (next) {
  		bool success;
  
  		/*
  		 * Eww.  We know this isn't going to deadlock
  		 * but lockdep probably doesn't.
  		 */
  		spin_lock(&next->vc.lock);
  		/* Re-check the state now that we have the lock */
  		success = next->state == S3C24XX_DMA_CHAN_WAITING;
  		if (success)
  			s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
  		spin_unlock(&next->vc.lock);
  
  		/* If the state changed, try to find another channel */
  		if (!success)
  			goto retry;
  	} else {
  		/* No more jobs, so free up the physical channel */
  		s3c24xx_dma_put_phy(s3cchan->phy);
  	}
  
  	s3cchan->phy = NULL;
  	s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
  }
  
  static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
  {
  	struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
  
  	if (!s3cchan->slave)
  		dma_descriptor_unmap(&vd->tx);
  
  	s3c24xx_dma_free_txd(txd);
  }
  
  static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
  {
  	struct s3c24xx_dma_phy *phy = data;
  	struct s3c24xx_dma_chan *s3cchan = phy->serving;
  	struct s3c24xx_txd *txd;
  
  	dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
  
  	/*
  	 * Interrupts happen to notify the completion of a transfer and the
  	 * channel should have moved into its stop state already on its own.
  	 * Therefore interrupts on channels not bound to a virtual channel
  	 * should never happen. Nevertheless send a terminate command to the
  	 * channel if the unlikely case happens.
  	 */
  	if (unlikely(!s3cchan)) {
  		dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
  			phy->id);
  
  		s3c24xx_dma_terminate_phy(phy);
  
  		return IRQ_HANDLED;
  	}
  
  	spin_lock(&s3cchan->vc.lock);
  	txd = s3cchan->at;
  	if (txd) {
  		/* when more sg's are in this txd, start the next one */
  		if (!list_is_last(txd->at, &txd->dsg_list)) {
  			txd->at = txd->at->next;
  			if (txd->cyclic)
  				vchan_cyclic_callback(&txd->vd);
  			s3c24xx_dma_start_next_sg(s3cchan, txd);
  		} else if (!txd->cyclic) {
  			s3cchan->at = NULL;
  			vchan_cookie_complete(&txd->vd);
  
  			/*
  			 * And start the next descriptor (if any),
  			 * otherwise free this channel.
  			 */
  			if (vchan_next_desc(&s3cchan->vc))
  				s3c24xx_dma_start_next_txd(s3cchan);
  			else
  				s3c24xx_dma_phy_free(s3cchan);
  		} else {
  			vchan_cyclic_callback(&txd->vd);
  
  			/* Cyclic: reset at beginning */
  			txd->at = txd->dsg_list.next;
  			s3c24xx_dma_start_next_sg(s3cchan, txd);
  		}
  	}
  	spin_unlock(&s3cchan->vc.lock);
  
  	return IRQ_HANDLED;
  }
  
  /*
   * The DMA ENGINE API
   */
  
  static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	unsigned long flags;
  	int ret = 0;
  
  	spin_lock_irqsave(&s3cchan->vc.lock, flags);
  
  	if (!s3cchan->phy && !s3cchan->at) {
  		dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
  			s3cchan->id);
  		ret = -EINVAL;
  		goto unlock;
  	}
  
  	s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
  
  	/* Mark physical channel as free */
  	if (s3cchan->phy)
  		s3c24xx_dma_phy_free(s3cchan);
  
  	/* Dequeue current job */
  	if (s3cchan->at) {
  		vchan_terminate_vdesc(&s3cchan->at->vd);
  		s3cchan->at = NULL;
  	}
  
  	/* Dequeue jobs not yet fired as well */
  	s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
  unlock:
  	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  
  	return ret;
  }
  
  static void s3c24xx_dma_synchronize(struct dma_chan *chan)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  
  	vchan_synchronize(&s3cchan->vc);
  }
  
  static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
  {
  	/* Ensure all queued descriptors are freed */
  	vchan_free_chan_resources(to_virt_chan(chan));
  }
  
  static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
  		dma_cookie_t cookie, struct dma_tx_state *txstate)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	struct s3c24xx_txd *txd;
  	struct s3c24xx_sg *dsg;
  	struct virt_dma_desc *vd;
  	unsigned long flags;
  	enum dma_status ret;
  	size_t bytes = 0;
  
  	spin_lock_irqsave(&s3cchan->vc.lock, flags);
  	ret = dma_cookie_status(chan, cookie, txstate);
  
  	/*
  	 * There's no point calculating the residue if there's
  	 * no txstate to store the value.
  	 */
  	if (ret == DMA_COMPLETE || !txstate) {
  		spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  		return ret;
  	}
  
  	vd = vchan_find_desc(&s3cchan->vc, cookie);
  	if (vd) {
  		/* On the issued list, so hasn't been processed yet */
  		txd = to_s3c24xx_txd(&vd->tx);
  
  		list_for_each_entry(dsg, &txd->dsg_list, node)
  			bytes += dsg->len;
  	} else {
  		/*
  		 * Currently running, so sum over the pending sg's and
  		 * the currently active one.
  		 */
  		txd = s3cchan->at;
  
  		dsg = list_entry(txd->at, struct s3c24xx_sg, node);
  		list_for_each_entry_from(dsg, &txd->dsg_list, node)
  			bytes += dsg->len;
  
  		bytes += s3c24xx_dma_getbytes_chan(s3cchan);
  	}
  	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  
  	/*
  	 * This cookie not complete yet
  	 * Get number of bytes left in the active transactions and queue
  	 */
  	dma_set_residue(txstate, bytes);
  
  	/* Whether waiting or running, we're in progress */
  	return ret;
  }
  
  /*
   * Initialize a descriptor to be used by memcpy submit
   */
  static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
  		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  		size_t len, unsigned long flags)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	struct s3c24xx_txd *txd;
  	struct s3c24xx_sg *dsg;
  	int src_mod, dest_mod;
  
  	dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
  			len, s3cchan->name);
  
  	if ((len & S3C24XX_DCON_TC_MASK) != len) {
  		dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
  		return NULL;
  	}
  
  	txd = s3c24xx_dma_get_txd();
  	if (!txd)
  		return NULL;
  
  	dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  	if (!dsg) {
  		s3c24xx_dma_free_txd(txd);
  		return NULL;
  	}
  	list_add_tail(&dsg->node, &txd->dsg_list);
  
  	dsg->src_addr = src;
  	dsg->dst_addr = dest;
  	dsg->len = len;
  
  	/*
  	 * Determine a suitable transfer width.
  	 * The DMA controller cannot fetch/store information which is not
  	 * naturally aligned on the bus, i.e., a 4 byte fetch must start at
  	 * an address divisible by 4 - more generally addr % width must be 0.
  	 */
  	src_mod = src % 4;
  	dest_mod = dest % 4;
  	switch (len % 4) {
  	case 0:
  		txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
  		break;
  	case 2:
  		txd->width = ((src_mod == 2 || src_mod == 0) &&
  			      (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
  		break;
  	default:
  		txd->width = 1;
  		break;
  	}
  
  	txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
  	txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
  	txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
  		     S3C24XX_DCON_SERV_WHOLE;
  
  	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  }
  
  static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
  	struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
  	enum dma_transfer_direction direction, unsigned long flags)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  	struct s3c24xx_txd *txd;
  	struct s3c24xx_sg *dsg;
  	unsigned sg_len;
  	dma_addr_t slave_addr;
  	u32 hwcfg = 0;
  	int i;
  
  	dev_dbg(&s3cdma->pdev->dev,
  		"prepare cyclic transaction of %zu bytes with period %zu from %s\n",
  		size, period, s3cchan->name);
  
  	if (!is_slave_direction(direction)) {
  		dev_err(&s3cdma->pdev->dev,
  			"direction %d unsupported\n", direction);
  		return NULL;
  	}
  
  	txd = s3c24xx_dma_get_txd();
  	if (!txd)
  		return NULL;
  
  	txd->cyclic = 1;
  
  	if (cdata->handshake)
  		txd->dcon |= S3C24XX_DCON_HANDSHAKE;
  
  	switch (cdata->bus) {
  	case S3C24XX_DMA_APB:
  		txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
  		hwcfg |= S3C24XX_DISRCC_LOC_APB;
  		break;
  	case S3C24XX_DMA_AHB:
  		txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
  		hwcfg |= S3C24XX_DISRCC_LOC_AHB;
  		break;
  	}
  
  	/*
  	 * Always assume our peripheral desintation is a fixed
  	 * address in memory.
  	 */
  	hwcfg |= S3C24XX_DISRCC_INC_FIXED;
  
  	/*
  	 * Individual dma operations are requested by the slave,
  	 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
  	 */
  	txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
  
  	if (direction == DMA_MEM_TO_DEV) {
  		txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
  			      S3C24XX_DISRCC_INC_INCREMENT;
  		txd->didstc = hwcfg;
  		slave_addr = s3cchan->cfg.dst_addr;
  		txd->width = s3cchan->cfg.dst_addr_width;
  	} else {
  		txd->disrcc = hwcfg;
  		txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
  			      S3C24XX_DIDSTC_INC_INCREMENT;
  		slave_addr = s3cchan->cfg.src_addr;
  		txd->width = s3cchan->cfg.src_addr_width;
  	}
  
  	sg_len = size / period;
  
  	for (i = 0; i < sg_len; i++) {
  		dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  		if (!dsg) {
  			s3c24xx_dma_free_txd(txd);
  			return NULL;
  		}
  		list_add_tail(&dsg->node, &txd->dsg_list);
  
  		dsg->len = period;
  		/* Check last period length */
  		if (i == sg_len - 1)
  			dsg->len = size - period * i;
  		if (direction == DMA_MEM_TO_DEV) {
  			dsg->src_addr = addr + period * i;
  			dsg->dst_addr = slave_addr;
  		} else { /* DMA_DEV_TO_MEM */
  			dsg->src_addr = slave_addr;
  			dsg->dst_addr = addr + period * i;
  		}
  	}
  
  	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  }
  
  static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
  		struct dma_chan *chan, struct scatterlist *sgl,
  		unsigned int sg_len, enum dma_transfer_direction direction,
  		unsigned long flags, void *context)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  	const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  	struct s3c24xx_txd *txd;
  	struct s3c24xx_sg *dsg;
  	struct scatterlist *sg;
  	dma_addr_t slave_addr;
  	u32 hwcfg = 0;
  	int tmp;
  
  	dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
  			sg_dma_len(sgl), s3cchan->name);
  
  	txd = s3c24xx_dma_get_txd();
  	if (!txd)
  		return NULL;
  
  	if (cdata->handshake)
  		txd->dcon |= S3C24XX_DCON_HANDSHAKE;
  
  	switch (cdata->bus) {
  	case S3C24XX_DMA_APB:
  		txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
  		hwcfg |= S3C24XX_DISRCC_LOC_APB;
  		break;
  	case S3C24XX_DMA_AHB:
  		txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
  		hwcfg |= S3C24XX_DISRCC_LOC_AHB;
  		break;
  	}
  
  	/*
  	 * Always assume our peripheral desintation is a fixed
  	 * address in memory.
  	 */
  	hwcfg |= S3C24XX_DISRCC_INC_FIXED;
  
  	/*
  	 * Individual dma operations are requested by the slave,
  	 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
  	 */
  	txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
  
  	if (direction == DMA_MEM_TO_DEV) {
  		txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
  			      S3C24XX_DISRCC_INC_INCREMENT;
  		txd->didstc = hwcfg;
  		slave_addr = s3cchan->cfg.dst_addr;
  		txd->width = s3cchan->cfg.dst_addr_width;
  	} else if (direction == DMA_DEV_TO_MEM) {
  		txd->disrcc = hwcfg;
  		txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
  			      S3C24XX_DIDSTC_INC_INCREMENT;
  		slave_addr = s3cchan->cfg.src_addr;
  		txd->width = s3cchan->cfg.src_addr_width;
  	} else {
  		s3c24xx_dma_free_txd(txd);
  		dev_err(&s3cdma->pdev->dev,
  			"direction %d unsupported\n", direction);
  		return NULL;
  	}
  
  	for_each_sg(sgl, sg, sg_len, tmp) {
  		dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  		if (!dsg) {
  			s3c24xx_dma_free_txd(txd);
  			return NULL;
  		}
  		list_add_tail(&dsg->node, &txd->dsg_list);
  
  		dsg->len = sg_dma_len(sg);
  		if (direction == DMA_MEM_TO_DEV) {
  			dsg->src_addr = sg_dma_address(sg);
  			dsg->dst_addr = slave_addr;
  		} else { /* DMA_DEV_TO_MEM */
  			dsg->src_addr = slave_addr;
  			dsg->dst_addr = sg_dma_address(sg);
  		}
  	}
  
  	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  }
  
  /*
   * Slave transactions callback to the slave device to allow
   * synchronization of slave DMA signals with the DMAC enable
   */
  static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
  {
  	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  	unsigned long flags;
  
  	spin_lock_irqsave(&s3cchan->vc.lock, flags);
  	if (vchan_issue_pending(&s3cchan->vc)) {
  		if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
  			s3c24xx_dma_phy_alloc_and_start(s3cchan);
  	}
  	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  }
  
  /*
   * Bringup and teardown
   */
  
  /*
   * Initialise the DMAC memcpy/slave channels.
   * Make a local wrapper to hold required data
   */
  static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
  		struct dma_device *dmadev, unsigned int channels, bool slave)
  {
  	struct s3c24xx_dma_chan *chan;
  	int i;
  
  	INIT_LIST_HEAD(&dmadev->channels);
  
  	/*
  	 * Register as many many memcpy as we have physical channels,
  	 * we won't always be able to use all but the code will have
  	 * to cope with that situation.
  	 */
  	for (i = 0; i < channels; i++) {
  		chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
  		if (!chan)
  			return -ENOMEM;
  
  		chan->id = i;
  		chan->host = s3cdma;
  		chan->state = S3C24XX_DMA_CHAN_IDLE;
  
  		if (slave) {
  			chan->slave = true;
  			chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
  			if (!chan->name)
  				return -ENOMEM;
  		} else {
  			chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
  			if (!chan->name)
  				return -ENOMEM;
  		}
  		dev_dbg(dmadev->dev,
  			 "initialize virtual channel \"%s\"\n",
  			 chan->name);
  
  		chan->vc.desc_free = s3c24xx_dma_desc_free;
  		vchan_init(&chan->vc, dmadev);
  	}
  	dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
  		 i, slave ? "slave" : "memcpy");
  	return i;
  }
  
  static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
  {
  	struct s3c24xx_dma_chan *chan = NULL;
  	struct s3c24xx_dma_chan *next;
  
  	list_for_each_entry_safe(chan,
  				 next, &dmadev->channels, vc.chan.device_node) {
  		list_del(&chan->vc.chan.device_node);
  		tasklet_kill(&chan->vc.task);
  	}
  }
  
  /* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */
  static struct soc_data soc_s3c2410 = {
  	.stride = 0x40,
  	.has_reqsel = false,
  	.has_clocks = false,
  };
  
  /* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */
  static struct soc_data soc_s3c2412 = {
  	.stride = 0x40,
  	.has_reqsel = true,
  	.has_clocks = true,
  };
  
  /* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */
  static struct soc_data soc_s3c2443 = {
  	.stride = 0x100,
  	.has_reqsel = true,
  	.has_clocks = true,
  };
  
  static const struct platform_device_id s3c24xx_dma_driver_ids[] = {
  	{
  		.name		= "s3c2410-dma",
  		.driver_data	= (kernel_ulong_t)&soc_s3c2410,
  	}, {
  		.name		= "s3c2412-dma",
  		.driver_data	= (kernel_ulong_t)&soc_s3c2412,
  	}, {
  		.name		= "s3c2443-dma",
  		.driver_data	= (kernel_ulong_t)&soc_s3c2443,
  	},
  	{ },
  };
  
  static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
  {
  	return (struct soc_data *)
  			 platform_get_device_id(pdev)->driver_data;
  }
  
  static int s3c24xx_dma_probe(struct platform_device *pdev)
  {
  	const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
  	struct s3c24xx_dma_engine *s3cdma;
  	struct soc_data *sdata;
  	struct resource *res;
  	int ret;
  	int i;
  
  	if (!pdata) {
  		dev_err(&pdev->dev, "platform data missing\n");
  		return -ENODEV;
  	}
  
  	/* Basic sanity check */
  	if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
  		dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
  			pdata->num_phy_channels, MAX_DMA_CHANNELS);
  		return -EINVAL;
  	}
  
  	sdata = s3c24xx_dma_get_soc_data(pdev);
  	if (!sdata)
  		return -EINVAL;
  
  	s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
  	if (!s3cdma)
  		return -ENOMEM;
  
  	s3cdma->pdev = pdev;
  	s3cdma->pdata = pdata;
  	s3cdma->sdata = sdata;
  
  	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  	s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
  	if (IS_ERR(s3cdma->base))
  		return PTR_ERR(s3cdma->base);
  
  	s3cdma->phy_chans = devm_kzalloc(&pdev->dev,
  					      sizeof(struct s3c24xx_dma_phy) *
  							pdata->num_phy_channels,
  					      GFP_KERNEL);
  	if (!s3cdma->phy_chans)
  		return -ENOMEM;
  
  	/* acquire irqs and clocks for all physical channels */
  	for (i = 0; i < pdata->num_phy_channels; i++) {
  		struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  		char clk_name[6];
  
  		phy->id = i;
  		phy->base = s3cdma->base + (i * sdata->stride);
  		phy->host = s3cdma;
  
  		phy->irq = platform_get_irq(pdev, i);
  		if (phy->irq < 0) {
  			dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
  				i, phy->irq);
  			continue;
  		}
  
  		ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
  				       0, pdev->name, phy);
  		if (ret) {
  			dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
  				i, ret);
  			continue;
  		}
  
  		if (sdata->has_clocks) {
  			sprintf(clk_name, "dma.%d", i);
  			phy->clk = devm_clk_get(&pdev->dev, clk_name);
  			if (IS_ERR(phy->clk) && sdata->has_clocks) {
  				dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n",
  					i, PTR_ERR(phy->clk));
  				continue;
  			}
  
  			ret = clk_prepare(phy->clk);
  			if (ret) {
  				dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
  					i, ret);
  				continue;
  			}
  		}
  
  		spin_lock_init(&phy->lock);
  		phy->valid = true;
  
  		dev_dbg(&pdev->dev, "physical channel %d is %s\n",
  			i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
  	}
  
  	/* Initialize memcpy engine */
  	dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
  	dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
  	s3cdma->memcpy.dev = &pdev->dev;
  	s3cdma->memcpy.device_free_chan_resources =
  					s3c24xx_dma_free_chan_resources;
  	s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
  	s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
  	s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
  	s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
  	s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
  	s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
  
  	/* Initialize slave engine for SoC internal dedicated peripherals */
  	dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
  	dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
  	dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
  	s3cdma->slave.dev = &pdev->dev;
  	s3cdma->slave.device_free_chan_resources =
  					s3c24xx_dma_free_chan_resources;
  	s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
  	s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
  	s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
  	s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
  	s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
  	s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
  	s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
  	s3cdma->slave.filter.map = pdata->slave_map;
  	s3cdma->slave.filter.mapcnt = pdata->slavecnt;
  	s3cdma->slave.filter.fn = s3c24xx_dma_filter;
  
  	/* Register as many memcpy channels as there are physical channels */
  	ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
  						pdata->num_phy_channels, false);
  	if (ret <= 0) {
  		dev_warn(&pdev->dev,
  			 "%s failed to enumerate memcpy channels - %d\n",
  			 __func__, ret);
  		goto err_memcpy;
  	}
  
  	/* Register slave channels */
  	ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
  				pdata->num_channels, true);
  	if (ret <= 0) {
  		dev_warn(&pdev->dev,
  			"%s failed to enumerate slave channels - %d\n",
  				__func__, ret);
  		goto err_slave;
  	}
  
  	ret = dma_async_device_register(&s3cdma->memcpy);
  	if (ret) {
  		dev_warn(&pdev->dev,
  			"%s failed to register memcpy as an async device - %d\n",
  			__func__, ret);
  		goto err_memcpy_reg;
  	}
  
  	ret = dma_async_device_register(&s3cdma->slave);
  	if (ret) {
  		dev_warn(&pdev->dev,
  			"%s failed to register slave as an async device - %d\n",
  			__func__, ret);
  		goto err_slave_reg;
  	}
  
  	platform_set_drvdata(pdev, s3cdma);
  	dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
  		 pdata->num_phy_channels);
  
  	return 0;
  
  err_slave_reg:
  	dma_async_device_unregister(&s3cdma->memcpy);
  err_memcpy_reg:
  	s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
  err_slave:
  	s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
  err_memcpy:
  	if (sdata->has_clocks)
  		for (i = 0; i < pdata->num_phy_channels; i++) {
  			struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  			if (phy->valid)
  				clk_unprepare(phy->clk);
  		}
  
  	return ret;
  }
  
  static void s3c24xx_dma_free_irq(struct platform_device *pdev,
  				struct s3c24xx_dma_engine *s3cdma)
  {
  	int i;
  
  	for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
  		struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  
  		devm_free_irq(&pdev->dev, phy->irq, phy);
  	}
  }
  
  static int s3c24xx_dma_remove(struct platform_device *pdev)
  {
  	const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
  	struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
  	struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
  	int i;
  
  	dma_async_device_unregister(&s3cdma->slave);
  	dma_async_device_unregister(&s3cdma->memcpy);
  
  	s3c24xx_dma_free_irq(pdev, s3cdma);
  
  	s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
  	s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
  
  	if (sdata->has_clocks)
  		for (i = 0; i < pdata->num_phy_channels; i++) {
  			struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  			if (phy->valid)
  				clk_unprepare(phy->clk);
  		}
  
  	return 0;
  }
  
  static struct platform_driver s3c24xx_dma_driver = {
  	.driver		= {
  		.name	= "s3c24xx-dma",
  	},
  	.id_table	= s3c24xx_dma_driver_ids,
  	.probe		= s3c24xx_dma_probe,
  	.remove		= s3c24xx_dma_remove,
  };
  
  module_platform_driver(s3c24xx_dma_driver);
  
  bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
  {
  	struct s3c24xx_dma_chan *s3cchan;
  
  	if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
  		return false;
  
  	s3cchan = to_s3c24xx_dma_chan(chan);
  
  	return s3cchan->id == (uintptr_t)param;
  }
  EXPORT_SYMBOL(s3c24xx_dma_filter);
  
  MODULE_DESCRIPTION("S3C24XX DMA Driver");
  MODULE_AUTHOR("Heiko Stuebner");
  MODULE_LICENSE("GPL v2");
  