diff options
| -rw-r--r-- | drivers/dma/Makefile | 2 | ||||
| -rw-r--r-- | drivers/dma/fsl-edma-common.c | 552 | ||||
| -rw-r--r-- | drivers/dma/fsl-edma-common.h | 241 | ||||
| -rw-r--r-- | drivers/dma/fsl-edma.c | 697 |
4 files changed, 795 insertions, 697 deletions
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index c91702d88b95..21db4e51b246 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -31,7 +31,7 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_HSU_DMA) += hsu/ obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c new file mode 100644 index 000000000000..3fc786b3d03c --- /dev/null +++ b/drivers/dma/fsl-edma-common.c @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: GPL-2.0+ +// +// Copyright (c) 2013-2014 Freescale Semiconductor, Inc +// Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> + +#include <linux/dmapool.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include "fsl-edma-common.h" + +#define EDMA_CR 0x00 +#define EDMA_ES 0x04 +#define EDMA_ERQ 0x0C +#define EDMA_EEI 0x14 +#define EDMA_SERQ 0x1B +#define EDMA_CERQ 0x1A +#define EDMA_SEEI 0x19 +#define EDMA_CEEI 0x18 +#define EDMA_CINT 0x1F +#define EDMA_CERR 0x1E +#define EDMA_SSRT 0x1D +#define EDMA_CDNE 0x1C +#define EDMA_INTR 0x24 +#define EDMA_ERR 0x2C + +#define EDMA64_ERQH 0x08 +#define EDMA64_EEIH 0x10 +#define EDMA64_SERQ 0x18 +#define EDMA64_CERQ 0x19 +#define EDMA64_SEEI 0x1a +#define EDMA64_CEEI 0x1b +#define EDMA64_CINT 0x1c +#define EDMA64_CERR 0x1d +#define EDMA64_SSRT 0x1e +#define EDMA64_CDNE 0x1f +#define EDMA64_INTH 0x20 +#define EDMA64_INTL 0x24 +#define EDMA64_ERRH 0x28 +#define EDMA64_ERRL 0x2c + +#define EDMA_TCD 0x1000 + +static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); + edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); +} + +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); + edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); +} +EXPORT_SYMBOL_GPL(fsl_edma_disable_request); + +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, + unsigned int slot, bool enable) +{ + u32 ch = fsl_chan->vchan.chan.chan_id; + void __iomem *muxaddr; + unsigned int chans_per_mux, ch_off; + + chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; + ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; + muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; + slot = EDMAMUX_CHCFG_SOURCE(slot); + + if (enable) + iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); + else + iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); +} +EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); + +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) +{ + switch (addr_width) { + case 1: + return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; + case 2: + return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; + case 4: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + case 8: + return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; + default: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + } +} + +void fsl_edma_free_desc(struct virt_dma_desc *vdesc) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = to_fsl_edma_desc(vdesc); + for (i = 0; i < fsl_desc->n_tcds; i++) + dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); +} +EXPORT_SYMBOL_GPL(fsl_edma_free_desc); + +int fsl_edma_terminate_all(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_chan->edesc = NULL; + fsl_chan->idle = true; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_terminate_all); + +int fsl_edma_pause(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_disable_request(fsl_chan); + fsl_chan->status = DMA_PAUSED; + fsl_chan->idle = true; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_pause); + +int fsl_edma_resume(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_resume); + +int fsl_edma_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + fsl_chan->fsc.dir = cfg->direction; + if (cfg->direction == DMA_DEV_TO_MEM) { + fsl_chan->fsc.dev_addr = cfg->src_addr; + fsl_chan->fsc.addr_width = cfg->src_addr_width; + fsl_chan->fsc.burst = cfg->src_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); + } else if (cfg->direction == DMA_MEM_TO_DEV) { + fsl_chan->fsc.dev_addr = cfg->dst_addr; + fsl_chan->fsc.addr_width = cfg->dst_addr_width; + fsl_chan->fsc.burst = cfg->dst_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); + } else + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_slave_config); + +static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, + struct virt_dma_desc *vdesc, bool in_progress) +{ + struct fsl_edma_desc *edesc = fsl_chan->edesc; + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + enum dma_transfer_direction dir = fsl_chan->fsc.dir; + dma_addr_t cur_addr, dma_addr; + size_t len, size; + int i; + + /* calculate the total size in this desc */ + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) + len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + + if (!in_progress) + return len; + + if (dir == DMA_MEM_TO_DEV) + cur_addr = edma_readl( + fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); + else + cur_addr = edma_readl( + fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); + + /* figure out the finished and calculate the residue */ + for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { + size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); + if (dir == DMA_MEM_TO_DEV) + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); + else + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); + + len -= size; + if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { + len += dma_addr + size - cur_addr; + break; + } + } + + return len; +} + +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE) + return status; + + if (!txstate) + return fsl_chan->status; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); + if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) + txstate->residue = + fsl_edma_desc_residue(fsl_chan, vdesc, true); + else if (vdesc) + txstate->residue = + fsl_edma_desc_residue(fsl_chan, vdesc, false); + else + txstate->residue = 0; + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + return fsl_chan->status; +} +EXPORT_SYMBOL_GPL(fsl_edma_tx_status); + +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, + struct fsl_edma_hw_tcd *tcd) +{ + struct fsl_edma_engine *edma = fsl_chan->edma; + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + /* + * TCD parameters are stored in struct fsl_edma_hw_tcd in little + * endian format. However, we need to load the TCD registers in + * big- or little-endian obeying the eDMA engine model endian. + */ + edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); + edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); + edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); + + edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); + edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); + + edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); + edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); + + edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); + + edma_writel(edma, + le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); + + edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); +} + +static inline +void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, + u16 biter, u16 doff, u32 dlast_sga, bool major_int, + bool disable_req, bool enable_sg) +{ + u16 csr = 0; + + /* + * eDMA hardware SGs require the TCDs to be stored in little + * endian format irrespective of the register endian model. + * So we put the value in little endian in memory, waiting + * for fsl_edma_set_tcd_regs doing the swap. + */ + tcd->saddr = cpu_to_le32(src); + tcd->daddr = cpu_to_le32(dst); + + tcd->attr = cpu_to_le16(attr); + + tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); + + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); + tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); + + tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); + tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); + + tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); + + tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); + if (major_int) + csr |= EDMA_TCD_CSR_INT_MAJOR; + + if (disable_req) + csr |= EDMA_TCD_CSR_D_REQ; + + if (enable_sg) + csr |= EDMA_TCD_CSR_E_SG; + + tcd->csr = cpu_to_le16(csr); +} + +static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, + int sg_len) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = kzalloc(sizeof(*fsl_desc) + + sizeof(struct fsl_edma_sw_tcd) * + sg_len, GFP_NOWAIT); + if (!fsl_desc) + return NULL; + + fsl_desc->echan = fsl_chan; + fsl_desc->n_tcds = sg_len; + for (i = 0; i < sg_len; i++) { + fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, + GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); + if (!fsl_desc->tcd[i].vtcd) + goto err; + } + return fsl_desc; + +err: + while (--i >= 0) + dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); + return NULL; +} + +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + dma_addr_t dma_buf_next; + int sg_len, i; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + sg_len = buf_len / period_len; + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = true; + + dma_buf_next = dma_addr; + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + iter = period_len / nbytes; + + for (i = 0; i < sg_len; i++) { + if (dma_buf_next >= dma_addr + buf_len) + dma_buf_next = dma_addr; + + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = dma_buf_next; + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = dma_buf_next; + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, + fsl_chan->fsc.attr, soff, nbytes, 0, iter, + iter, doff, last_sg, true, false, true); + dma_buf_next += period_len; + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_prep_dma_cyclic); + +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + struct scatterlist *sg; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + int i; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = false; + + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + for_each_sg(sgl, sg, sg_len, i) { + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = sg_dma_address(sg); + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = sg_dma_address(sg); + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + iter = sg_dma_len(sg) / nbytes; + if (i < sg_len - 1) { + last_sg = fsl_desc->tcd[(i + 1)].ptcd; + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + false, false, true); + } else { + last_sg = 0; + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + true, true, false); + } + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg); + +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) +{ + struct virt_dma_desc *vdesc; + + vdesc = vchan_next_desc(&fsl_chan->vchan); + if (!vdesc) + return; + fsl_chan->edesc = to_fsl_edma_desc(vdesc); + fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; +} +EXPORT_SYMBOL_GPL(fsl_edma_xfer_desc); + +void fsl_edma_issue_pending(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + + if (unlikely(fsl_chan->pm_state != RUNNING)) { + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + /* cannot submit due to suspend */ + return; + } + + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); +} +EXPORT_SYMBOL_GPL(fsl_edma_issue_pending); + +int fsl_edma_alloc_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, + sizeof(struct fsl_edma_hw_tcd), + 32, 0); + return 0; +} +EXPORT_SYMBOL_GPL(fsl_edma_alloc_chan_resources); + +void fsl_edma_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + dma_pool_destroy(fsl_chan->tcd_pool); + fsl_chan->tcd_pool = NULL; +} +EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources); + +void fsl_edma_cleanup_vchan(struct dma_device *dmadev) +{ + struct fsl_edma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&chan->vchan.chan.device_node); + tasklet_kill(&chan->vchan.task); + } +} +EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h new file mode 100644 index 000000000000..5458dd3b6da8 --- /dev/null +++ b/drivers/dma/fsl-edma-common.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2013-2014 Freescale Semiconductor, Inc. + * Copyright 2018 Angelo Dureghello <angelo@sysam.it> + */ +#ifndef _FSL_EDMA_COMMON_H_ +#define _FSL_EDMA_COMMON_H_ + +#include "virt-dma.h" + +#define EDMA_CR 0x00 +#define EDMA_ES 0x04 +#define EDMA_ERQ 0x0C +#define EDMA_EEI 0x14 +#define EDMA_SERQ 0x1B +#define EDMA_CERQ 0x1A +#define EDMA_SEEI 0x19 +#define EDMA_CEEI 0x18 +#define EDMA_CINT 0x1F +#define EDMA_CERR 0x1E +#define EDMA_SSRT 0x1D +#define EDMA_CDNE 0x1C +#define EDMA_INTR 0x24 +#define EDMA_ERR 0x2C + +#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) +#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) +#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) +#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) +#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) +#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) +#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) +#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) +#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) +#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) +#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) + +#define EDMA_CR_EDBG BIT(1) +#define EDMA_CR_ERCA BIT(2) +#define EDMA_CR_ERGA BIT(3) +#define EDMA_CR_HOE BIT(4) +#define EDMA_CR_HALT BIT(5) +#define EDMA_CR_CLM BIT(6) +#define EDMA_CR_EMLM BIT(7) +#define EDMA_CR_ECX BIT(16) +#define EDMA_CR_CX BIT(17) + +#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) +#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) +#define EDMA_CINT_CINT(x) ((x) & 0x1F) +#define EDMA_CERR_CERR(x) ((x) & 0x1F) + +#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) +#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) +#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) +#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) +#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) +#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) +#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) +#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) +#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) +#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) +#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) +#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) + +#define EDMA_TCD_SOFF_SOFF(x) (x) +#define EDMA_TCD_NBYTES_NBYTES(x) (x) +#define EDMA_TCD_SLAST_SLAST(x) (x) +#define EDMA_TCD_DADDR_DADDR(x) (x) +#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) +#define EDMA_TCD_DOFF_DOFF(x) (x) +#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) +#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) + +#define EDMA_TCD_CSR_START BIT(0) +#define EDMA_TCD_CSR_INT_MAJOR BIT(1) +#define EDMA_TCD_CSR_INT_HALF BIT(2) +#define EDMA_TCD_CSR_D_REQ BIT(3) +#define EDMA_TCD_CSR_E_SG BIT(4) +#define EDMA_TCD_CSR_E_LINK BIT(5) +#define EDMA_TCD_CSR_ACTIVE BIT(6) +#define EDMA_TCD_CSR_DONE BIT(7) + +#define EDMAMUX_CHCFG_DIS 0x0 +#define EDMAMUX_CHCFG_ENBL 0x80 +#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) + +#define DMAMUX_NR 2 + +#define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) +enum fsl_edma_pm_state { + RUNNING = 0, + SUSPENDED, +}; + +struct fsl_edma_hw_tcd { + __le32 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le32 slast; + __le32 daddr; + __le16 doff; + __le16 citer; + __le32 dlast_sga; + __le16 csr; + __le16 biter; +}; + +struct fsl_edma_sw_tcd { + dma_addr_t ptcd; + struct fsl_edma_hw_tcd *vtcd; +}; + +struct fsl_edma_slave_config { + enum dma_transfer_direction dir; + enum dma_slave_buswidth addr_width; + u32 dev_addr; + u32 burst; + u32 attr; +}; + +struct fsl_edma_chan { + struct virt_dma_chan vchan; + enum dma_status status; + enum fsl_edma_pm_state pm_state; + bool idle; + u32 slave_id; + struct fsl_edma_engine *edma; + struct fsl_edma_desc *edesc; + struct fsl_edma_slave_config fsc; + struct dma_pool *tcd_pool; +}; + +struct fsl_edma_desc { + struct virt_dma_desc vdesc; + struct fsl_edma_chan *echan; + bool iscyclic; + unsigned int n_tcds; + struct fsl_edma_sw_tcd tcd[]; +}; + +struct fsl_edma_engine { + struct dma_device dma_dev; + void __iomem *membase; + void __iomem *muxbase[DMAMUX_NR]; + struct clk *muxclk[DMAMUX_NR]; + struct mutex fsl_edma_mutex; + u32 n_chans; + int txirq; + int errirq; + bool big_endian; + struct fsl_edma_chan chans[]; +}; + +/* + * R/W functions for big- or little-endian registers: + * The eDMA controller's endian is independent of the CPU core's endian. + * For the big-endian IP module, the offset for 8-bit or 16-bit registers + * should also be swapped opposite to that in little-endian IP. + */ +static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) +{ + if (edma->big_endian) + return ioread32be(addr); + else + return ioread32(addr); +} + +static inline void edma_writeb(struct fsl_edma_engine *edma, + u8 val, void __iomem *addr) +{ + /* swap the reg offset for these in big-endian mode */ + if (edma->big_endian) + iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); + else + iowrite8(val, addr); +} + +static inline void edma_writew(struct fsl_edma_engine *edma, + u16 val, void __iomem *addr) +{ + /* swap the reg offset for these in big-endian mode */ + if (edma->big_endian) + iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); + else + iowrite16(val, addr); +} + +static inline void edma_writel(struct fsl_edma_engine *edma, + u32 val, void __iomem *addr) +{ + if (edma->big_endian) + iowrite32be(val, addr); + else + iowrite32(val, addr); +} + +static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct fsl_edma_chan, vchan.chan); +} + +static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct fsl_edma_desc, vdesc); +} + +void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan); +void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, + unsigned int slot, bool enable); +void fsl_edma_free_desc(struct virt_dma_desc *vdesc); +int fsl_edma_terminate_all(struct dma_chan *chan); +int fsl_edma_pause(struct dma_chan *chan); +int fsl_edma_resume(struct dma_chan *chan); +int fsl_edma_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg); +enum dma_status fsl_edma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate); +struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags); +struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context); +void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan); +void fsl_edma_issue_pending(struct dma_chan *chan); +int fsl_edma_alloc_chan_resources(struct dma_chan *chan); +void fsl_edma_free_chan_resources(struct dma_chan *chan); +void fsl_edma_cleanup_vchan(struct dma_device *dmadev); + +#endif /* _FSL_EDMA_COMMON_H_ */ diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index c7568869284e..337ad0c0ea78 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -13,653 +13,16 @@ * option) any later version. */ -#include <linux/init.h> #include <linux/module.h> #include <linux/interrupt.h> #include <linux/clk.h> -#include <linux/dma-mapping.h> -#include <linux/dmapool.h> -#include <linux/slab.h> -#include <linux/spinlock.h> #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_address.h> #include <linux/of_irq.h> #include <linux/of_dma.h> -#include "virt-dma.h" - -#define EDMA_CR 0x00 -#define EDMA_ES 0x04 -#define EDMA_ERQ 0x0C -#define EDMA_EEI 0x14 -#define EDMA_SERQ 0x1B -#define EDMA_CERQ 0x1A -#define EDMA_SEEI 0x19 -#define EDMA_CEEI 0x18 -#define EDMA_CINT 0x1F -#define EDMA_CERR 0x1E -#define EDMA_SSRT 0x1D -#define EDMA_CDNE 0x1C -#define EDMA_INTR 0x24 -#define EDMA_ERR 0x2C - -#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) -#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) -#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) -#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) -#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) -#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) -#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) -#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) -#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) -#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) -#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) -#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) -#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) - -#define EDMA_CR_EDBG BIT(1) -#define EDMA_CR_ERCA BIT(2) -#define EDMA_CR_ERGA BIT(3) -#define EDMA_CR_HOE BIT(4) -#define EDMA_CR_HALT BIT(5) -#define EDMA_CR_CLM BIT(6) -#define EDMA_CR_EMLM BIT(7) -#define EDMA_CR_ECX BIT(16) -#define EDMA_CR_CX BIT(17) - -#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) -#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) -#define EDMA_CINT_CINT(x) ((x) & 0x1F) -#define EDMA_CERR_CERR(x) ((x) & 0x1F) - -#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) -#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) -#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) -#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) -#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) -#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) -#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) -#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) -#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) -#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) -#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) -#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) -#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) -#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) - -#define EDMA_TCD_SOFF_SOFF(x) (x) -#define EDMA_TCD_NBYTES_NBYTES(x) (x) -#define EDMA_TCD_SLAST_SLAST(x) (x) -#define EDMA_TCD_DADDR_DADDR(x) (x) -#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) -#define EDMA_TCD_DOFF_DOFF(x) (x) -#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) -#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) - -#define EDMA_TCD_CSR_START BIT(0) -#define EDMA_TCD_CSR_INT_MAJOR BIT(1) -#define EDMA_TCD_CSR_INT_HALF BIT(2) -#define EDMA_TCD_CSR_D_REQ BIT(3) -#define EDMA_TCD_CSR_E_SG BIT(4) -#define EDMA_TCD_CSR_E_LINK BIT(5) -#define EDMA_TCD_CSR_ACTIVE BIT(6) -#define EDMA_TCD_CSR_DONE BIT(7) - -#define EDMAMUX_CHCFG_DIS 0x0 -#define EDMAMUX_CHCFG_ENBL 0x80 -#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) - -#define DMAMUX_NR 2 - -#define FSL_EDMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) -enum fsl_edma_pm_state { - RUNNING = 0, - SUSPENDED, -}; - -struct fsl_edma_hw_tcd { - __le32 saddr; - __le16 soff; - __le16 attr; - __le32 nbytes; - __le32 slast; - __le32 daddr; - __le16 doff; - __le16 citer; - __le32 dlast_sga; - __le16 csr; - __le16 biter; -}; - -struct fsl_edma_sw_tcd { - dma_addr_t ptcd; - struct fsl_edma_hw_tcd *vtcd; -}; - -struct fsl_edma_slave_config { - enum dma_transfer_direction dir; - enum dma_slave_buswidth addr_width; - u32 dev_addr; - u32 burst; - u32 attr; -}; - -struct fsl_edma_chan { - struct virt_dma_chan vchan; - enum dma_status status; - enum fsl_edma_pm_state pm_state; - bool idle; - u32 slave_id; - struct fsl_edma_engine *edma; - struct fsl_edma_desc *edesc; - struct fsl_edma_slave_config fsc; - struct dma_pool *tcd_pool; -}; - -struct fsl_edma_desc { - struct virt_dma_desc vdesc; - struct fsl_edma_chan *echan; - bool iscyclic; - unsigned int n_tcds; - struct fsl_edma_sw_tcd tcd[]; -}; - -struct fsl_edma_engine { - struct dma_device dma_dev; - void __iomem *membase; - void __iomem *muxbase[DMAMUX_NR]; - struct clk *muxclk[DMAMUX_NR]; - struct mutex fsl_edma_mutex; - u32 n_chans; - int txirq; - int errirq; - bool big_endian; - struct fsl_edma_chan chans[]; -}; - -/* - * R/W functions for big- or little-endian registers: - * The eDMA controller's endian is independent of the CPU core's endian. - * For the big-endian IP module, the offset for 8-bit or 16-bit registers - * should also be swapped opposite to that in little-endian IP. - */ - -static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) -{ - if (edma->big_endian) - return ioread32be(addr); - else - return ioread32(addr); -} - -static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) -{ - /* swap the reg offset for these in big-endian mode */ - if (edma->big_endian) - iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); - else - iowrite8(val, addr); -} - -static void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) -{ - /* swap the reg offset for these in big-endian mode */ - if (edma->big_endian) - iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); - else - iowrite16(val, addr); -} - -static void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) -{ - if (edma->big_endian) - iowrite32be(val, addr); - else - iowrite32(val, addr); -} - -static struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) -{ - return container_of(chan, struct fsl_edma_chan, vchan.chan); -} - -static struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) -{ - return container_of(vd, struct fsl_edma_desc, vdesc); -} - -static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) -{ - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); - edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); -} - -static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) -{ - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); - edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); -} - -static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, - unsigned int slot, bool enable) -{ - u32 ch = fsl_chan->vchan.chan.chan_id; - void __iomem *muxaddr; - unsigned chans_per_mux, ch_off; - - chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; - ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; - muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; - slot = EDMAMUX_CHCFG_SOURCE(slot); - - if (enable) - iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); - else - iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); -} - -static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) -{ - switch (addr_width) { - case 1: - return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; - case 2: - return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; - case 4: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - case 8: - return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; - default: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - } -} - -static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) -{ - struct fsl_edma_desc *fsl_desc; - int i; - - fsl_desc = to_fsl_edma_desc(vdesc); - for (i = 0; i < fsl_desc->n_tcds; i++) - dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, - fsl_desc->tcd[i].ptcd); - kfree(fsl_desc); -} - -static int fsl_edma_terminate_all(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - LIST_HEAD(head); - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - fsl_edma_disable_request(fsl_chan); - fsl_chan->edesc = NULL; - fsl_chan->idle = true; - vchan_get_all_descriptors(&fsl_chan->vchan, &head); - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); - return 0; -} - -static int fsl_edma_pause(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - if (fsl_chan->edesc) { - fsl_edma_disable_request(fsl_chan); - fsl_chan->status = DMA_PAUSED; - fsl_chan->idle = true; - } - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - return 0; -} - -static int fsl_edma_resume(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - if (fsl_chan->edesc) { - fsl_edma_enable_request(fsl_chan); - fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; - } - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - return 0; -} - -static int fsl_edma_slave_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - fsl_chan->fsc.dir = cfg->direction; - if (cfg->direction == DMA_DEV_TO_MEM) { - fsl_chan->fsc.dev_addr = cfg->src_addr; - fsl_chan->fsc.addr_width = cfg->src_addr_width; - fsl_chan->fsc.burst = cfg->src_maxburst; - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); - } else if (cfg->direction == DMA_MEM_TO_DEV) { - fsl_chan->fsc.dev_addr = cfg->dst_addr; - fsl_chan->fsc.addr_width = cfg->dst_addr_width; - fsl_chan->fsc.burst = cfg->dst_maxburst; - fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); - } else { - return -EINVAL; - } - return 0; -} - -static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, - struct virt_dma_desc *vdesc, bool in_progress) -{ - struct fsl_edma_desc *edesc = fsl_chan->edesc; - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - enum dma_transfer_direction dir = fsl_chan->fsc.dir; - dma_addr_t cur_addr, dma_addr; - size_t len, size; - int i; - - /* calculate the total size in this desc */ - for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) - len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); - - if (!in_progress) - return len; - - if (dir == DMA_MEM_TO_DEV) - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); - else - cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); - - /* figure out the finished and calculate the residue */ - for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { - size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); - if (dir == DMA_MEM_TO_DEV) - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); - else - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); - - len -= size; - if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { - len += dma_addr + size - cur_addr; - break; - } - } - - return len; -} - -static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct virt_dma_desc *vdesc; - enum dma_status status; - unsigned long flags; - - status = dma_cookie_status(chan, cookie, txstate); - if (status == DMA_COMPLETE) - return status; - - if (!txstate) - return fsl_chan->status; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); - if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true); - else if (vdesc) - txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false); - else - txstate->residue = 0; - - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - - return fsl_chan->status; -} - -static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, - struct fsl_edma_hw_tcd *tcd) -{ - struct fsl_edma_engine *edma = fsl_chan->edma; - void __iomem *addr = fsl_chan->edma->membase; - u32 ch = fsl_chan->vchan.chan.chan_id; - - /* - * TCD parameters are stored in struct fsl_edma_hw_tcd in little - * endian format. However, we need to load the TCD registers in - * big- or little-endian obeying the eDMA engine model endian. - */ - edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); - edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); - edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); - - edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); - edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); - - edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); - edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); - - edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); - edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); - edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); - - edma_writel(edma, le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); - - edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); -} - -static inline -void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, - u16 biter, u16 doff, u32 dlast_sga, bool major_int, - bool disable_req, bool enable_sg) -{ - u16 csr = 0; - - /* - * eDMA hardware SGs require the TCDs to be stored in little - * endian format irrespective of the register endian model. - * So we put the value in little endian in memory, waiting - * for fsl_edma_set_tcd_regs doing the swap. - */ - tcd->saddr = cpu_to_le32(src); - tcd->daddr = cpu_to_le32(dst); - - tcd->attr = cpu_to_le16(attr); - - tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); - - tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); - tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); - - tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); - tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); - - tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); - - tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); - if (major_int) - csr |= EDMA_TCD_CSR_INT_MAJOR; - - if (disable_req) - csr |= EDMA_TCD_CSR_D_REQ; - - if (enable_sg) - csr |= EDMA_TCD_CSR_E_SG; - - tcd->csr = cpu_to_le16(csr); -} - -static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, - int sg_len) -{ - struct fsl_edma_desc *fsl_desc; - int i; - - fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct fsl_edma_sw_tcd) * sg_len, - GFP_NOWAIT); - if (!fsl_desc) - return NULL; - - fsl_desc->echan = fsl_chan; - fsl_desc->n_tcds = sg_len; - for (i = 0; i < sg_len; i++) { - fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, - GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); - if (!fsl_desc->tcd[i].vtcd) - goto err; - } - return fsl_desc; - -err: - while (--i >= 0) - dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, - fsl_desc->tcd[i].ptcd); - kfree(fsl_desc); - return NULL; -} - -static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct fsl_edma_desc *fsl_desc; - dma_addr_t dma_buf_next; - int sg_len, i; - u32 src_addr, dst_addr, last_sg, nbytes; - u16 soff, doff, iter; - - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - - sg_len = buf_len / period_len; - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) - return NULL; - fsl_desc->iscyclic = true; - - dma_buf_next = dma_addr; - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; - iter = period_len / nbytes; - - for (i = 0; i < sg_len; i++) { - if (dma_buf_next >= dma_addr + buf_len) - dma_buf_next = dma_addr; - - /* get next sg's physical address */ - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = dma_buf_next; - dst_addr = fsl_chan->fsc.dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { - src_addr = fsl_chan->fsc.dev_addr; - dst_addr = dma_buf_next; - soff = 0; - doff = fsl_chan->fsc.addr_width; - } - - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, - fsl_chan->fsc.attr, soff, nbytes, 0, iter, - iter, doff, last_sg, true, false, true); - dma_buf_next += period_len; - } - - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); -} - -static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - struct fsl_edma_desc *fsl_desc; - struct scatterlist *sg; - u32 src_addr, dst_addr, last_sg, nbytes; - u16 soff, doff, iter; - int i; - - if (!is_slave_direction(fsl_chan->fsc.dir)) - return NULL; - - fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); - if (!fsl_desc) - return NULL; - fsl_desc->iscyclic = false; - - nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; - for_each_sg(sgl, sg, sg_len, i) { - /* get next sg's physical address */ - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; - - if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { - src_addr = sg_dma_address(sg); - dst_addr = fsl_chan->fsc.dev_addr; - soff = fsl_chan->fsc.addr_width; - doff = 0; - } else { - src_addr = fsl_chan->fsc.dev_addr; - dst_addr = sg_dma_address(sg); - soff = 0; - doff = fsl_chan->fsc.addr_width; - } - - iter = sg_dma_len(sg) / nbytes; - if (i < sg_len - 1) { - last_sg = fsl_desc->tcd[(i + 1)].ptcd; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, - dst_addr, fsl_chan->fsc.attr, soff, - nbytes, 0, iter, iter, doff, last_sg, - false, false, true); - } else { - last_sg = 0; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, - dst_addr, fsl_chan->fsc.attr, soff, - nbytes, 0, iter, iter, doff, last_sg, - true, true, false); - } - } - - return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); -} - -static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) -{ - struct virt_dma_desc *vdesc; - - vdesc = vchan_next_desc(&fsl_chan->vchan); - if (!vdesc) - return; - fsl_chan->edesc = to_fsl_edma_desc(vdesc); - fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); - fsl_edma_enable_request(fsl_chan); - fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; -} +#include "fsl-edma-common.h" static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) { @@ -730,25 +93,6 @@ static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) return fsl_edma_err_handler(irq, dev_id); } -static void fsl_edma_issue_pending(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - - if (unlikely(fsl_chan->pm_state != RUNNING)) { - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - /* cannot submit due to suspend */ - return; - } - - if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) - fsl_edma_xfer_desc(fsl_chan); - - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); -} - static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { @@ -781,34 +125,6 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, return NULL; } -static int fsl_edma_alloc_chan_resources(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, - sizeof(struct fsl_edma_hw_tcd), - 32, 0); - return 0; -} - -static void fsl_edma_free_chan_resources(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - unsigned long flags; - LIST_HEAD(head); - - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - fsl_edma_disable_request(fsl_chan); - fsl_edma_chan_mux(fsl_chan, 0, false); - fsl_chan->edesc = NULL; - vchan_get_all_descriptors(&fsl_chan->vchan, &head); - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - - vchan_dma_desc_free_list(&fsl_chan->vchan, &head); - dma_pool_destroy(fsl_chan->tcd_pool); - fsl_chan->tcd_pool = NULL; -} - static int fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) { @@ -995,17 +311,6 @@ static int fsl_edma_probe(struct platform_device *pdev) return 0; } -static void fsl_edma_cleanup_vchan(struct dma_device *dmadev) -{ - struct fsl_edma_chan *chan, *_chan; - - list_for_each_entry_safe(chan, _chan, - &dmadev->channels, vchan.chan.device_node) { - list_del(&chan->vchan.chan.device_node); - tasklet_kill(&chan->vchan.task); - } -} - static int fsl_edma_remove(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; |