diff options
Diffstat (limited to 'drivers/dma')
36 files changed, 2037 insertions, 656 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 5142da401db3..092483644315 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -616,8 +616,8 @@ config TXX9_DMAC integrated in chips such as the Toshiba TX4927/38/39. config TEGRA20_APB_DMA - bool "NVIDIA Tegra20 APB DMA support" - depends on ARCH_TEGRA + tristate "NVIDIA Tegra20 APB DMA support" + depends on ARCH_TEGRA || COMPILE_TEST select DMA_ENGINE help Support for the NVIDIA Tegra20 APB DMA controller driver. The @@ -658,6 +658,17 @@ config UNIPHIER_MDMAC UniPhier platform. This DMA controller is used as the external DMA engine of the SD/eMMC controllers of the LD4, Pro4, sLD8 SoCs. +config UNIPHIER_XDMAC + tristate "UniPhier XDMAC support" + depends on ARCH_UNIPHIER || COMPILE_TEST + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the XDMAC (external DMA controller) on the + UniPhier platform. This DMA controller can transfer data from + memory to memory, memory to peripheral and peripheral to memory. + config XGENE_DMA tristate "APM X-Gene DMA support" depends on ARCH_XGENE || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 1d908394fbea..e60f81331d4c 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -78,6 +78,7 @@ obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o obj-$(CONFIG_TIMB_DMA) += timb_dma.o obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o +obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 672c73b4a2d4..73a20780744b 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -146,17 +146,8 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) "scanned %u descriptors on freelist\n", i); /* no more descriptor available in initial pool: create one more */ - if (!ret) { - ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); - if (ret) { - spin_lock_irqsave(&atchan->lock, flags); - atchan->descs_allocated++; - spin_unlock_irqrestore(&atchan->lock, flags); - } else { - dev_err(chan2dev(&atchan->chan_common), - "not enough descriptors available\n"); - } - } + if (!ret) + ret = atc_alloc_descriptor(&atchan->chan_common, GFP_NOWAIT); return ret; } @@ -435,17 +426,19 @@ static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) * atc_chain_complete - finish work for one transaction chain * @atchan: channel we work on * @desc: descriptor at the head of the chain we want do complete - * - * Called with atchan->lock held and bh disabled */ + */ static void atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + unsigned long flags; dev_vdbg(chan2dev(&atchan->chan_common), "descriptor %u complete\n", txd->cookie); + spin_lock_irqsave(&atchan->lock, flags); + /* mark the descriptor as complete for non cyclic cases only */ if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); @@ -462,16 +455,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); + spin_unlock_irqrestore(&atchan->lock, flags); + dma_descriptor_unmap(txd); /* for cyclic transfers, * no need to replay callback function while stopping */ - if (!atc_chan_is_cyclic(atchan)) { - /* - * The API requires that no submissions are done from a - * callback, so we don't need to drop the lock here - */ + if (!atc_chan_is_cyclic(atchan)) dmaengine_desc_get_callback_invoke(txd, NULL); - } dma_run_dependencies(txd); } @@ -489,9 +479,12 @@ static void atc_complete_all(struct at_dma_chan *atchan) { struct at_desc *desc, *_desc; LIST_HEAD(list); + unsigned long flags; dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); + spin_lock_irqsave(&atchan->lock, flags); + /* * Submit queued descriptors ASAP, i.e. before we go through * the completed ones. @@ -503,6 +496,8 @@ static void atc_complete_all(struct at_dma_chan *atchan) /* empty queue list by moving descriptors (if any) to active_list */ list_splice_init(&atchan->queue, &atchan->active_list); + spin_unlock_irqrestore(&atchan->lock, flags); + list_for_each_entry_safe(desc, _desc, &list, desc_node) atc_chain_complete(atchan, desc); } @@ -510,38 +505,44 @@ static void atc_complete_all(struct at_dma_chan *atchan) /** * atc_advance_work - at the end of a transaction, move forward * @atchan: channel where the transaction ended - * - * Called with atchan->lock held and bh disabled */ static void atc_advance_work(struct at_dma_chan *atchan) { + unsigned long flags; + int ret; + dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); - if (atc_chan_is_enabled(atchan)) + spin_lock_irqsave(&atchan->lock, flags); + ret = atc_chan_is_enabled(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + if (ret) return; if (list_empty(&atchan->active_list) || - list_is_singular(&atchan->active_list)) { - atc_complete_all(atchan); - } else { - atc_chain_complete(atchan, atc_first_active(atchan)); - /* advance work */ - atc_dostart(atchan, atc_first_active(atchan)); - } + list_is_singular(&atchan->active_list)) + return atc_complete_all(atchan); + + atc_chain_complete(atchan, atc_first_active(atchan)); + + /* advance work */ + spin_lock_irqsave(&atchan->lock, flags); + atc_dostart(atchan, atc_first_active(atchan)); + spin_unlock_irqrestore(&atchan->lock, flags); } /** * atc_handle_error - handle errors reported by DMA controller * @atchan: channel where error occurs - * - * Called with atchan->lock held and bh disabled */ static void atc_handle_error(struct at_dma_chan *atchan) { struct at_desc *bad_desc; struct at_desc *child; + unsigned long flags; + spin_lock_irqsave(&atchan->lock, flags); /* * The descriptor currently at the head of the active list is * broked. Since we don't have any way to report errors, we'll @@ -573,6 +574,8 @@ static void atc_handle_error(struct at_dma_chan *atchan) list_for_each_entry(child, &bad_desc->tx_list, desc_node) atc_dump_lli(atchan, &child->lli); + spin_unlock_irqrestore(&atchan->lock, flags); + /* Pretend the descriptor completed successfully */ atc_chain_complete(atchan, bad_desc); } @@ -580,8 +583,6 @@ static void atc_handle_error(struct at_dma_chan *atchan) /** * atc_handle_cyclic - at the end of a period, run callback function * @atchan: channel used for cyclic operations - * - * Called with atchan->lock held and bh disabled */ static void atc_handle_cyclic(struct at_dma_chan *atchan) { @@ -600,17 +601,14 @@ static void atc_handle_cyclic(struct at_dma_chan *atchan) static void atc_tasklet(unsigned long data) { struct at_dma_chan *atchan = (struct at_dma_chan *)data; - unsigned long flags; - spin_lock_irqsave(&atchan->lock, flags); if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) - atc_handle_error(atchan); - else if (atc_chan_is_cyclic(atchan)) - atc_handle_cyclic(atchan); - else - atc_advance_work(atchan); + return atc_handle_error(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); + if (atc_chan_is_cyclic(atchan)) + return atc_handle_cyclic(atchan); + + atc_advance_work(atchan); } static irqreturn_t at_dma_interrupt(int irq, void *dev_id) @@ -940,7 +938,7 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, return NULL; } - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", __func__); @@ -998,7 +996,7 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, return NULL; } - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); if (!vaddr) { dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", __func__); @@ -1446,6 +1444,8 @@ static int atc_terminate_all(struct dma_chan *chan) list_splice_init(&atchan->queue, &list); list_splice_init(&atchan->active_list, &list); + spin_unlock_irqrestore(&atchan->lock, flags); + /* Flush all pending and queued descriptors */ list_for_each_entry_safe(desc, _desc, &list, desc_node) atc_chain_complete(atchan, desc); @@ -1454,8 +1454,6 @@ static int atc_terminate_all(struct dma_chan *chan) /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); - spin_unlock_irqrestore(&atchan->lock, flags); - return 0; } @@ -1516,7 +1514,6 @@ atc_tx_status(struct dma_chan *chan, static void atc_issue_pending(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); - unsigned long flags; dev_vdbg(chan2dev(chan), "issue_pending\n"); @@ -1524,15 +1521,12 @@ static void atc_issue_pending(struct dma_chan *chan) if (atc_chan_is_cyclic(atchan)) return; - spin_lock_irqsave(&atchan->lock, flags); atc_advance_work(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); } /** * atc_alloc_chan_resources - allocate resources for DMA channel * @chan: allocate descriptor resources for this channel - * @client: current client requesting the channel be ready for requests * * return - the number of allocated descriptors */ @@ -1542,10 +1536,8 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; struct at_dma_slave *atslave; - unsigned long flags; int i; u32 cfg; - LIST_HEAD(tmp_list); dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); @@ -1555,6 +1547,11 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) return -EIO; } + if (!list_empty(&atchan->free_list)) { + dev_dbg(chan2dev(chan), "can't allocate channel resources (channel not freed from a previous use)\n"); + return -EIO; + } + cfg = ATC_DEFAULT_CFG; atslave = chan->private; @@ -1570,11 +1567,6 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) cfg = atslave->cfg; } - /* have we already been set up? - * reconfigure channel but no need to reallocate descriptors */ - if (!list_empty(&atchan->free_list)) - return atchan->descs_allocated; - /* Allocate initial pool of descriptors */ for (i = 0; i < init_nr_desc_per_channel; i++) { desc = atc_alloc_descriptor(chan, GFP_KERNEL); @@ -1583,23 +1575,18 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) "Only %d initial descriptors\n", i); break; } - list_add_tail(&desc->desc_node, &tmp_list); + list_add_tail(&desc->desc_node, &atchan->free_list); } - spin_lock_irqsave(&atchan->lock, flags); - atchan->descs_allocated = i; - list_splice(&tmp_list, &atchan->free_list); dma_cookie_init(chan); - spin_unlock_irqrestore(&atchan->lock, flags); /* channel parameters */ channel_writel(atchan, CFG, cfg); dev_dbg(chan2dev(chan), - "alloc_chan_resources: allocated %d descriptors\n", - atchan->descs_allocated); + "alloc_chan_resources: allocated %d descriptors\n", i); - return atchan->descs_allocated; + return i; } /** @@ -1613,9 +1600,6 @@ static void atc_free_chan_resources(struct dma_chan *chan) struct at_desc *desc, *_desc; LIST_HEAD(list); - dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n", - atchan->descs_allocated); - /* ASSERT: channel is idle */ BUG_ON(!list_empty(&atchan->active_list)); BUG_ON(!list_empty(&atchan->queue)); @@ -1628,7 +1612,6 @@ static void atc_free_chan_resources(struct dma_chan *chan) dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys); } list_splice_init(&atchan->free_list, &list); - atchan->descs_allocated = 0; atchan->status = 0; /* @@ -1671,7 +1654,7 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - atslave = kzalloc(sizeof(*atslave), GFP_KERNEL); + atslave = kmalloc(sizeof(*atslave), GFP_KERNEL); if (!atslave) return NULL; diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index fe8a5853ec49..397692e937b3 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -243,7 +243,6 @@ enum atc_status { * @active_list: list of descriptors dmaengine is being running on * @queue: list of descriptors ready to be submitted to engine * @free_list: list of descriptors usable by the channel - * @descs_allocated: records the actual size of the descriptor pool */ struct at_dma_chan { struct dma_chan chan_common; @@ -264,7 +263,6 @@ struct at_dma_chan { struct list_head active_list; struct list_head queue; struct list_head free_list; - unsigned int descs_allocated; }; #define channel_readl(atchan, name) \ diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index f71c9f77d405..bb0eaf38b594 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1543,9 +1543,6 @@ static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan, static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) { struct at_xdmac_desc *desc; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); /* * If channel is enabled, do nothing, advance_work will be triggered @@ -1559,8 +1556,6 @@ static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) if (!desc->active_xfer) at_xdmac_start_xfer(atchan, desc); } - - spin_unlock_irqrestore(&atchan->lock, flags); } static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) @@ -1596,7 +1591,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) if (atchan->irq_status & AT_XDMAC_CIS_ROIS) dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); - spin_lock_bh(&atchan->lock); + spin_lock_irq(&atchan->lock); /* Channel must be disabled first as it's not done automatically */ at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); @@ -1607,7 +1602,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) struct at_xdmac_desc, xfer_node); - spin_unlock_bh(&atchan->lock); + spin_unlock_irq(&atchan->lock); /* Print bad descriptor's details if needed */ dev_dbg(chan2dev(&atchan->chan), @@ -1640,31 +1635,31 @@ static void at_xdmac_tasklet(unsigned long data) if (atchan->irq_status & error_mask) at_xdmac_handle_error(atchan); - spin_lock(&atchan->lock); + spin_lock_irq(&atchan->lock); desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); if (!desc->active_xfer) { dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); - spin_unlock(&atchan->lock); + spin_unlock_irq(&atchan->lock); return; } txd = &desc->tx_dma_desc; at_xdmac_remove_xfer(atchan, desc); - spin_unlock(&atchan->lock); + spin_unlock_irq(&atchan->lock); - if (!at_xdmac_chan_is_cyclic(atchan)) { - dma_cookie_complete(txd); - if (txd->flags & DMA_PREP_INTERRUPT) - dmaengine_desc_get_callback_invoke(txd, NULL); - } + dma_cookie_complete(txd); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); dma_run_dependencies(txd); + spin_lock_irq(&atchan->lock); at_xdmac_advance_work(atchan); + spin_unlock_irq(&atchan->lock); } } @@ -1725,11 +1720,15 @@ static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) static void at_xdmac_issue_pending(struct dma_chan *chan) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + unsigned long flags; dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); - if (!at_xdmac_chan_is_cyclic(atchan)) + if (!at_xdmac_chan_is_cyclic(atchan)) { + spin_lock_irqsave(&atchan->lock, flags); at_xdmac_advance_work(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + } return; } @@ -1822,26 +1821,21 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac_desc *desc; int i; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); if (at_xdmac_chan_is_enabled(atchan)) { dev_err(chan2dev(chan), "can't allocate channel resources (channel enabled)\n"); - i = -EIO; - goto spin_unlock; + return -EIO; } if (!list_empty(&atchan->free_descs_list)) { dev_err(chan2dev(chan), "can't allocate channel resources (channel not free from a previous use)\n"); - i = -EIO; - goto spin_unlock; + return -EIO; } for (i = 0; i < init_nr_desc_per_channel; i++) { - desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC); + desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); if (!desc) { dev_warn(chan2dev(chan), "only %d descriptors have been allocated\n", i); @@ -1854,8 +1848,6 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); -spin_unlock: - spin_unlock_irqrestore(&atchan->lock, flags); return i; } diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 275e90fa829d..64239da02e74 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -120,7 +120,7 @@ struct sba_request { struct brcm_message msg; struct dma_async_tx_descriptor tx; /* SBA commands */ - struct brcm_sba_command cmds[0]; + struct brcm_sba_command cmds[]; }; enum sba_version { diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index e51d836afcc7..1092d4ce723e 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -1947,8 +1947,6 @@ static void dma_tc_handle(struct coh901318_chan *cohc) return; } - spin_lock(&cohc->lock); - /* * When we reach this point, at least one queue item * should have been moved over from cohc->queue to @@ -1969,8 +1967,6 @@ static void dma_tc_handle(struct coh901318_chan *cohc) if (coh901318_queue_start(cohc) == NULL) cohc->busy = 0; - spin_unlock(&cohc->lock); - /* * This tasklet will remove items from cohc->active * and thus terminates them. diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index c3b1283b6d31..4830ba658ce1 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -58,6 +58,87 @@ static DEFINE_IDA(dma_ida); static LIST_HEAD(dma_device_list); static long dmaengine_ref_count; +/* --- debugfs implementation --- */ +#ifdef CONFIG_DEBUG_FS +#include <linux/debugfs.h> + +static struct dentry *rootdir; + +static void dmaengine_debug_register(struct dma_device *dma_dev) +{ + dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev), + rootdir); + if (IS_ERR(dma_dev->dbg_dev_root)) + dma_dev->dbg_dev_root = NULL; +} + +static void dmaengine_debug_unregister(struct dma_device *dma_dev) +{ + debugfs_remove_recursive(dma_dev->dbg_dev_root); + dma_dev->dbg_dev_root = NULL; +} + +static void dmaengine_dbg_summary_show(struct seq_file *s, + struct dma_device *dma_dev) +{ + struct dma_chan *chan; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) { + seq_printf(s, " %-13s| %s", dma_chan_name(chan), + chan->dbg_client_name ?: "in-use"); + + if (chan->router) + seq_printf(s, " (via router: %s)\n", + dev_name(chan->router->dev)); + else + seq_puts(s, "\n"); + } + } +} + +static int dmaengine_summary_show(struct seq_file *s, void *data) +{ + struct dma_device *dma_dev = NULL; + + mutex_lock(&dma_list_mutex); + list_for_each_entry(dma_dev, &dma_device_list, global_node) { + seq_printf(s, "dma%d (%s): number of channels: %u\n", + dma_dev->dev_id, dev_name(dma_dev->dev), + dma_dev->chancnt); + + if (dma_dev->dbg_summary_show) + dma_dev->dbg_summary_show(s, dma_dev); + else + dmaengine_dbg_summary_show(s, dma_dev); + + if (!list_is_last(&dma_dev->global_node, &dma_device_list)) + seq_puts(s, "\n"); + } + mutex_unlock(&dma_list_mutex); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dmaengine_summary); + +static void __init dmaengine_debugfs_init(void) +{ + rootdir = debugfs_create_dir("dmaengine", NULL); + + /* /sys/kernel/debug/dmaengine/summary */ + debugfs_create_file("summary", 0444, rootdir, NULL, + &dmaengine_summary_fops); +} +#else +static inline void dmaengine_debugfs_init(void) { } +static inline int dmaengine_debug_register(struct dma_device *dma_dev) +{ + return 0; +} + +static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { } +#endif /* DEBUG_FS */ + /* --- sysfs implementation --- */ #define DMA_SLAVE_NAME "slave" @@ -760,6 +841,11 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) return chan ? chan : ERR_PTR(-EPROBE_DEFER); found: +#ifdef CONFIG_DEBUG_FS + chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), + name); +#endif + chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); if (!chan->name) return chan; @@ -837,6 +923,11 @@ void dma_release_channel(struct dma_chan *chan) chan->name = NULL; chan->slave = NULL; } + +#ifdef CONFIG_DEBUG_FS + kfree(chan->dbg_client_name); + chan->dbg_client_name = NULL; +#endif mutex_unlock(&dma_list_mutex); } EXPORT_SYMBOL_GPL(dma_release_channel); @@ -1151,7 +1242,7 @@ int dma_async_device_register(struct dma_device *device) } if (!device->device_release) - dev_warn(device->dev, + dev_dbg(device->dev, "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n"); kref_init(&device->ref); @@ -1196,6 +1287,8 @@ int dma_async_device_register(struct dma_device *device) dma_channel_rebalance(); mutex_unlock(&dma_list_mutex); + dmaengine_debug_register(device); + return 0; err_out: @@ -1229,6 +1322,8 @@ void dma_async_device_unregister(struct dma_device *device) { struct dma_chan *chan, *n; + dmaengine_debug_unregister(device); + list_for_each_entry_safe(chan, n, &device->channels, device_node) __dma_async_device_channel_unregister(device, chan); @@ -1559,6 +1654,11 @@ static int __init dma_bus_init(void) if (err) return err; - return class_register(&dma_devclass); + + err = class_register(&dma_devclass); + if (!err) + dmaengine_debugfs_init(); + + return err; } arch_initcall(dma_bus_init); diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index e8a320c9e57c..1bfbd64b1371 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -182,4 +182,20 @@ dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb) struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); struct dma_chan *dma_get_any_slave_channel(struct dma_device *device); +#ifdef CONFIG_DEBUG_FS +#include <linux/debugfs.h> + +static inline struct dentry * +dmaengine_get_debugfs_root(struct dma_device *dma_dev) { + return dma_dev->dbg_dev_root; +} +#else +struct dentry; +static inline struct dentry * +dmaengine_get_debugfs_root(struct dma_device *dma_dev) +{ + return NULL; +} +#endif /* CONFIG_DEBUG_FS */ + #endif diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c index c70a7965f140..4ec909e0b810 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -790,6 +790,20 @@ static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) return 0; } +static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) +{ + struct dpaa2_qdma_priv *priv; + struct device *dev; + + dev = &ls_dev->dev; + priv = dev_get_drvdata(dev); + + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); + dpaa2_dpdmai_dpio_unbind(priv); + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); + dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); +} + static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { { .vendor = FSL_MC_VENDOR_FREESCALE, @@ -805,6 +819,7 @@ static struct fsl_mc_driver dpaa2_qdma_driver = { }, .probe = dpaa2_qdma_probe, .remove = dpaa2_qdma_remove, + .shutdown = dpaa2_qdma_shutdown, .match_id_table = dpaa2_qdma_id_table }; diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c index f8d22115154a..878662aaa1c2 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -160,6 +160,27 @@ int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, } /** + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPDMAI object + * + * Return: '0' on Success; error code otherwise. + */ +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct fsl_mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} +EXPORT_SYMBOL_GPL(dpdmai_destroy); + +/** * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h index 6d785093da8e..b13b9bf0c003 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -18,6 +18,7 @@ #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) #define DPDMAI_CMDID_OPEN DPDMAI_CMDID_FORMAT(0x80E) #define DPDMAI_CMDID_CREATE DPDMAI_CMDID_FORMAT(0x90E) +#define DPDMAI_CMDID_DESTROY DPDMAI_CMDID_FORMAT(0x900) #define DPDMAI_CMDID_ENABLE DPDMAI_CMDID_FORMAT(0x002) #define DPDMAI_CMDID_DISABLE DPDMAI_CMDID_FORMAT(0x003) @@ -160,6 +161,7 @@ struct dpdmai_rx_queue_attr { int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token); int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, const struct dpdmai_cfg *cfg, u16 *token); int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 1d7347825b95..ff49847e37a8 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -74,16 +74,14 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) struct idxd_device *idxd; struct idxd_wq *wq; struct device *dev; - struct idxd_cdev *idxd_cdev; wq = inode_wq(inode); idxd = wq->idxd; dev = &idxd->pdev->dev; - idxd_cdev = &wq->idxd_cdev; - dev_dbg(dev, "%s called\n", __func__); + dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); - if (idxd_wq_refcount(wq) > 1 && wq_dedicated(wq)) + if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) return -EBUSY; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -139,6 +137,8 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) dev_dbg(&pdev->dev, "%s called\n", __func__); rc = check_vma(wq, vma, __func__); + if (rc < 0) + return rc; vma->vm_flags |= VM_DONTCOPY; pfn = (base + idxd_get_wq_portal_full_offset(wq->id, @@ -204,6 +204,7 @@ static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); if (minor < 0) { rc = minor; + kfree(dev); goto ida_err; } @@ -212,7 +213,6 @@ static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) rc = device_register(dev); if (rc < 0) { dev_err(&idxd->pdev->dev, "device register failed\n"); - put_device(dev); goto dev_reg_err; } idxd_cdev->minor = minor; @@ -221,8 +221,8 @@ static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq) dev_reg_err: ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt)); + put_device(dev); ida_err: - kfree(dev); idxd_cdev->dev = NULL; return rc; } diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index ada69e722f84..f6f49f0f6fae 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -584,11 +584,11 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) struct idxd_group *group = &idxd->groups[i]; if (group->tc_a == -1) - group->grpcfg.flags.tc_a = 0; + group->tc_a = group->grpcfg.flags.tc_a = 0; else group->grpcfg.flags.tc_a = group->tc_a; if (group->tc_b == -1) - group->grpcfg.flags.tc_b = 1; + group->tc_b = group->grpcfg.flags.tc_b = 1; else group->grpcfg.flags.tc_b = group->tc_b; group->grpcfg.flags.use_token_limit = group->use_token_limit; diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 6d907fe150aa..3999827970ab 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -124,6 +124,7 @@ static int idxd_config_bus_probe(struct device *dev) rc = idxd_device_config(idxd); if (rc < 0) { spin_unlock_irqrestore(&idxd->dev_lock, flags); + module_put(THIS_MODULE); dev_warn(dev, "Device config failed: %d\n", rc); return rc; } @@ -132,6 +133,7 @@ static int idxd_config_bus_probe(struct device *dev) rc = idxd_device_enable(idxd); if (rc < 0) { spin_unlock_irqrestore(&idxd->dev_lock, flags); + module_put(THIS_MODULE); dev_warn(dev, "Device enable failed: %d\n", rc); return rc; } @@ -142,6 +144,7 @@ static int idxd_config_bus_probe(struct device *dev) rc = idxd_register_dma_device(idxd); if (rc < 0) { spin_unlock_irqrestore(&idxd->dev_lock, flags); + module_put(THIS_MODULE); dev_dbg(dev, "Failed to register dmaengine device\n"); return rc; } @@ -416,7 +419,7 @@ static ssize_t engine_group_id_store(struct device *dev, struct idxd_device *idxd = engine->idxd; long id; int rc; - struct idxd_group *prevg, *group; + struct idxd_group *prevg; rc = kstrtol(buf, 10, &id); if (rc < 0) @@ -436,7 +439,6 @@ static ssize_t engine_group_id_store(struct device *dev, return count; } - group = &idxd->groups[id]; prevg = engine->group; if (prevg) @@ -510,13 +512,10 @@ static ssize_t group_tokens_reserved_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (idxd->token_limit == 0) - return -EPERM; - if (val > idxd->max_tokens) return -EINVAL; - if (val > idxd->nr_tokens) + if (val > idxd->nr_tokens + group->tokens_reserved) return -EINVAL; group->tokens_reserved = val; @@ -558,8 +557,6 @@ static ssize_t group_tokens_allowed_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (idxd->token_limit == 0) - return -EPERM; if (val < 4 * group->num_engines || val > group->tokens_reserved + idxd->nr_tokens) return -EINVAL; @@ -901,6 +898,20 @@ static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, return sprintf(buf, "%u\n", wq->size); } +static int total_claimed_wq_size(struct idxd_device *idxd) +{ + int i; + int wq_size = 0; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = &idxd->wqs[i]; + + wq_size += wq->size; + } + + return wq_size; +} + static ssize_t wq_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -920,7 +931,7 @@ static ssize_t wq_size_store(struct device *dev, if (wq->state != IDXD_WQ_DISABLED) return -EPERM; - if (size > idxd->max_wq_size) + if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size) return -EINVAL; wq->size = size; @@ -999,12 +1010,14 @@ static ssize_t wq_type_store(struct device *dev, return -EPERM; old_type = wq->type; - if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL])) + if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE])) + wq->type = IDXD_WQT_NONE; + else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL])) wq->type = IDXD_WQT_KERNEL; else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER])) wq->type = IDXD_WQT_USER; else - wq->type = IDXD_WQT_NONE; + return -EINVAL; /* If we are changing queue type, clear the name */ if (wq->type != old_type) @@ -1161,6 +1174,16 @@ static ssize_t op_cap_show(struct device *dev, } static DEVICE_ATTR_RO(op_cap); +static ssize_t gen_cap_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = + container_of(dev, struct idxd_device, conf_dev); + + return sprintf(buf, "%#llx\n", idxd->hw.gen_cap.bits); +} +static DEVICE_ATTR_RO(gen_cap); + static ssize_t configurable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1298,6 +1321,7 @@ static struct attribute *idxd_device_attributes[] = { &dev_attr_max_batch_size.attr, &dev_attr_max_transfer_size.attr, &dev_attr_op_cap.attr, + &dev_attr_gen_cap.attr, &dev_attr_configurable.attr, &dev_attr_clients.attr, &dev_attr_state.attr, diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 066b21a32232..4d4477df4ede 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1331,13 +1331,14 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_channel_synchronize(chan); - if (sdmac->event_id0) + if (sdmac->event_id0 >= 0) sdma_event_disable(sdmac, sdmac->event_id0); if (sdmac->event_id1) sdma_event_disable(sdmac, sdmac->event_id1); sdmac->event_id0 = 0; sdmac->event_id1 = 0; + sdmac->context_loaded = false; sdma_set_channel_priority(sdmac, 0); @@ -1631,7 +1632,7 @@ static int sdma_config(struct dma_chan *chan, memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); /* Set ENBLn earlier to make sure dma request triggered after that */ - if (sdmac->event_id0) { + if (sdmac->event_id0 >= 0) { if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) return -EINVAL; sdma_event_enable(sdmac, sdmac->event_id0); diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index be61c32a876f..0be385587c4c 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -102,7 +102,7 @@ struct ioat_dca_priv { int max_requesters; int requester_count; u8 tag_map[IOAT_TAG_MAP_LEN]; - struct ioat_dca_slot req_slots[0]; + struct ioat_dca_slot req_slots[]; }; static int ioat_dca_dev_managed(struct dca_provider *dca, diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index fbabd2e88a18..4db000d5f01c 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -4303,7 +4303,7 @@ static ssize_t devices_show(struct device_driver *dev, char *buf) for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { if (ppc440spe_adma_devices[i] == -1) continue; - size += snprintf(buf + size, PAGE_SIZE - size, + size += scnprintf(buf + size, PAGE_SIZE - size, "PPC440SP(E)-ADMA.%d: %s\n", i, ppc_adma_errors[ppc440spe_adma_devices[i]]); } diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index afb68055ed1b..0fa7f14a65a1 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -78,7 +78,7 @@ struct sa11x0_dma_desc { bool cyclic; unsigned sglen; - struct sa11x0_dma_sg sg[0]; + struct sa11x0_dma_sg sg[]; }; struct sa11x0_dma_phy; diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index f06016d38a05..59b36ab5d684 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1219,7 +1219,7 @@ rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, sg_len = buf_len / period_len; if (sg_len > RCAR_DMAC_MAX_SG_LEN) { dev_err(chan->device->dev, - "chan%u: sg length %d exceds limit %d", + "chan%u: sg length %d exceeds limit %d", rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); return NULL; } diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index c51de498b5b4..2deeaab078a4 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -709,7 +709,7 @@ static struct dma_async_tx_descriptor *shdma_prep_dma_cyclic( BUG_ON(!schan->desc_num); if (sg_len > SHDMA_MAX_SG_LEN) { - dev_err(schan->dev, "sg length %d exceds limit %d", + dev_err(schan->dev, "sg length %d exceeds limit %d", sg_len, SHDMA_MAX_SG_LEN); return NULL; } diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 9a31a315dbef..0ef5ca81ba4d 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -212,7 +212,7 @@ struct sprd_dma_dev { struct clk *ashb_clk; int irq; u32 total_chns; - struct sprd_dma_chn channels[0]; + struct sprd_dma_chn channels[]; }; static void sprd_dma_free_desc(struct virt_dma_desc *vd); @@ -486,6 +486,28 @@ static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan) return 0; } +static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable) +{ + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); + u32 reg, val, req_id; + + if (schan->dev_id == SPRD_DMA_SOFTWARE_UID) + return; + + /* The DMA request id always starts from 0. */ + req_id = schan->dev_id - 1; + + if (req_id < 32) { + reg = SPRD_DMA_GLB_REQ_PEND0_EN; + val = BIT(req_id); + } else { + reg = SPRD_DMA_GLB_REQ_PEND1_EN; + val = BIT(req_id - 32); + } + + sprd_dma_glb_update(sdev, reg, val, enable ? val : 0); +} + static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, struct sprd_dma_desc *sdesc) { @@ -532,6 +554,7 @@ static void sprd_dma_start(struct sprd_dma_chn *schan) */ sprd_dma_set_chn_config(schan, schan->cur_desc); sprd_dma_set_uid(schan); + sprd_dma_set_pending(schan, true); sprd_dma_enable_chn(schan); if (schan->dev_id == SPRD_DMA_SOFTWARE_UID && @@ -543,6 +566,7 @@ static void sprd_dma_start(struct sprd_dma_chn *schan) static void sprd_dma_stop(struct sprd_dma_chn *schan) { sprd_dma_stop_and_disable(schan); + sprd_dma_set_pending(schan, false); sprd_dma_unset_uid(schan); sprd_dma_clear_int(schan); schan->cur_desc = NULL; diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 5989b0893521..0ddbaa4b4f0b 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -15,6 +15,7 @@ #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/init.h> +#include <linux/iopoll.h> #include <linux/jiffies.h> #include <linux/list.h> #include <linux/module.h> @@ -207,7 +208,6 @@ struct stm32_dma_device { struct dma_device ddev; void __iomem *base; struct clk *clk; - struct reset_control *rst; bool mem2mem; struct stm32_dma_chan chan[STM32_DMA_MAX_CHANNELS]; }; @@ -422,29 +422,19 @@ static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags) static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) { struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); - unsigned long timeout = jiffies + msecs_to_jiffies(5000); - u32 dma_scr, id; + u32 dma_scr, id, reg; id = chan->id; - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + reg = STM32_DMA_SCR(id); + dma_scr = stm32_dma_read(dmadev, reg); if (dma_scr & STM32_DMA_SCR_EN) { dma_scr &= ~STM32_DMA_SCR_EN; - stm32_dma_write(dmadev, STM32_DMA_SCR(id), dma_scr); - - do { - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); - dma_scr &= STM32_DMA_SCR_EN; - if (!dma_scr) - break; - - if (time_after_eq(jiffies, timeout)) { - dev_err(chan2dev(chan), "%s: timeout!\n", - __func__); - return -EBUSY; - } - cond_resched(); - } while (1); + stm32_dma_write(dmadev, reg, dma_scr); + + return readl_relaxed_poll_timeout_atomic(dmadev->base + reg, + dma_scr, !(dma_scr & STM32_DMA_SCR_EN), + 10, 1000000); } return 0; @@ -488,8 +478,10 @@ static int stm32_dma_terminate_all(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); - if (chan->busy) { - stm32_dma_stop(chan); + if (chan->desc) { + vchan_terminate_vdesc(&chan->desc->vdesc); + if (chan->busy) + stm32_dma_stop(chan); chan->desc = NULL; } @@ -545,6 +537,8 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) if (!vdesc) return; + list_del(&vdesc->node); + chan->desc = to_stm32_dma_desc(vdesc); chan->next_sg = 0; } @@ -555,6 +549,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) sg_req = &chan->desc->sg_req[chan->next_sg]; reg = &sg_req->chan_reg; + reg->dma_scr &= ~STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar); @@ -622,7 +617,6 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan) } else { chan->busy = false; if (chan->next_sg == chan->desc->num_sgs) { - list_del(&chan->desc->vdesc.node); vchan_cookie_complete(&chan->desc->vdesc); chan->desc = NULL; } @@ -1275,6 +1269,7 @@ static int stm32_dma_probe(struct platform_device *pdev) struct dma_device *dd; const struct of_device_id *match; struct resource *res; + struct reset_control *rst; int i, ret; match = of_match_device(stm32_dma_of_match, &pdev->dev); @@ -1296,8 +1291,10 @@ static int stm32_dma_probe(struct platform_device *pdev) dmadev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(dmadev->clk)) { - dev_err(&pdev->dev, "Error: Missing controller clock\n"); - return PTR_ERR(dmadev->clk); + ret = PTR_ERR(dmadev->clk); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Can't get clock\n"); + return ret; } ret = clk_prepare_enable(dmadev->clk); @@ -1309,13 +1306,19 @@ static int stm32_dma_probe(struct platform_device *pdev) dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, "st,mem2mem"); - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(dmadev->rst)) { - reset_control_assert(dmadev->rst); + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto clk_free; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(dmadev->rst); + reset_control_deassert(rst); } + dma_set_max_seg_size(&pdev->dev, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); + dma_cap_set(DMA_SLAVE, dd->cap_mask); dma_cap_set(DMA_PRIVATE, dd->cap_mask); dma_cap_set(DMA_CYCLIC, dd->cap_mask); @@ -1336,7 +1339,9 @@ static int stm32_dma_probe(struct platform_device *pdev) BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dd->copy_align = DMAENGINE_ALIGN_32_BYTES; dd->max_burst = STM32_DMA_MAX_BURST; + dd->descriptor_reuse = true; dd->dev = &pdev->dev; INIT_LIST_HEAD(&dd->channels); @@ -1427,7 +1432,39 @@ static int stm32_dma_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_dma_suspend(struct device *dev) +{ + struct stm32_dma_device *dmadev = dev_get_drvdata(dev); + int id, ret, scr; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (id = 0; id < STM32_DMA_MAX_CHANNELS; id++) { + scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + if (scr & STM32_DMA_SCR_EN) { + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); + return -EBUSY; + } + } + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_dma_resume(struct device *dev) +{ + return pm_runtime_force_resume(dev); +} +#endif + static const struct dev_pm_ops stm32_dma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_suspend, stm32_dma_resume) SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend, stm32_dma_runtime_resume, NULL) }; @@ -1438,10 +1475,11 @@ static struct platform_driver stm32_dma_driver = { .of_match_table = stm32_dma_of_match, .pm = &stm32_dma_pm_ops, }, + .probe = stm32_dma_probe, }; static int __init stm32_dma_init(void) { - return platform_driver_probe(&stm32_dma_driver, stm32_dma_probe); + return platform_driver_register(&stm32_dma_driver); } subsys_initcall(stm32_dma_init); diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index 3c89bd39e096..12f7637e13a1 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -35,12 +35,14 @@ struct stm32_dmamux { struct stm32_dmamux_data { struct dma_router dmarouter; struct clk *clk; - struct reset_control *rst; void __iomem *iomem; u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ spinlock_t lock; /* Protects register access */ unsigned long *dma_inuse; /* Used DMA channel */ + u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register + * in suspend + */ u32 dma_reqs[]; /* Number of DMA Request per DMA masters. * [0] holds number of DMA Masters. * To be kept at very end end of this structure @@ -179,6 +181,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) struct stm32_dmamux_data *stm32_dmamux; struct resource *res; void __iomem *iomem; + struct reset_control *rst; int i, count, ret; u32 dma_req; @@ -251,16 +254,26 @@ static int stm32_dmamux_probe(struct platform_device *pdev) stm32_dmamux->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(stm32_dmamux->clk)) { ret = PTR_ERR(stm32_dmamux->clk); - if (ret == -EPROBE_DEFER) - dev_info(&pdev->dev, "Missing controller clock\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Missing clock controller\n"); + return ret; + } + + ret = clk_prepare_enable(stm32_dmamux->clk); + if (ret < 0) { + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); return ret; } - stm32_dmamux->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(stm32_dmamux->rst)) { - reset_control_assert(stm32_dmamux->rst); + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_clk; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(stm32_dmamux->rst); + reset_control_deassert(rst); } stm32_dmamux->iomem = iomem; @@ -271,14 +284,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - if (!IS_ERR(stm32_dmamux->clk)) { - ret = clk_prepare_enable(stm32_dmamux->clk); - if (ret < 0) { - dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); - return ret; - } - } - pm_runtime_get_noresume(&pdev->dev); /* Reset the dmamux */ @@ -287,8 +292,17 @@ static int stm32_dmamux_probe(struct platform_device *pdev) pm_runtime_put(&pdev->dev); - return of_dma_router_register(node, stm32_dmamux_route_allocate, + ret = of_dma_router_register(node, stm32_dmamux_route_allocate, &stm32_dmamux->dmarouter); + if (ret) + goto err_clk; + + return 0; + +err_clk: + clk_disable_unprepare(stm32_dmamux->clk); + + return ret; } #ifdef CONFIG_PM @@ -318,7 +332,54 @@ static int stm32_dmamux_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_dmamux_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + int i, ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (i = 0; i < stm32_dmamux->dma_requests; i++) + stm32_dmamux->ccr[i] = stm32_dmamux_read(stm32_dmamux->iomem, + STM32_DMAMUX_CCR(i)); + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_dmamux_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); + int i, ret; + + ret = pm_runtime_force_resume(dev); + if (ret < 0) + return ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (i = 0; i < stm32_dmamux->dma_requests; i++) + stm32_dmamux_write(stm32_dmamux->iomem, STM32_DMAMUX_CCR(i), + stm32_dmamux->ccr[i]); + + pm_runtime_put_sync(dev); + + return 0; +} +#endif + static const struct dev_pm_ops stm32_dmamux_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_dmamux_suspend, stm32_dmamux_resume) SET_RUNTIME_PM_OPS(stm32_dmamux_runtime_suspend, stm32_dmamux_runtime_resume, NULL) }; diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index 5838311cf990..5469563703d1 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -273,7 +273,6 @@ struct stm32_mdma_device { void __iomem *base; struct clk *clk; int irq; - struct reset_control *rst; u32 nr_channels; u32 nr_requests; u32 nr_ahb_addr_masks; @@ -1127,6 +1126,8 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) return; } + list_del(&vdesc->node); + chan->desc = to_stm32_mdma_desc(vdesc); hwdesc = chan->desc->node[0].hwdesc; chan->curr_hwdesc = 0; @@ -1242,8 +1243,10 @@ static int stm32_mdma_terminate_all(struct dma_chan *c) LIST_HEAD(head); spin_lock_irqsave(&chan->vchan.lock, flags); - if (chan->busy) { - stm32_mdma_stop(chan); + if (chan->desc) { + vchan_terminate_vdesc(&chan->desc->vdesc); + if (chan->busy) + stm32_mdma_stop(chan); chan->desc = NULL; } vchan_get_all_descriptors(&chan->vchan, &head); @@ -1331,7 +1334,6 @@ static enum dma_status stm32_mdma_tx_status(struct dma_chan *c, static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) { - list_del(&chan->desc->vdesc.node); vchan_cookie_complete(&chan->desc->vdesc); chan->desc = NULL; chan->busy = false; @@ -1532,6 +1534,7 @@ static int stm32_mdma_probe(struct platform_device *pdev) struct dma_device *dd; struct device_node *of_node; struct resource *res; + struct reset_control *rst; u32 nr_channels, nr_requests; int i, count, ret; @@ -1579,8 +1582,8 @@ static int stm32_mdma_probe(struct platform_device *pdev) dmadev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(dmadev->clk)) { ret = PTR_ERR(dmadev->clk); - if (ret == -EPROBE_DEFER) - dev_info(&pdev->dev, "Missing controller clock\n"); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Missing clock controller\n"); return ret; } @@ -1590,11 +1593,15 @@ static int stm32_mdma_probe(struct platform_device *pdev) return ret; } - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); - if (!IS_ERR(dmadev->rst)) { - reset_control_assert(dmadev->rst); + rst = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(rst)) { + ret = PTR_ERR(rst); + if (ret == -EPROBE_DEFER) + goto err_clk; + } else { + reset_control_assert(rst); udelay(2); - reset_control_deassert(dmadev->rst); + reset_control_deassert(rst); } dd = &dmadev->ddev; @@ -1614,6 +1621,8 @@ static int stm32_mdma_probe(struct platform_device *pdev) dd->device_resume = stm32_mdma_resume; dd->device_terminate_all = stm32_mdma_terminate_all; dd->device_synchronize = stm32_mdma_synchronize; + dd->descriptor_reuse = true; + dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | @@ -1637,25 +1646,27 @@ static int stm32_mdma_probe(struct platform_device *pdev) } dmadev->irq = platform_get_irq(pdev, 0); - if (dmadev->irq < 0) - return dmadev->irq; + if (dmadev->irq < 0) { + ret = dmadev->irq; + goto err_clk; + } ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler, 0, dev_name(&pdev->dev), dmadev); if (ret) { dev_err(&pdev->dev, "failed to request IRQ\n"); - return ret; + goto err_clk; } ret = dmaenginem_async_device_register(dd); if (ret) - return ret; + goto err_clk; ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev); if (ret < 0) { dev_err(&pdev->dev, "STM32 MDMA DMA OF registration failed %d\n", ret); - goto err_unregister; + goto err_clk; } platform_set_drvdata(pdev, dmadev); @@ -1668,7 +1679,9 @@ static int stm32_mdma_probe(struct platform_device *pdev) return 0; -err_unregister: +err_clk: + clk_disable_unprepare(dmadev->clk); + return ret; } @@ -1697,7 +1710,40 @@ static int stm32_mdma_runtime_resume(struct device *dev) } #endif +#ifdef CONFIG_PM_SLEEP +static int stm32_mdma_pm_suspend(struct device *dev) +{ + struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); + u32 ccr, id; + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) + return ret; + + for (id = 0; id < dmadev->nr_channels; id++) { + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); + if (ccr & STM32_MDMA_CCR_EN) { + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); + return -EBUSY; + } + } + + pm_runtime_put_sync(dev); + + pm_runtime_force_suspend(dev); + + return 0; +} + +static int stm32_mdma_pm_resume(struct device *dev) +{ + return pm_runtime_force_resume(dev); +} +#endif + static const struct dev_pm_ops stm32_mdma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume) SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend, stm32_mdma_runtime_resume, NULL) }; diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index bbc2bda3b902..e7ff09a5031d 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -697,11 +697,13 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, dest = sconfig->dst_addr; endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | - SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type); + SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); } else { src = sconfig->src_addr; dest = buf; endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); } diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 3a45079d11ec..265303a396ca 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -24,6 +24,7 @@ #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> +#include <linux/wait.h> #include "dmaengine.h" @@ -59,7 +60,7 @@ #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC #define TEGRA_APBDMA_CHAN_CSRE 0x00C -#define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31) +#define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31) /* AHB memory address */ #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 @@ -120,21 +121,21 @@ struct tegra_dma; * @support_separate_wcount_reg: Support separate word count register. */ struct tegra_dma_chip_data { - int nr_channels; - int channel_reg_size; - int max_dma_count; + unsigned int nr_channels; + unsigned int channel_reg_size; + unsigned int max_dma_count; bool support_channel_pause; bool support_separate_wcount_reg; }; /* DMA channel registers */ struct tegra_dma_channel_regs { - unsigned long csr; - unsigned long ahb_ptr; - unsigned long apb_ptr; - unsigned long ahb_seq; - unsigned long apb_seq; - unsigned long wcount; + u32 csr; + u32 ahb_ptr; + u32 apb_ptr; + u32 ahb_seq; + u32 apb_seq; + u32 wcount; }; /* @@ -168,7 +169,7 @@ struct tegra_dma_desc { struct list_head node; struct list_head tx_list; struct list_head cb_node; - int cb_count; + unsigned int cb_count; }; struct tegra_dma_channel; @@ -181,8 +182,7 @@ struct tegra_dma_channel { struct dma_chan dma_chan; char name[12]; bool config_init; - int id; - int irq; + unsigned int id; void __iomem *chan_addr; spinlock_t lock; bool busy; @@ -202,7 +202,9 @@ struct tegra_dma_channel { /* Channel-slave specific configuration */ unsigned int slave_id; struct dma_slave_config dma_sconfig; - struct tegra_dma_channel_regs channel_reg; + struct tegra_dma_channel_regs channel_reg; + + struct wait_queue_head wq; }; /* tegra_dma: Tegra DMA specific information */ @@ -222,9 +224,6 @@ struct tegra_dma { */ u32 global_pause_count; - /* Some register need to be cache before suspend */ - u32 reg_gen; - /* Last member of the structure */ struct tegra_dma_channel channels[0]; }; @@ -240,7 +239,7 @@ static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) } static inline void tdc_write(struct tegra_dma_channel *tdc, - u32 reg, u32 val) + u32 reg, u32 val) { writel(val, tdc->chan_addr + reg); } @@ -255,8 +254,8 @@ static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) return container_of(dc, struct tegra_dma_channel, dma_chan); } -static inline struct tegra_dma_desc *txd_to_tegra_dma_desc( - struct dma_async_tx_descriptor *td) +static inline struct tegra_dma_desc * +txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) { return container_of(td, struct tegra_dma_desc, txd); } @@ -267,12 +266,9 @@ static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) } static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); -static int tegra_dma_runtime_suspend(struct device *dev); -static int tegra_dma_runtime_resume(struct device *dev); /* Get DMA desc from free list, if not there then allocate it. */ -static struct tegra_dma_desc *tegra_dma_desc_get( - struct tegra_dma_channel *tdc) +static struct tegra_dma_desc *tegra_dma_desc_get(struct tegra_dma_channel *tdc) { struct tegra_dma_desc *dma_desc; unsigned long flags; @@ -281,7 +277,7 @@ static struct tegra_dma_desc *tegra_dma_desc_get( /* Do not allocate if desc are waiting for ack */ list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { - if (async_tx_test_ack(&dma_desc->txd)) { + if (async_tx_test_ack(&dma_desc->txd) && !dma_desc->cb_count) { list_del(&dma_desc->node); spin_unlock_irqrestore(&tdc->lock, flags); dma_desc->txd.flags = 0; @@ -299,11 +295,12 @@ static struct tegra_dma_desc *tegra_dma_desc_get( dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); dma_desc->txd.tx_submit = tegra_dma_tx_submit; dma_desc->txd.flags = 0; + return dma_desc; } static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, - struct tegra_dma_desc *dma_desc) + struct tegra_dma_desc *dma_desc) { unsigned long flags; @@ -314,29 +311,29 @@ static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, spin_unlock_irqrestore(&tdc->lock, flags); } -static struct tegra_dma_sg_req *tegra_dma_sg_req_get( - struct tegra_dma_channel *tdc) +static struct tegra_dma_sg_req * +tegra_dma_sg_req_get(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *sg_req = NULL; + struct tegra_dma_sg_req *sg_req; unsigned long flags; spin_lock_irqsave(&tdc->lock, flags); if (!list_empty(&tdc->free_sg_req)) { - sg_req = list_first_entry(&tdc->free_sg_req, - typeof(*sg_req), node); + sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), + node); list_del(&sg_req->node); spin_unlock_irqrestore(&tdc->lock, flags); return sg_req; } spin_unlock_irqrestore(&tdc->lock, flags); - sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT); + sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT); return sg_req; } static int tegra_dma_slave_config(struct dma_chan *dc, - struct dma_slave_config *sconfig) + struct dma_slave_config *sconfig) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); @@ -353,11 +350,12 @@ static int tegra_dma_slave_config(struct dma_chan *dc, tdc->slave_id = sconfig->slave_id; } tdc->config_init = true; + return 0; } static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, - bool wait_for_burst_complete) + bool wait_for_burst_complete) { struct tegra_dma *tdma = tdc->tdma; @@ -392,13 +390,13 @@ out: } static void tegra_dma_pause(struct tegra_dma_channel *tdc, - bool wait_for_burst_complete) + bool wait_for_burst_complete) { struct tegra_dma *tdma = tdc->tdma; if (tdma->chip_data->support_channel_pause) { tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, - TEGRA_APBDMA_CHAN_CSRE_PAUSE); + TEGRA_APBDMA_CHAN_CSRE_PAUSE); if (wait_for_burst_complete) udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); } else { @@ -410,17 +408,15 @@ static void tegra_dma_resume(struct tegra_dma_channel *tdc) { struct tegra_dma *tdma = tdc->tdma; - if (tdma->chip_data->support_channel_pause) { + if (tdma->chip_data->support_channel_pause) tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); - } else { + else tegra_dma_global_resume(tdc); - } } static void tegra_dma_stop(struct tegra_dma_channel *tdc) { - u32 csr; - u32 status; + u32 csr, status; /* Disable interrupts */ csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); @@ -441,7 +437,7 @@ static void tegra_dma_stop(struct tegra_dma_channel *tdc) } static void tegra_dma_start(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *sg_req) + struct tegra_dma_sg_req *sg_req) { struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; @@ -455,11 +451,11 @@ static void tegra_dma_start(struct tegra_dma_channel *tdc, /* Start DMA */ tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - ch_regs->csr | TEGRA_APBDMA_CSR_ENB); + ch_regs->csr | TEGRA_APBDMA_CSR_ENB); } static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *nsg_req) + struct tegra_dma_sg_req *nsg_req) { unsigned long status; @@ -493,9 +489,9 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); if (tdc->tdma->chip_data->support_separate_wcount_reg) tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, - nsg_req->ch_regs.wcount); + nsg_req->ch_regs.wcount); tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); + nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); nsg_req->configured = true; nsg_req->words_xferred = 0; @@ -506,11 +502,7 @@ static void tdc_start_head_req(struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *sg_req; - if (list_empty(&tdc->pending_sg_req)) - return; - - sg_req = list_first_entry(&tdc->pending_sg_req, - typeof(*sg_req), node); + sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); tegra_dma_start(tdc, sg_req); sg_req->configured = true; sg_req->words_xferred = 0; @@ -519,34 +511,32 @@ static void tdc_start_head_req(struct tegra_dma_channel *tdc) static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *hsgreq; - struct tegra_dma_sg_req *hnsgreq; - - if (list_empty(&tdc->pending_sg_req)) - return; + struct tegra_dma_sg_req *hsgreq, *hnsgreq; hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { - hnsgreq = list_first_entry(&hsgreq->node, - typeof(*hnsgreq), node); + hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), + node); tegra_dma_configure_for_next(tdc, hnsgreq); } } -static inline int get_current_xferred_count(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *sg_req, unsigned long status) +static inline unsigned int +get_current_xferred_count(struct tegra_dma_channel *tdc, + struct tegra_dma_sg_req *sg_req, + unsigned long status) { return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; } static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; while (!list_empty(&tdc->pending_sg_req)) { - sgreq = list_first_entry(&tdc->pending_sg_req, - typeof(*sgreq), node); + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), + node); list_move_tail(&sgreq->node, &tdc->free_sg_req); if (sgreq->last_sg) { dma_desc = sgreq->dma_desc; @@ -556,7 +546,7 @@ static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) /* Add in cb list if it is not there. */ if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, - &tdc->cb_desc); + &tdc->cb_desc); dma_desc->cb_count++; } } @@ -564,15 +554,9 @@ static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) } static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, - struct tegra_dma_sg_req *last_sg_req, bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *hsgreq = NULL; - - if (list_empty(&tdc->pending_sg_req)) { - dev_err(tdc2dev(tdc), "DMA is running without req\n"); - tegra_dma_stop(tdc); - return false; - } + struct tegra_dma_sg_req *hsgreq; /* * Check that head req on list should be in flight. @@ -582,7 +566,8 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!hsgreq->configured) { tegra_dma_stop(tdc); - dev_err(tdc2dev(tdc), "Error in DMA transfer, aborting DMA\n"); + pm_runtime_put(tdc->tdma->dev); + dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n"); tegra_dma_abort_all(tdc); return false; } @@ -590,14 +575,15 @@ static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, /* Configure next request */ if (!to_terminate) tdc_configure_next_head_desc(tdc); + return true; } static void handle_once_dma_done(struct tegra_dma_channel *tdc, - bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; tdc->busy = false; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); @@ -616,17 +602,22 @@ static void handle_once_dma_done(struct tegra_dma_channel *tdc, list_add_tail(&sgreq->node, &tdc->free_sg_req); /* Do not start DMA if it is going to be terminate */ - if (to_terminate || list_empty(&tdc->pending_sg_req)) + if (to_terminate) return; + if (list_empty(&tdc->pending_sg_req)) { + pm_runtime_put(tdc->tdma->dev); + return; + } + tdc_start_head_req(tdc); } static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, - bool to_terminate) + bool to_terminate) { - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; bool st; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); @@ -647,7 +638,7 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { list_move_tail(&sgreq->node, &tdc->pending_sg_req); sgreq->configured = false; - st = handle_continuous_head_request(tdc, sgreq, to_terminate); + st = handle_continuous_head_request(tdc, to_terminate); if (!st) dma_desc->dma_status = DMA_ERROR; } @@ -658,13 +649,13 @@ static void tegra_dma_tasklet(unsigned long data) struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; struct dmaengine_desc_callback cb; struct tegra_dma_desc *dma_desc; + unsigned int cb_count; unsigned long flags; - int cb_count; spin_lock_irqsave(&tdc->lock, flags); while (!list_empty(&tdc->cb_desc)) { - dma_desc = list_first_entry(&tdc->cb_desc, - typeof(*dma_desc), cb_node); + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), + cb_node); list_del(&dma_desc->cb_node); dmaengine_desc_get_callback(&dma_desc->txd, &cb); cb_count = dma_desc->cb_count; @@ -682,10 +673,9 @@ static void tegra_dma_tasklet(unsigned long data) static irqreturn_t tegra_dma_isr(int irq, void *dev_id) { struct tegra_dma_channel *tdc = dev_id; - unsigned long status; - unsigned long flags; + u32 status; - spin_lock_irqsave(&tdc->lock, flags); + spin_lock(&tdc->lock); trace_tegra_dma_isr(&tdc->dma_chan, irq); status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); @@ -693,13 +683,15 @@ static irqreturn_t tegra_dma_isr(int irq, void *dev_id) tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); tdc->isr_handler(tdc, false); tasklet_schedule(&tdc->tasklet); - spin_unlock_irqrestore(&tdc->lock, flags); + wake_up_all(&tdc->wq); + spin_unlock(&tdc->lock); return IRQ_HANDLED; } - spin_unlock_irqrestore(&tdc->lock, flags); - dev_info(tdc2dev(tdc), - "Interrupt already served status 0x%08lx\n", status); + spin_unlock(&tdc->lock); + dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n", + status); + return IRQ_NONE; } @@ -715,6 +707,7 @@ static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) cookie = dma_cookie_assign(&dma_desc->txd); list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); spin_unlock_irqrestore(&tdc->lock, flags); + return cookie; } @@ -722,6 +715,7 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); unsigned long flags; + int err; spin_lock_irqsave(&tdc->lock, flags); if (list_empty(&tdc->pending_sg_req)) { @@ -729,6 +723,12 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) goto end; } if (!tdc->busy) { + err = pm_runtime_get_sync(tdc->tdma->dev); + if (err < 0) { + dev_err(tdc2dev(tdc), "Failed to enable DMA\n"); + goto end; + } + tdc_start_head_req(tdc); /* Continuous single mode: Configure next req */ @@ -748,18 +748,13 @@ end: static int tegra_dma_terminate_all(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sgreq; unsigned long flags; - unsigned long status; - unsigned long wcount; + u32 status, wcount; bool was_busy; spin_lock_irqsave(&tdc->lock, flags); - if (list_empty(&tdc->pending_sg_req)) { - spin_unlock_irqrestore(&tdc->lock, flags); - return 0; - } if (!tdc->busy) goto skip_dma_stop; @@ -782,30 +777,60 @@ static int tegra_dma_terminate_all(struct dma_chan *dc) tegra_dma_stop(tdc); if (!list_empty(&tdc->pending_sg_req) && was_busy) { - sgreq = list_first_entry(&tdc->pending_sg_req, - typeof(*sgreq), node); + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), + node); sgreq->dma_desc->bytes_transferred += get_current_xferred_count(tdc, sgreq, wcount); } tegra_dma_resume(tdc); + pm_runtime_put(tdc->tdma->dev); + wake_up_all(&tdc->wq); + skip_dma_stop: tegra_dma_abort_all(tdc); while (!list_empty(&tdc->cb_desc)) { - dma_desc = list_first_entry(&tdc->cb_desc, - typeof(*dma_desc), cb_node); + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), + cb_node); list_del(&dma_desc->cb_node); dma_desc->cb_count = 0; } spin_unlock_irqrestore(&tdc->lock, flags); + return 0; } +static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc) +{ + unsigned long flags; + u32 status; + + spin_lock_irqsave(&tdc->lock, flags); + status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); + spin_unlock_irqrestore(&tdc->lock, flags); + + return !(status & TEGRA_APBDMA_STATUS_ISE_EOC); +} + +static void tegra_dma_synchronize(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + + /* + * CPU, which handles interrupt, could be busy in + * uninterruptible state, in this case sibling CPU + * should wait until interrupt is handled. + */ + wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc)); + + tasklet_kill(&tdc->tasklet); +} + static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *sg_req) { - unsigned long status, wcount = 0; + u32 status, wcount = 0; if (!list_is_first(&sg_req->node, &tdc->pending_sg_req)) return 0; @@ -862,7 +887,8 @@ static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, } static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, - dma_cookie_t cookie, struct dma_tx_state *txstate) + dma_cookie_t cookie, + struct dma_tx_state *txstate) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; @@ -909,11 +935,12 @@ found: trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); spin_unlock_irqrestore(&tdc->lock, flags); + return ret; } -static inline int get_bus_width(struct tegra_dma_channel *tdc, - enum dma_slave_buswidth slave_bw) +static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc, + enum dma_slave_buswidth slave_bw) { switch (slave_bw) { case DMA_SLAVE_BUSWIDTH_1_BYTE: @@ -926,16 +953,17 @@ static inline int get_bus_width(struct tegra_dma_channel *tdc, return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; default: dev_warn(tdc2dev(tdc), - "slave bw is not supported, using 32bits\n"); + "slave bw is not supported, using 32bits\n"); return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; } } -static inline int get_burst_size(struct tegra_dma_channel *tdc, - u32 burst_size, enum dma_slave_buswidth slave_bw, int len) +static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc, + u32 burst_size, + enum dma_slave_buswidth slave_bw, + u32 len) { - int burst_byte; - int burst_ahb_width; + unsigned int burst_byte, burst_ahb_width; /* * burst_size from client is in terms of the bus_width. @@ -962,9 +990,12 @@ static inline int get_burst_size(struct tegra_dma_channel *tdc, } static int get_transfer_param(struct tegra_dma_channel *tdc, - enum dma_transfer_direction direction, unsigned long *apb_addr, - unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size, - enum dma_slave_buswidth *slave_bw) + enum dma_transfer_direction direction, + u32 *apb_addr, + u32 *apb_seq, + u32 *csr, + unsigned int *burst_size, + enum dma_slave_buswidth *slave_bw) { switch (direction) { case DMA_MEM_TO_DEV: @@ -985,13 +1016,15 @@ static int get_transfer_param(struct tegra_dma_channel *tdc, default: dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); - return -EINVAL; + break; } + return -EINVAL; } static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, - struct tegra_dma_channel_regs *ch_regs, u32 len) + struct tegra_dma_channel_regs *ch_regs, + u32 len) { u32 len_field = (len - 4) & 0xFFFC; @@ -1001,20 +1034,23 @@ static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, ch_regs->csr |= len_field; } -static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( - struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long flags, - void *context) +static struct dma_async_tx_descriptor * +tegra_dma_prep_slave_sg(struct dma_chan *dc, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, + void *context) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + struct tegra_dma_sg_req *sg_req = NULL; + u32 csr, ahb_seq, apb_ptr, apb_seq; + enum dma_slave_buswidth slave_bw; struct tegra_dma_desc *dma_desc; - unsigned int i; - struct scatterlist *sg; - unsigned long csr, ahb_seq, apb_ptr, apb_seq; struct list_head req_list; - struct tegra_dma_sg_req *sg_req = NULL; - u32 burst_size; - enum dma_slave_buswidth slave_bw; + struct scatterlist *sg; + unsigned int burst_size; + unsigned int i; if (!tdc->config_init) { dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); @@ -1026,7 +1062,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( } if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw) < 0) + &burst_size, &slave_bw) < 0) return NULL; INIT_LIST_HEAD(&req_list); @@ -1072,7 +1108,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( len = sg_dma_len(sg); if ((len & 3) || (mem & 3) || - (len > tdc->tdma->chip_data->max_dma_count)) { + len > tdc->tdma->chip_data->max_dma_count) { dev_err(tdc2dev(tdc), "DMA length/memory address is not supported\n"); tegra_dma_desc_put(tdc, dma_desc); @@ -1124,20 +1160,21 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( return &dma_desc->txd; } -static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( - struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags) +static struct dma_async_tx_descriptor * +tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, + size_t buf_len, + size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma_desc *dma_desc = NULL; struct tegra_dma_sg_req *sg_req = NULL; - unsigned long csr, ahb_seq, apb_ptr, apb_seq; - int len; - size_t remain_len; - dma_addr_t mem = buf_addr; - u32 burst_size; + u32 csr, ahb_seq, apb_ptr, apb_seq; enum dma_slave_buswidth slave_bw; + struct tegra_dma_desc *dma_desc; + dma_addr_t mem = buf_addr; + unsigned int burst_size; + size_t len, remain_len; if (!buf_len || !period_len) { dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); @@ -1171,13 +1208,13 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( len = period_len; if ((len & 3) || (buf_addr & 3) || - (len > tdc->tdma->chip_data->max_dma_count)) { + len > tdc->tdma->chip_data->max_dma_count) { dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); return NULL; } if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw) < 0) + &burst_size, &slave_bw) < 0) return NULL; ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; @@ -1263,15 +1300,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma *tdma = tdc->tdma; - int ret; dma_cookie_init(&tdc->dma_chan); - tdc->config_init = false; - - ret = pm_runtime_get_sync(tdma->dev); - if (ret < 0) - return ret; return 0; } @@ -1279,33 +1309,29 @@ static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) static void tegra_dma_free_chan_resources(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); - struct tegra_dma *tdma = tdc->tdma; struct tegra_dma_desc *dma_desc; struct tegra_dma_sg_req *sg_req; struct list_head dma_desc_list; struct list_head sg_req_list; - unsigned long flags; INIT_LIST_HEAD(&dma_desc_list); INIT_LIST_HEAD(&sg_req_list); dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); - if (tdc->busy) - tegra_dma_terminate_all(dc); + tegra_dma_terminate_all(dc); + tasklet_kill(&tdc->tasklet); - spin_lock_irqsave(&tdc->lock, flags); list_splice_init(&tdc->pending_sg_req, &sg_req_list); list_splice_init(&tdc->free_sg_req, &sg_req_list); list_splice_init(&tdc->free_dma_desc, &dma_desc_list); INIT_LIST_HEAD(&tdc->cb_desc); tdc->config_init = false; tdc->isr_handler = NULL; - spin_unlock_irqrestore(&tdc->lock, flags); while (!list_empty(&dma_desc_list)) { - dma_desc = list_first_entry(&dma_desc_list, - typeof(*dma_desc), node); + dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc), + node); list_del(&dma_desc->node); kfree(dma_desc); } @@ -1315,7 +1341,6 @@ static void tegra_dma_free_chan_resources(struct dma_chan *dc) list_del(&sg_req->node); kfree(sg_req); } - pm_runtime_put(tdma->dev); tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; } @@ -1324,8 +1349,8 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct tegra_dma *tdma = ofdma->of_dma_data; - struct dma_chan *chan; struct tegra_dma_channel *tdc; + struct dma_chan *chan; if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) { dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]); @@ -1378,23 +1403,48 @@ static const struct tegra_dma_chip_data tegra148_dma_chip_data = { .support_separate_wcount_reg = true, }; +static int tegra_dma_init_hw(struct tegra_dma *tdma) +{ + int err; + + err = reset_control_assert(tdma->rst); + if (err) { + dev_err(tdma->dev, "failed to assert reset: %d\n", err); + return err; + } + + err = clk_enable(tdma->dma_clk); + if (err) { + dev_err(tdma->dev, "failed to enable clk: %d\n", err); + return err; + } + + /* reset DMA controller */ + udelay(2); + reset_control_deassert(tdma->rst); + + /* enable global DMA registers */ + tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); + tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); + tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF); + + clk_disable(tdma->dma_clk); + + return 0; +} + static int tegra_dma_probe(struct platform_device *pdev) { - struct resource *res; + const struct tegra_dma_chip_data *cdata; struct tegra_dma *tdma; + unsigned int i; + size_t size; int ret; - int i; - const struct tegra_dma_chip_data *cdata; cdata = of_device_get_match_data(&pdev->dev); - if (!cdata) { - dev_err(&pdev->dev, "Error: No device match data found\n"); - return -ENODEV; - } + size = struct_size(tdma, channels, cdata->nr_channels); - tdma = devm_kzalloc(&pdev->dev, - struct_size(tdma, channels, cdata->nr_channels), - GFP_KERNEL); + tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); if (!tdma) return -ENOMEM; @@ -1402,8 +1452,7 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->chip_data = cdata; platform_set_drvdata(pdev, tdma); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(tdma->base_addr)) return PTR_ERR(tdma->base_addr); @@ -1421,64 +1470,55 @@ static int tegra_dma_probe(struct platform_device *pdev) spin_lock_init(&tdma->global_lock); - pm_runtime_enable(&pdev->dev); - if (!pm_runtime_enabled(&pdev->dev)) - ret = tegra_dma_runtime_resume(&pdev->dev); - else - ret = pm_runtime_get_sync(&pdev->dev); - - if (ret < 0) { - pm_runtime_disable(&pdev->dev); + ret = clk_prepare(tdma->dma_clk); + if (ret) return ret; - } - /* Reset DMA controller */ - reset_control_assert(tdma->rst); - udelay(2); - reset_control_deassert(tdma->rst); - - /* Enable global DMA registers */ - tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); + ret = tegra_dma_init_hw(tdma); + if (ret) + goto err_clk_unprepare; - pm_runtime_put(&pdev->dev); + pm_runtime_irq_safe(&pdev->dev); + pm_runtime_enable(&pdev->dev); INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < cdata->nr_channels; i++) { struct tegra_dma_channel *tdc = &tdma->channels[i]; + int irq; tdc->chan_addr = tdma->base_addr + TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + (i * cdata->channel_reg_size); - res = platform_get_resource(pdev, IORESOURCE_IRQ, i); - if (!res) { - ret = -EINVAL; + irq = platform_get_irq(pdev, i); + if (irq < 0) { + ret = irq; dev_err(&pdev->dev, "No irq resource for chan %d\n", i); - goto err_irq; + goto err_pm_disable; } - tdc->irq = res->start; + snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); - ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc); + ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0, + tdc->name, tdc); if (ret) { dev_err(&pdev->dev, "request_irq failed with err %d channel %d\n", ret, i); - goto err_irq; + goto err_pm_disable; } tdc->dma_chan.device = &tdma->dma_dev; dma_cookie_init(&tdc->dma_chan); list_add_tail(&tdc->dma_chan.device_node, - &tdma->dma_dev.channels); + &tdma->dma_dev.channels); tdc->tdma = tdma; tdc->id = i; tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; tasklet_init(&tdc->tasklet, tegra_dma_tasklet, - (unsigned long)tdc); + (unsigned long)tdc); spin_lock_init(&tdc->lock); + init_waitqueue_head(&tdc->wq); INIT_LIST_HEAD(&tdc->pending_sg_req); INIT_LIST_HEAD(&tdc->free_sg_req); @@ -1510,6 +1550,7 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; tdma->dma_dev.device_config = tegra_dma_slave_config; tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; + tdma->dma_dev.device_synchronize = tegra_dma_synchronize; tdma->dma_dev.device_tx_status = tegra_dma_tx_status; tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; @@ -1517,7 +1558,7 @@ static int tegra_dma_probe(struct platform_device *pdev) if (ret < 0) { dev_err(&pdev->dev, "Tegra20 APB DMA driver registration failed %d\n", ret); - goto err_irq; + goto err_pm_disable; } ret = of_dma_controller_register(pdev->dev.of_node, @@ -1528,118 +1569,92 @@ static int tegra_dma_probe(struct platform_device *pdev) goto err_unregister_dma_dev; } - dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n", - cdata->nr_channels); + dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n", + cdata->nr_channels); + return 0; err_unregister_dma_dev: dma_async_device_unregister(&tdma->dma_dev); -err_irq: - while (--i >= 0) { - struct tegra_dma_channel *tdc = &tdma->channels[i]; - - free_irq(tdc->irq, tdc); - tasklet_kill(&tdc->tasklet); - } +err_pm_disable: pm_runtime_disable(&pdev->dev); - if (!pm_runtime_status_suspended(&pdev->dev)) - tegra_dma_runtime_suspend(&pdev->dev); + +err_clk_unprepare: + clk_unprepare(tdma->dma_clk); + return ret; } static int tegra_dma_remove(struct platform_device *pdev) { struct tegra_dma *tdma = platform_get_drvdata(pdev); - int i; - struct tegra_dma_channel *tdc; + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - - for (i = 0; i < tdma->chip_data->nr_channels; ++i) { - tdc = &tdma->channels[i]; - free_irq(tdc->irq, tdc); - tasklet_kill(&tdc->tasklet); - } - pm_runtime_disable(&pdev->dev); - if (!pm_runtime_status_suspended(&pdev->dev)) - tegra_dma_runtime_suspend(&pdev->dev); + clk_unprepare(tdma->dma_clk); return 0; } -static int tegra_dma_runtime_suspend(struct device *dev) +static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) { struct tegra_dma *tdma = dev_get_drvdata(dev); - int i; - - tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL); - for (i = 0; i < tdma->chip_data->nr_channels; i++) { - struct tegra_dma_channel *tdc = &tdma->channels[i]; - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; - - /* Only save the state of DMA channels that are in use */ - if (!tdc->config_init) - continue; - - ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); - ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR); - ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR); - ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ); - ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ); - if (tdma->chip_data->support_separate_wcount_reg) - ch_reg->wcount = tdc_read(tdc, - TEGRA_APBDMA_CHAN_WCOUNT); - } - clk_disable_unprepare(tdma->dma_clk); + clk_disable(tdma->dma_clk); return 0; } -static int tegra_dma_runtime_resume(struct device *dev) +static int __maybe_unused tegra_dma_runtime_resume(struct device *dev) { struct tegra_dma *tdma = dev_get_drvdata(dev); - int i, ret; - ret = clk_prepare_enable(tdma->dma_clk); - if (ret < 0) { - dev_err(dev, "clk_enable failed: %d\n", ret); - return ret; - } + return clk_enable(tdma->dma_clk); +} - tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen); - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); +static int __maybe_unused tegra_dma_dev_suspend(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + bool busy; for (i = 0; i < tdma->chip_data->nr_channels; i++) { struct tegra_dma_channel *tdc = &tdma->channels[i]; - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; - - /* Only restore the state of DMA channels that are in use */ - if (!tdc->config_init) - continue; - - if (tdma->chip_data->support_separate_wcount_reg) - tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, - ch_reg->wcount); - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq); - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr); - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq); - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr); - tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, - (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB)); + + tasklet_kill(&tdc->tasklet); + + spin_lock_irqsave(&tdc->lock, flags); + busy = tdc->busy; + spin_unlock_irqrestore(&tdc->lock, flags); + + if (busy) { + dev_err(tdma->dev, "channel %u busy\n", i); + return -EBUSY; + } } - return 0; + return pm_runtime_force_suspend(dev); +} + +static int __maybe_unused tegra_dma_dev_resume(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + int err; + + err = tegra_dma_init_hw(tdma); + if (err) + return err; + + return pm_runtime_force_resume(dev); } static const struct dev_pm_ops tegra_dma_dev_pm_ops = { SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) + SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume) }; static const struct of_device_id tegra_dma_of_match[] = { @@ -1672,7 +1687,6 @@ static struct platform_driver tegra_dmac_driver = { module_platform_driver(tegra_dmac_driver); -MODULE_ALIAS("platform:tegra20-apbdma"); MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 6e1268552f74..c4ce5dfb149b 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -164,7 +164,7 @@ struct tegra_adma { const struct tegra_adma_chip_data *cdata; /* Last member of the structure */ - struct tegra_adma_chan channels[0]; + struct tegra_adma_chan channels[]; }; static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index f255056696ee..4ba8fa5d9c36 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -133,7 +133,6 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev) const struct of_device_id *match; struct device_node *dma_node; struct ti_am335x_xbar_data *xbar; - struct resource *res; void __iomem *iomem; int i, ret; @@ -173,8 +172,7 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev) xbar->xbar_events = TI_AM335X_XBAR_LINES; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iomem = devm_ioremap_resource(&pdev->dev, res); + iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iomem)) return PTR_ERR(iomem); @@ -323,7 +321,6 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev) struct device_node *dma_node; struct ti_dra7_xbar_data *xbar; struct property *prop; - struct resource *res; u32 safe_val; int sz; void __iomem *iomem; @@ -403,8 +400,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev) kfree(rsv_events); } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iomem = devm_ioremap_resource(&pdev->dev, res); + iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iomem)) return PTR_ERR(iomem); diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 03a7f647f7b2..c4a5c170c1f9 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -1275,6 +1275,81 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy( return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); } +static struct dma_async_tx_descriptor * +edma_prep_dma_interleaved(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long tx_flags) +{ + struct device *dev = chan->device->dev; + struct edma_chan *echan = to_edma_chan(chan); + struct edmacc_param *param; + struct edma_desc *edesc; + size_t src_icg, dst_icg; + int src_bidx, dst_bidx; + + /* Slave mode is not supported */ + if (is_slave_direction(xt->dir)) + return NULL; + + if (xt->frame_size != 1 || xt->numf == 0) + return NULL; + + if (xt->sgl[0].size > SZ_64K || xt->numf > SZ_64K) + return NULL; + + src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); + if (src_icg) { + src_bidx = src_icg + xt->sgl[0].size; + } else if (xt->src_inc) { + src_bidx = xt->sgl[0].size; + } else { + dev_err(dev, "%s: SRC constant addressing is not supported\n", + __func__); + return NULL; + } + + dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); + if (dst_icg) { + dst_bidx = dst_icg + xt->sgl[0].size; + } else if (xt->dst_inc) { + dst_bidx = xt->sgl[0].size; + } else { + dev_err(dev, "%s: DST constant addressing is not supported\n", + __func__); + return NULL; + } + + if (src_bidx > SZ_64K || dst_bidx > SZ_64K) + return NULL; + + edesc = kzalloc(struct_size(edesc, pset, 1), GFP_ATOMIC); + if (!edesc) + return NULL; + + edesc->direction = DMA_MEM_TO_MEM; + edesc->echan = echan; + edesc->pset_nr = 1; + + param = &edesc->pset[0].param; + + param->src = xt->src_start; + param->dst = xt->dst_start; + param->a_b_cnt = xt->numf << 16 | xt->sgl[0].size; + param->ccnt = 1; + param->src_dst_bidx = (dst_bidx << 16) | src_bidx; + param->src_dst_cidx = 0; + + param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); + param->opt |= ITCCHEN; + /* Enable transfer complete interrupt if requested */ + if (tx_flags & DMA_PREP_INTERRUPT) + param->opt |= TCINTEN; + else + edesc->polled = true; + + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); +} + static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, @@ -1917,7 +1992,9 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) "Legacy memcpy is enabled, things might not work\n"); dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask); + dma_cap_set(DMA_INTERLEAVE, s_ddev->cap_mask); s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; + s_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; s_ddev->directions = BIT(DMA_MEM_TO_MEM); } @@ -1953,8 +2030,10 @@ static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode) dma_cap_zero(m_ddev->cap_mask); dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask); + dma_cap_set(DMA_INTERLEAVE, m_ddev->cap_mask); m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; + m_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources; m_ddev->device_free_chan_resources = edma_free_chan_resources; m_ddev->device_issue_pending = edma_issue_pending; diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index c1511298ece2..64c8955e0cf1 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -32,6 +32,7 @@ struct k3_udma_glue_common { bool epib; u32 psdata_size; u32 swdata_size; + u32 atype; }; struct k3_udma_glue_tx_channel { @@ -121,6 +122,15 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, return -ENOENT; thread_id = dma_spec.args[0]; + if (dma_spec.args_count == 2) { + if (dma_spec.args[1] > 2) { + dev_err(common->dev, "Invalid channel atype: %u\n", + dma_spec.args[1]); + ret = -EINVAL; + goto out_put_spec; + } + common->atype = dma_spec.args[1]; + } if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { ret = -EINVAL; @@ -202,7 +212,8 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID; + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; req.nav_id = tisci_rm->tisci_dev_id; req.index = tx_chn->udma_tchan_id; if (tx_chn->tx_pause_on_err) @@ -216,6 +227,7 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) req.tx_supr_tdpkt = 1; req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); + req.tx_atype = tx_chn->common.atype; return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } @@ -502,7 +514,8 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; req.nav_id = tisci_rm->tisci_dev_id; req.index = rx_chn->udma_rchan_id; @@ -519,6 +532,7 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) req.flowid_cnt = rx_chn->flow_num; } req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; + req.rx_atype = rx_chn->common.atype; ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); if (ret) @@ -564,12 +578,12 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, if (IS_ERR(flow->udma_rflow)) { ret = PTR_ERR(flow->udma_rflow); dev_err(dev, "UDMAX rflow get err %d\n", ret); - goto err; + return ret; } if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) { - xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); - return -ENODEV; + ret = -ENODEV; + goto err_rflow_put; } /* request and cfg rings */ @@ -578,7 +592,7 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, if (!flow->ringrx) { ret = -ENODEV; dev_err(dev, "Failed to get RX ring\n"); - goto err; + goto err_rflow_put; } flow->ringrxfdq = k3_ringacc_request_ring(rx_chn->common.ringacc, @@ -586,19 +600,19 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, if (!flow->ringrxfdq) { ret = -ENODEV; dev_err(dev, "Failed to get RXFDQ ring\n"); - goto err; + goto err_ringrx_free; } ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringrx %d\n", ret); - goto err; + goto err_ringrxfdq_free; } ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg); if (ret) { dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret); - goto err; + goto err_ringrxfdq_free; } if (rx_chn->remote) { @@ -648,7 +662,7 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, if (ret) { dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id, ret); - goto err; + goto err_ringrxfdq_free; } rx_chn->flows_ready++; @@ -656,8 +670,17 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, flow->udma_rflow_id, rx_chn->flows_ready); return 0; -err: - k3_udma_glue_release_rx_flow(rx_chn, flow_idx); + +err_ringrxfdq_free: + k3_ringacc_ring_free(flow->ringrxfdq); + +err_ringrx_free: + k3_ringacc_ring_free(flow->ringrx); + +err_rflow_put: + xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow); + flow->udma_rflow = NULL; + return ret; } diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index ea79c2df28e0..a9c0251adf1a 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> @@ -96,6 +97,24 @@ struct udma_match_data { u32 level_start_idx[]; }; +struct udma_hwdesc { + size_t cppi5_desc_size; + void *cppi5_desc_vaddr; + dma_addr_t cppi5_desc_paddr; + + /* TR descriptor internal pointers */ + void *tr_req_base; + struct cppi5_tr_resp_t *tr_resp_base; +}; + +struct udma_rx_flush { + struct udma_hwdesc hwdescs[2]; + + size_t buffer_size; + void *buffer_vaddr; + dma_addr_t buffer_paddr; +}; + struct udma_dev { struct dma_device ddev; struct device *dev; @@ -112,6 +131,8 @@ struct udma_dev { struct list_head desc_to_purge; spinlock_t lock; + struct udma_rx_flush rx_flush; + int tchan_cnt; int echan_cnt; int rchan_cnt; @@ -128,16 +149,7 @@ struct udma_dev { struct udma_chan *channels; u32 psil_base; -}; - -struct udma_hwdesc { - size_t cppi5_desc_size; - void *cppi5_desc_vaddr; - dma_addr_t cppi5_desc_paddr; - - /* TR descriptor internal pointers */ - void *tr_req_base; - struct cppi5_tr_resp_t *tr_resp_base; + u32 atype; }; struct udma_desc { @@ -169,7 +181,7 @@ enum udma_chan_state { struct udma_tx_drain { struct delayed_work work; - unsigned long jiffie; + ktime_t tstamp; u32 residue; }; @@ -181,6 +193,7 @@ struct udma_chan_config { u32 hdesc_size; /* Size of a packet descriptor in packet mode */ bool notdpkt; /* Suppress sending TDC packet */ int remote_thread_id; + u32 atype; u32 src_thread; u32 dst_thread; enum psil_endpoint_type ep_type; @@ -502,7 +515,7 @@ static bool udma_is_chan_paused(struct udma_chan *uc) { u32 val, pause_mask; - switch (uc->desc->dir) { + switch (uc->config.dir) { case DMA_DEV_TO_MEM: val = udma_rchanrt_read(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG); @@ -551,12 +564,17 @@ static void udma_sync_for_device(struct udma_chan *uc, int idx) } } +static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc) +{ + return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; +} + static int udma_push_to_ring(struct udma_chan *uc, int idx) { struct udma_desc *d = uc->desc; - struct k3_ring *ring = NULL; - int ret = -EINVAL; + dma_addr_t paddr; + int ret; switch (uc->config.dir) { case DMA_DEV_TO_MEM: @@ -567,21 +585,37 @@ static int udma_push_to_ring(struct udma_chan *uc, int idx) ring = uc->tchan->t_ring; break; default: - break; + return -EINVAL; } - if (ring) { - dma_addr_t desc_addr = udma_curr_cppi5_desc_paddr(d, idx); + /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */ + if (idx == -1) { + paddr = udma_get_rx_flush_hwdesc_paddr(uc); + } else { + paddr = udma_curr_cppi5_desc_paddr(d, idx); wmb(); /* Ensure that writes are not moved over this point */ udma_sync_for_device(uc, idx); - ret = k3_ringacc_ring_push(ring, &desc_addr); - uc->in_ring_cnt++; } + ret = k3_ringacc_ring_push(ring, &paddr); + if (!ret) + uc->in_ring_cnt++; + return ret; } +static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr) +{ + if (uc->config.dir != DMA_DEV_TO_MEM) + return false; + + if (addr == udma_get_rx_flush_hwdesc_paddr(uc)) + return true; + + return false; +} + static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) { struct k3_ring *ring = NULL; @@ -610,6 +644,10 @@ static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr) if (cppi5_desc_is_tdcm(*addr)) return ret; + /* Check for flush descriptor */ + if (udma_desc_is_rx_flush(uc, *addr)) + return -ENOENT; + d = udma_udma_desc_from_paddr(uc, *addr); if (d) @@ -890,6 +928,9 @@ static int udma_stop(struct udma_chan *uc) switch (uc->config.dir) { case DMA_DEV_TO_MEM: + if (!uc->cyclic && !uc->desc) + udma_push_to_ring(uc, -1); + udma_rchanrt_write(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); @@ -946,9 +987,10 @@ static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) peer_bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_PEER_BCNT_REG); bcnt = udma_tchanrt_read(uc->tchan, UDMA_TCHAN_RT_BCNT_REG); + /* Transfer is incomplete, store current residue and time stamp */ if (peer_bcnt < bcnt) { uc->tx_drain.residue = bcnt - peer_bcnt; - uc->tx_drain.jiffie = jiffies; + uc->tx_drain.tstamp = ktime_get(); return false; } @@ -961,35 +1003,59 @@ static void udma_check_tx_completion(struct work_struct *work) tx_drain.work.work); bool desc_done = true; u32 residue_diff; - unsigned long jiffie_diff, delay; + ktime_t time_diff; + unsigned long delay; + + while (1) { + if (uc->desc) { + /* Get previous residue and time stamp */ + residue_diff = uc->tx_drain.residue; + time_diff = uc->tx_drain.tstamp; + /* + * Get current residue and time stamp or see if + * transfer is complete + */ + desc_done = udma_is_desc_really_done(uc, uc->desc); + } - if (uc->desc) { - residue_diff = uc->tx_drain.residue; - jiffie_diff = uc->tx_drain.jiffie; - desc_done = udma_is_desc_really_done(uc, uc->desc); - } - - if (!desc_done) { - jiffie_diff = uc->tx_drain.jiffie - jiffie_diff; - residue_diff -= uc->tx_drain.residue; - if (residue_diff) { - /* Try to guess when we should check next time */ - residue_diff /= jiffie_diff; - delay = uc->tx_drain.residue / residue_diff / 3; - if (jiffies_to_msecs(delay) < 5) - delay = 0; - } else { - /* No progress, check again in 1 second */ - delay = HZ; + if (!desc_done) { + /* + * Find the time delta and residue delta w.r.t + * previous poll + */ + time_diff = ktime_sub(uc->tx_drain.tstamp, + time_diff) + 1; + residue_diff -= uc->tx_drain.residue; + if (residue_diff) { + /* + * Try to guess when we should check + * next time by calculating rate at + * which data is being drained at the + * peer device + */ + delay = (time_diff / residue_diff) * + uc->tx_drain.residue; + } else { + /* No progress, check again in 1 second */ + schedule_delayed_work(&uc->tx_drain.work, HZ); + break; + } + + usleep_range(ktime_to_us(delay), + ktime_to_us(delay) + 10); + continue; } - schedule_delayed_work(&uc->tx_drain.work, delay); - } else if (uc->desc) { - struct udma_desc *d = uc->desc; + if (uc->desc) { + struct udma_desc *d = uc->desc; - uc->bcnt += d->residue; - udma_start(uc); - vchan_cookie_complete(&d->vd); + uc->bcnt += d->residue; + udma_start(uc); + vchan_cookie_complete(&d->vd); + break; + } + + break; } } @@ -1033,29 +1099,27 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data) goto out; } - if (uc->cyclic) { - /* push the descriptor back to the ring */ - if (d == uc->desc) { + if (d == uc->desc) { + /* active descriptor */ + if (uc->cyclic) { udma_cyclic_packet_elapsed(uc); vchan_cyclic_callback(&d->vd); - } - } else { - bool desc_done = false; - - if (d == uc->desc) { - desc_done = udma_is_desc_really_done(uc, d); - - if (desc_done) { + } else { + if (udma_is_desc_really_done(uc, d)) { uc->bcnt += d->residue; udma_start(uc); + vchan_cookie_complete(&d->vd); } else { schedule_delayed_work(&uc->tx_drain.work, 0); } } - - if (desc_done) - vchan_cookie_complete(&d->vd); + } else { + /* + * terminated descriptor, mark the descriptor as + * completed to update the channel's cookie marker + */ + dma_cookie_complete(&d->vd.tx); } } out: @@ -1507,7 +1571,8 @@ err_rflow: TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID) + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) #define TISCI_RCHAN_VALID_PARAMS ( \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ @@ -1517,7 +1582,8 @@ err_rflow: TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID) + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) static int udma_tisci_m2m_channel_config(struct udma_chan *uc) { @@ -1539,6 +1605,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_tx.txcq_qnum = tc_ring; + req_tx.tx_atype = ud->atype; ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) { @@ -1552,6 +1619,7 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_rx.rxcq_qnum = tc_ring; req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; + req_rx.rx_atype = ud->atype; ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) @@ -1587,6 +1655,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) req_tx.tx_supr_tdpkt = uc->config.notdpkt; req_tx.tx_fetch_size = fetch_size >> 2; req_tx.txcq_qnum = tc_ring; + req_tx.tx_atype = uc->config.atype; ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -1623,6 +1692,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) req_rx.rx_fetch_size = fetch_size >> 2; req_rx.rxcq_qnum = rx_ring; req_rx.rx_chan_type = mode; + req_rx.rx_atype = uc->config.atype; ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) { @@ -1965,36 +2035,81 @@ static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc, return d; } +/** + * udma_get_tr_counters - calculate TR counters for a given length + * @len: Length of the trasnfer + * @align_to: Preferred alignment + * @tr0_cnt0: First TR icnt0 + * @tr0_cnt1: First TR icnt1 + * @tr1_cnt0: Second (if used) TR icnt0 + * + * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated + * For len >= SZ_64K two TRs are used in a simple way: + * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1) + * Second TR: the remaining length (tr1_cnt0) + * + * Returns the number of TRs the length needs (1 or 2) + * -EINVAL if the length can not be supported + */ +static int udma_get_tr_counters(size_t len, unsigned long align_to, + u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0) +{ + if (len < SZ_64K) { + *tr0_cnt0 = len; + *tr0_cnt1 = 1; + + return 1; + } + + if (align_to > 3) + align_to = 3; + +realign: + *tr0_cnt0 = SZ_64K - BIT(align_to); + if (len / *tr0_cnt0 >= SZ_64K) { + if (align_to) { + align_to--; + goto realign; + } + return -EINVAL; + } + + *tr0_cnt1 = len / *tr0_cnt0; + *tr1_cnt0 = len % *tr0_cnt0; + + return 2; +} + static struct udma_desc * udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, unsigned int sglen, enum dma_transfer_direction dir, unsigned long tx_flags, void *context) { - enum dma_slave_buswidth dev_width; struct scatterlist *sgent; struct udma_desc *d; - size_t tr_size; struct cppi5_tr_type1_t *tr_req = NULL; + u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; unsigned int i; - u32 burst; + size_t tr_size; + int num_tr = 0; + int tr_idx = 0; - if (dir == DMA_DEV_TO_MEM) { - dev_width = uc->cfg.src_addr_width; - burst = uc->cfg.src_maxburst; - } else if (dir == DMA_MEM_TO_DEV) { - dev_width = uc->cfg.dst_addr_width; - burst = uc->cfg.dst_maxburst; - } else { - dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); + if (!is_slave_direction(dir)) { + dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); return NULL; } - if (!burst) - burst = 1; + /* estimate the number of TRs we will need */ + for_each_sg(sgl, sgent, sglen, i) { + if (sg_dma_len(sgent) < SZ_64K) + num_tr++; + else + num_tr += 2; + } /* Now allocate and setup the descriptor. */ tr_size = sizeof(struct cppi5_tr_type1_t); - d = udma_alloc_tr_desc(uc, tr_size, sglen, dir); + d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir); if (!d) return NULL; @@ -2002,19 +2117,46 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl, tr_req = d->hwdesc[0].tr_req_base; for_each_sg(sgl, sgent, sglen, i) { - d->residue += sg_dma_len(sgent); + dma_addr_t sg_addr = sg_dma_address(sgent); + + num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr), + &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); + if (num_tr < 0) { + dev_err(uc->ud->dev, "size %u is not supported\n", + sg_dma_len(sgent)); + udma_free_hwdesc(uc, d); + kfree(d); + return NULL; + } cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); cppi5_tr_csf_set(&tr_req[i].flags, CPPI5_TR_CSF_SUPR_EVT); - tr_req[i].addr = sg_dma_address(sgent); - tr_req[i].icnt0 = burst * dev_width; - tr_req[i].dim1 = burst * dev_width; - tr_req[i].icnt1 = sg_dma_len(sgent) / tr_req[i].icnt0; + tr_req[tr_idx].addr = sg_addr; + tr_req[tr_idx].icnt0 = tr0_cnt0; + tr_req[tr_idx].icnt1 = tr0_cnt1; + tr_req[tr_idx].dim1 = tr0_cnt0; + tr_idx++; + + if (num_tr == 2) { + cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, + false, false, + CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, + CPPI5_TR_CSF_SUPR_EVT); + + tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0; + tr_req[tr_idx].icnt0 = tr1_cnt0; + tr_req[tr_idx].icnt1 = 1; + tr_req[tr_idx].dim1 = tr1_cnt0; + tr_idx++; + } + + d->residue += sg_dma_len(sgent); } - cppi5_tr_csf_set(&tr_req[i - 1].flags, CPPI5_TR_CSF_EOP); + cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, CPPI5_TR_CSF_EOP); return d; } @@ -2319,47 +2461,66 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { - enum dma_slave_buswidth dev_width; struct udma_desc *d; - size_t tr_size; + size_t tr_size, period_addr; struct cppi5_tr_type1_t *tr_req; - unsigned int i; unsigned int periods = buf_len / period_len; - u32 burst; + u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; + unsigned int i; + int num_tr; - if (dir == DMA_DEV_TO_MEM) { - dev_width = uc->cfg.src_addr_width; - burst = uc->cfg.src_maxburst; - } else if (dir == DMA_MEM_TO_DEV) { - dev_width = uc->cfg.dst_addr_width; - burst = uc->cfg.dst_maxburst; - } else { - dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); + if (!is_slave_direction(dir)) { + dev_err(uc->ud->dev, "Only slave cyclic is supported\n"); return NULL; } - if (!burst) - burst = 1; + num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, + &tr0_cnt1, &tr1_cnt0); + if (num_tr < 0) { + dev_err(uc->ud->dev, "size %zu is not supported\n", + period_len); + return NULL; + } /* Now allocate and setup the descriptor. */ tr_size = sizeof(struct cppi5_tr_type1_t); - d = udma_alloc_tr_desc(uc, tr_size, periods, dir); + d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir); if (!d) return NULL; tr_req = d->hwdesc[0].tr_req_base; + period_addr = buf_addr; for (i = 0; i < periods; i++) { - cppi5_tr_init(&tr_req[i].flags, CPPI5_TR_TYPE1, false, false, - CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + int tr_idx = i * num_tr; + + cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false, + false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + + tr_req[tr_idx].addr = period_addr; + tr_req[tr_idx].icnt0 = tr0_cnt0; + tr_req[tr_idx].icnt1 = tr0_cnt1; + tr_req[tr_idx].dim1 = tr0_cnt0; + + if (num_tr == 2) { + cppi5_tr_csf_set(&tr_req[tr_idx].flags, + CPPI5_TR_CSF_SUPR_EVT); + tr_idx++; + + cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, + false, false, + CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - tr_req[i].addr = buf_addr + period_len * i; - tr_req[i].icnt0 = dev_width; - tr_req[i].icnt1 = period_len / dev_width; - tr_req[i].dim1 = dev_width; + tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0; + tr_req[tr_idx].icnt0 = tr1_cnt0; + tr_req[tr_idx].icnt1 = 1; + tr_req[tr_idx].dim1 = tr1_cnt0; + } if (!(flags & DMA_PREP_INTERRUPT)) - cppi5_tr_csf_set(&tr_req[i].flags, + cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); + + period_addr += period_len; } return d; @@ -2517,29 +2678,12 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, return NULL; } - if (len < SZ_64K) { - num_tr = 1; - tr0_cnt0 = len; - tr0_cnt1 = 1; - } else { - unsigned long align_to = __ffs(src | dest); - - if (align_to > 3) - align_to = 3; - /* - * Keep simple: tr0: SZ_64K-alignment blocks, - * tr1: the remaining - */ - num_tr = 2; - tr0_cnt0 = (SZ_64K - BIT(align_to)); - if (len / tr0_cnt0 >= SZ_64K) { - dev_err(uc->ud->dev, "size %zu is not supported\n", - len); - return NULL; - } - - tr0_cnt1 = len / tr0_cnt0; - tr1_cnt0 = len % tr0_cnt0; + num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0, + &tr0_cnt1, &tr1_cnt0); + if (num_tr < 0) { + dev_err(uc->ud->dev, "size %zu is not supported\n", + len); + return NULL; } d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM); @@ -2631,6 +2775,9 @@ static enum dma_status udma_tx_status(struct dma_chan *chan, ret = dma_cookie_status(chan, cookie, txstate); + if (!udma_is_chan_running(uc)) + ret = DMA_COMPLETE; + if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc)) ret = DMA_PAUSED; @@ -2697,11 +2844,8 @@ static int udma_pause(struct dma_chan *chan) { struct udma_chan *uc = to_udma_chan(chan); - if (!uc->desc) - return -EINVAL; - /* pause the channel */ - switch (uc->desc->dir) { + switch (uc->config.dir) { case DMA_DEV_TO_MEM: udma_rchanrt_update_bits(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, @@ -2730,11 +2874,8 @@ static int udma_resume(struct dma_chan *chan) { struct udma_chan *uc = to_udma_chan(chan); - if (!uc->desc) - return -EINVAL; - /* resume the channel */ - switch (uc->desc->dir) { + switch (uc->config.dir) { case DMA_DEV_TO_MEM: udma_rchanrt_update_bits(uc->rchan, UDMA_RCHAN_RT_PEER_RT_EN_REG, @@ -2930,13 +3071,18 @@ static void udma_free_chan_resources(struct dma_chan *chan) static struct platform_driver udma_driver; +struct udma_filter_param { + int remote_thread_id; + u32 atype; +}; + static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) { struct udma_chan_config *ucc; struct psil_endpoint_config *ep_config; + struct udma_filter_param *filter_param; struct udma_chan *uc; struct udma_dev *ud; - u32 *args; if (chan->device->dev->driver != &udma_driver.driver) return false; @@ -2944,9 +3090,16 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) uc = to_udma_chan(chan); ucc = &uc->config; ud = uc->ud; - args = param; + filter_param = param; - ucc->remote_thread_id = args[0]; + if (filter_param->atype > 2) { + dev_err(ud->dev, "Invalid channel atype: %u\n", + filter_param->atype); + return false; + } + + ucc->remote_thread_id = filter_param->remote_thread_id; + ucc->atype = filter_param->atype; if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) ucc->dir = DMA_MEM_TO_DEV; @@ -2959,6 +3112,7 @@ static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) ucc->remote_thread_id); ucc->dir = DMA_MEM_TO_MEM; ucc->remote_thread_id = -1; + ucc->atype = 0; return false; } @@ -2997,13 +3151,20 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, { struct udma_dev *ud = ofdma->of_dma_data; dma_cap_mask_t mask = ud->ddev.cap_mask; + struct udma_filter_param filter_param; struct dma_chan *chan; - if (dma_spec->args_count != 1) + if (dma_spec->args_count != 1 && dma_spec->args_count != 2) return NULL; - chan = __dma_request_channel(&mask, udma_dma_filter_fn, - &dma_spec->args[0], ofdma->of_node); + filter_param.remote_thread_id = dma_spec->args[0]; + if (dma_spec->args_count == 2) + filter_param.atype = dma_spec->args[1]; + else + filter_param.atype = 0; + + chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, + ofdma->of_node); if (!chan) { dev_err(ud->dev, "get channel fail in %s.\n", __func__); return ERR_PTR(-EINVAL); @@ -3248,6 +3409,158 @@ static int udma_setup_resources(struct udma_dev *ud) return ch_count; } +static int udma_setup_rx_flush(struct udma_dev *ud) +{ + struct udma_rx_flush *rx_flush = &ud->rx_flush; + struct cppi5_desc_hdr_t *tr_desc; + struct cppi5_tr_type1_t *tr_req; + struct cppi5_host_desc_t *desc; + struct device *dev = ud->dev; + struct udma_hwdesc *hwdesc; + size_t tr_size; + + /* Allocate 1K buffer for discarded data on RX channel teardown */ + rx_flush->buffer_size = SZ_1K; + rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size, + GFP_KERNEL); + if (!rx_flush->buffer_vaddr) + return -ENOMEM; + + rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr, + rx_flush->buffer_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, rx_flush->buffer_paddr)) + return -ENOMEM; + + /* Set up descriptor to be used for TR mode */ + hwdesc = &rx_flush->hwdescs[0]; + tr_size = sizeof(struct cppi5_tr_type1_t); + hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1); + hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size, + ud->desc_align); + + hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, + GFP_KERNEL); + if (!hwdesc->cppi5_desc_vaddr) + return -ENOMEM; + + hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, + hwdesc->cppi5_desc_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) + return -ENOMEM; + + /* Start of the TR req records */ + hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size; + /* Start address of the TR response array */ + hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size; + + tr_desc = hwdesc->cppi5_desc_vaddr; + cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0); + cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); + cppi5_desc_set_retpolicy(tr_desc, 0, 0); + + tr_req = hwdesc->tr_req_base; + cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false, + CPPI5_TR_EVENT_SIZE_COMPLETION, 0); + cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT); + + tr_req->addr = rx_flush->buffer_paddr; + tr_req->icnt0 = rx_flush->buffer_size; + tr_req->icnt1 = 1; + + /* Set up descriptor to be used for packet mode */ + hwdesc = &rx_flush->hwdescs[1]; + hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) + + CPPI5_INFO0_HDESC_EPIB_SIZE + + CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE, + ud->desc_align); + + hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size, + GFP_KERNEL); + if (!hwdesc->cppi5_desc_vaddr) + return -ENOMEM; + + hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr, + hwdesc->cppi5_desc_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr)) + return -ENOMEM; + + desc = hwdesc->cppi5_desc_vaddr; + cppi5_hdesc_init(desc, 0, 0); + cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT); + cppi5_desc_set_retpolicy(&desc->hdr, 0, 0); + + cppi5_hdesc_attach_buf(desc, + rx_flush->buffer_paddr, rx_flush->buffer_size, + rx_flush->buffer_paddr, rx_flush->buffer_size); + + dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr, + hwdesc->cppi5_desc_size, DMA_TO_DEVICE); + return 0; +} + +#ifdef CONFIG_DEBUG_FS +static void udma_dbg_summary_show_chan(struct seq_file *s, + struct dma_chan *chan) +{ + struct udma_chan *uc = to_udma_chan(chan); + struct udma_chan_config *ucc = &uc->config; + + seq_printf(s, " %-13s| %s", dma_chan_name(chan), + chan->dbg_client_name ?: "in-use"); + seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); + + switch (uc->config.dir) { + case DMA_MEM_TO_MEM: + seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, + ucc->src_thread, ucc->dst_thread); + break; + case DMA_DEV_TO_MEM: + seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, + ucc->src_thread, ucc->dst_thread); + break; + case DMA_MEM_TO_DEV: + seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, + ucc->src_thread, ucc->dst_thread); + break; + default: + seq_printf(s, ")\n"); + return; + } + + if (ucc->ep_type == PSIL_EP_NATIVE) { + seq_printf(s, "PSI-L Native"); + if (ucc->metadata_size) { + seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); + if (ucc->psd_size) + seq_printf(s, " PSDsize:%u", ucc->psd_size); + seq_printf(s, " ]"); + } + } else { + seq_printf(s, "PDMA"); + if (ucc->enable_acc32 || ucc->enable_burst) + seq_printf(s, "[%s%s ]", + ucc->enable_acc32 ? " ACC32" : "", + ucc->enable_burst ? " BURST" : ""); + } + + seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); +} + +static void udma_dbg_summary_show(struct seq_file *s, + struct dma_device *dma_dev) +{ + struct dma_chan *chan; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) + udma_dbg_summary_show_chan(s, chan); + } +} +#endif /* CONFIG_DEBUG_FS */ + #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ @@ -3294,6 +3607,12 @@ static int udma_probe(struct platform_device *pdev) return ret; } + ret = of_property_read_u32(navss_node, "ti,udma-atype", &ud->atype); + if (!ret && ud->atype > 2) { + dev_err(dev, "Invalid atype: %u\n", ud->atype); + return -EINVAL; + } + ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; @@ -3328,6 +3647,9 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.device_resume = udma_resume; ud->ddev.device_terminate_all = udma_terminate_all; ud->ddev.device_synchronize = udma_synchronize; +#ifdef CONFIG_DEBUG_FS + ud->ddev.dbg_summary_show = udma_dbg_summary_show; +#endif ud->ddev.device_free_chan_resources = udma_free_chan_resources; ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; @@ -3361,6 +3683,10 @@ static int udma_probe(struct platform_device *pdev) if (ud->desc_align < dma_get_cache_alignment()) ud->desc_align = dma_get_cache_alignment(); + ret = udma_setup_rx_flush(ud); + if (ret) + return ret; + for (i = 0; i < ud->tchan_cnt; i++) { struct udma_tchan *tchan = &ud->tchans[i]; diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c index a014ab96e673..918301e17552 100644 --- a/drivers/dma/ti/omap-dma.c +++ b/drivers/dma/ti/omap-dma.c @@ -124,7 +124,7 @@ struct omap_desc { uint32_t csdp; /* CSDP value */ unsigned sglen; - struct omap_sg sg[0]; + struct omap_sg sg[]; }; enum { diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c index 21b8f1131d55..618839df0748 100644 --- a/drivers/dma/uniphier-mdmac.c +++ b/drivers/dma/uniphier-mdmac.c @@ -68,7 +68,7 @@ struct uniphier_mdmac_device { struct dma_device ddev; struct clk *clk; void __iomem *reg_base; - struct uniphier_mdmac_chan channels[0]; + struct uniphier_mdmac_chan channels[]; }; static struct uniphier_mdmac_chan * diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c new file mode 100644 index 000000000000..7b2f8a8c2d31 --- /dev/null +++ b/drivers/dma/uniphier-xdmac.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * External DMA controller driver for UniPhier SoCs + * Copyright 2019 Socionext Inc. + * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com> + */ + +#include <linux/bitops.h> +#include <linux/bitfield.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define XDMAC_CH_WIDTH 0x100 + +#define XDMAC_TFA 0x08 +#define XDMAC_TFA_MCNT_MASK GENMASK(23, 16) +#define XDMAC_TFA_MASK GENMASK(5, 0) +#define XDMAC_SADM 0x10 +#define XDMAC_SADM_STW_MASK GENMASK(25, 24) +#define XDMAC_SADM_SAM BIT(4) +#define XDMAC_SADM_SAM_FIXED XDMAC_SADM_SAM +#define XDMAC_SADM_SAM_INC 0 +#define XDMAC_DADM 0x14 +#define XDMAC_DADM_DTW_MASK XDMAC_SADM_STW_MASK +#define XDMAC_DADM_DAM XDMAC_SADM_SAM +#define XDMAC_DADM_DAM_FIXED XDMAC_SADM_SAM_FIXED +#define XDMAC_DADM_DAM_INC XDMAC_SADM_SAM_INC +#define XDMAC_EXSAD 0x18 +#define XDMAC_EXDAD 0x1c +#define XDMAC_SAD 0x20 +#define XDMAC_DAD 0x24 +#define XDMAC_ITS 0x28 +#define XDMAC_ITS_MASK GENMASK(25, 0) +#define XDMAC_TNUM 0x2c +#define XDMAC_TNUM_MASK GENMASK(15, 0) +#define XDMAC_TSS 0x30 +#define XDMAC_TSS_REQ BIT(0) +#define XDMAC_IEN 0x34 +#define XDMAC_IEN_ERRIEN BIT(1) +#define XDMAC_IEN_ENDIEN BIT(0) +#define XDMAC_STAT 0x40 +#define XDMAC_STAT_TENF BIT(0) +#define XDMAC_IR 0x44 +#define XDMAC_IR_ERRF BIT(1) +#define XDMAC_IR_ENDF BIT(0) +#define XDMAC_ID 0x48 +#define XDMAC_ID_ERRIDF BIT(1) +#define XDMAC_ID_ENDIDF BIT(0) + +#define XDMAC_MAX_CHANS 16 +#define XDMAC_INTERVAL_CLKS 20 +#define XDMAC_MAX_WORDS XDMAC_TNUM_MASK + +/* cut lower bit for maintain alignment of maximum transfer size */ +#define XDMAC_MAX_WORD_SIZE (XDMAC_ITS_MASK & ~GENMASK(3, 0)) + +#define UNIPHIER_XDMAC_BUSWIDTHS \ + (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +struct uniphier_xdmac_desc_node { + dma_addr_t src; + dma_addr_t dst; + u32 burst_size; + u32 nr_burst; +}; + +struct uniphier_xdmac_desc { + struct virt_dma_desc vd; + + unsigned int nr_node; + unsigned int cur_node; + enum dma_transfer_direction dir; + struct uniphier_xdmac_desc_node nodes[]; +}; + +struct uniphier_xdmac_chan { + struct virt_dma_chan vc; + struct uniphier_xdmac_device *xdev; + struct uniphier_xdmac_desc *xd; + void __iomem *reg_ch_base; + struct dma_slave_config sconfig; + int id; + unsigned int req_factor; +}; + +struct uniphier_xdmac_device { + struct dma_device ddev; + void __iomem *reg_base; + int nr_chans; + struct uniphier_xdmac_chan channels[]; +}; + +static struct uniphier_xdmac_chan * +to_uniphier_xdmac_chan(struct virt_dma_chan *vc) +{ + return container_of(vc, struct uniphier_xdmac_chan, vc); +} + +static struct uniphier_xdmac_desc * +to_uniphier_xdmac_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct uniphier_xdmac_desc, vd); +} + +/* xc->vc.lock must be held by caller */ +static struct uniphier_xdmac_desc * +uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc) +{ + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&xc->vc); + if (!vd) + return NULL; + + list_del(&vd->node); + + return to_uniphier_xdmac_desc(vd); +} + +/* xc->vc.lock must be held by caller */ +static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc, + struct uniphier_xdmac_desc *xd) +{ + u32 src_mode, src_addr, src_width; + u32 dst_mode, dst_addr, dst_width; + u32 val, its, tnum; + enum dma_slave_buswidth buswidth; + + src_addr = xd->nodes[xd->cur_node].src; + dst_addr = xd->nodes[xd->cur_node].dst; + its = xd->nodes[xd->cur_node].burst_size; + tnum = xd->nodes[xd->cur_node].nr_burst; + + /* + * The width of MEM side must be 4 or 8 bytes, that does not + * affect that of DEV side and transfer size. + */ + if (xd->dir == DMA_DEV_TO_MEM) { + src_mode = XDMAC_SADM_SAM_FIXED; + buswidth = xc->sconfig.src_addr_width; + } else { + src_mode = XDMAC_SADM_SAM_INC; + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; + } + src_width = FIELD_PREP(XDMAC_SADM_STW_MASK, __ffs(buswidth)); + + if (xd->dir == DMA_MEM_TO_DEV) { + dst_mode = XDMAC_DADM_DAM_FIXED; + buswidth = xc->sconfig.dst_addr_width; + } else { + dst_mode = XDMAC_DADM_DAM_INC; + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; + } + dst_width = FIELD_PREP(XDMAC_DADM_DTW_MASK, __ffs(buswidth)); + + /* setup transfer factor */ + val = FIELD_PREP(XDMAC_TFA_MCNT_MASK, XDMAC_INTERVAL_CLKS); + val |= FIELD_PREP(XDMAC_TFA_MASK, xc->req_factor); + writel(val, xc->reg_ch_base + XDMAC_TFA); + + /* setup the channel */ + writel(lower_32_bits(src_addr), xc->reg_ch_base + XDMAC_SAD); + writel(upper_32_bits(src_addr), xc->reg_ch_base + XDMAC_EXSAD); + + writel(lower_32_bits(dst_addr), xc->reg_ch_base + XDMAC_DAD); + writel(upper_32_bits(dst_addr), xc->reg_ch_base + XDMAC_EXDAD); + + src_mode |= src_width; + dst_mode |= dst_width; + writel(src_mode, xc->reg_ch_base + XDMAC_SADM); + writel(dst_mode, xc->reg_ch_base + XDMAC_DADM); + + writel(its, xc->reg_ch_base + XDMAC_ITS); + writel(tnum, xc->reg_ch_base + XDMAC_TNUM); + + /* enable interrupt */ + writel(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN, + xc->reg_ch_base + XDMAC_IEN); + + /* start XDMAC */ + val = readl(xc->reg_ch_base + XDMAC_TSS); + val |= XDMAC_TSS_REQ; + writel(val, xc->reg_ch_base + XDMAC_TSS); +} + +/* xc->vc.lock must be held by caller */ +static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc) +{ + u32 val; + + /* disable interrupt */ + val = readl(xc->reg_ch_base + XDMAC_IEN); + val &= ~(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN); + writel(val, xc->reg_ch_base + XDMAC_IEN); + + /* stop XDMAC */ + val = readl(xc->reg_ch_base + XDMAC_TSS); + val &= ~XDMAC_TSS_REQ; + writel(0, xc->reg_ch_base + XDMAC_TSS); + + /* wait until transfer is stopped */ + return readl_poll_timeout(xc->reg_ch_base + XDMAC_STAT, val, + !(val & XDMAC_STAT_TENF), 100, 1000); +} + +/* xc->vc.lock must be held by caller */ +static void uniphier_xdmac_start(struct uniphier_xdmac_chan *xc) +{ + struct uniphier_xdmac_desc *xd; + + xd = uniphier_xdmac_next_desc(xc); + if (xd) + uniphier_xdmac_chan_start(xc, xd); + + /* set desc to chan regardless of xd is null */ + xc->xd = xd; +} + +static void uniphier_xdmac_chan_irq(struct uniphier_xdmac_chan *xc) +{ + u32 stat; + int ret; + + spin_lock(&xc->vc.lock); + + stat = readl(xc->reg_ch_base + XDMAC_ID); + + if (stat & XDMAC_ID_ERRIDF) { + ret = uniphier_xdmac_chan_stop(xc); + if (ret) + dev_err(xc->xdev->ddev.dev, + "DMA transfer error with aborting issue\n"); + else + dev_err(xc->xdev->ddev.dev, + "DMA transfer error\n"); + + } else if ((stat & XDMAC_ID_ENDIDF) && xc->xd) { + xc->xd->cur_node++; + if (xc->xd->cur_node >= xc->xd->nr_node) { + vchan_cookie_complete(&xc->xd->vd); + uniphier_xdmac_start(xc); + } else { + uniphier_xdmac_chan_start(xc, xc->xd); + } + } + + /* write bits to clear */ + writel(stat, xc->reg_ch_base + XDMAC_IR); + + spin_unlock(&xc->vc.lock); +} + +static irqreturn_t uniphier_xdmac_irq_handler(int irq, void *dev_id) +{ + struct uniphier_xdmac_device *xdev = dev_id; + int i; + + for (i = 0; i < xdev->nr_chans; i++) + uniphier_xdmac_chan_irq(&xdev->channels[i]); + + return IRQ_HANDLED; +} + +static void uniphier_xdmac_free_chan_resources(struct dma_chan *chan) +{ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static struct dma_async_tx_descriptor * +uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_desc *xd; + unsigned int nr; + size_t burst_size, tlen; + int i; + + if (len > XDMAC_MAX_WORD_SIZE * XDMAC_MAX_WORDS) + return NULL; + + nr = 1 + len / XDMAC_MAX_WORD_SIZE; + + xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT); + if (!xd) + return NULL; + + for (i = 0; i < nr; i++) { + burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE); + xd->nodes[i].src = src; + xd->nodes[i].dst = dst; + xd->nodes[i].burst_size = burst_size; + xd->nodes[i].nr_burst = len / burst_size; + tlen = rounddown(len, burst_size); + src += tlen; + dst += tlen; + len -= tlen; + } + + xd->dir = DMA_MEM_TO_MEM; + xd->nr_node = nr; + xd->cur_node = 0; + + return vchan_tx_prep(vc, &xd->vd, flags); +} + +static struct dma_async_tx_descriptor * +uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + struct uniphier_xdmac_desc *xd; + struct scatterlist *sg; + enum dma_slave_buswidth buswidth; + u32 maxburst; + int i; + + if (!is_slave_direction(direction)) + return NULL; + + if (direction == DMA_DEV_TO_MEM) { + buswidth = xc->sconfig.src_addr_width; + maxburst = xc->sconfig.src_maxburst; + } else { + buswidth = xc->sconfig.dst_addr_width; + maxburst = xc->sconfig.dst_maxburst; + } + + if (!maxburst) + maxburst = 1; + if (maxburst > xc->xdev->ddev.max_burst) { + dev_err(xc->xdev->ddev.dev, + "Exceed maximum number of burst words\n"); + return NULL; + } + + xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT); + if (!xd) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) { + xd->nodes[i].src = (direction == DMA_DEV_TO_MEM) + ? xc->sconfig.src_addr : sg_dma_address(sg); + xd->nodes[i].dst = (direction == DMA_MEM_TO_DEV) + ? xc->sconfig.dst_addr : sg_dma_address(sg); + xd->nodes[i].burst_size = maxburst * buswidth; + xd->nodes[i].nr_burst = + sg_dma_len(sg) / xd->nodes[i].burst_size; + + /* + * Currently transfer that size doesn't align the unit size + * (the number of burst words * bus-width) is not allowed, + * because the driver does not support the way to transfer + * residue size. As a matter of fact, in order to transfer + * arbitrary size, 'src_maxburst' or 'dst_maxburst' of + * dma_slave_config must be 1. + */ + if (sg_dma_len(sg) % xd->nodes[i].burst_size) { + dev_err(xc->xdev->ddev.dev, + "Unaligned transfer size: %d", sg_dma_len(sg)); + kfree(xd); + return NULL; + } + + if (xd->nodes[i].nr_burst > XDMAC_MAX_WORDS) { + dev_err(xc->xdev->ddev.dev, + "Exceed maximum transfer size"); + kfree(xd); + return NULL; + } + } + + xd->dir = direction; + xd->nr_node = sg_len; + xd->cur_node = 0; + + return vchan_tx_prep(vc, &xd->vd, flags); +} + +static int uniphier_xdmac_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + + memcpy(&xc->sconfig, config, sizeof(*config)); + + return 0; +} + +static int uniphier_xdmac_terminate_all(struct dma_chan *chan) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + unsigned long flags; + int ret = 0; + LIST_HEAD(head); + + spin_lock_irqsave(&vc->lock, flags); + + if (xc->xd) { + vchan_terminate_vdesc(&xc->xd->vd); + xc->xd = NULL; + ret = uniphier_xdmac_chan_stop(xc); + } + + vchan_get_all_descriptors(vc, &head); + + spin_unlock_irqrestore(&vc->lock, flags); + + vchan_dma_desc_free_list(vc, &head); + + return ret; +} + +static void uniphier_xdmac_synchronize(struct dma_chan *chan) +{ + vchan_synchronize(to_virt_chan(chan)); +} + +static void uniphier_xdmac_issue_pending(struct dma_chan *chan) +{ + struct virt_dma_chan *vc = to_virt_chan(chan); + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); + unsigned long flags; + + spin_lock_irqsave(&vc->lock, flags); + + if (vchan_issue_pending(vc) && !xc->xd) + uniphier_xdmac_start(xc); + + spin_unlock_irqrestore(&vc->lock, flags); +} + +static void uniphier_xdmac_desc_free(struct virt_dma_desc *vd) +{ + kfree(to_uniphier_xdmac_desc(vd)); +} + +static void uniphier_xdmac_chan_init(struct uniphier_xdmac_device *xdev, + int ch) +{ + struct uniphier_xdmac_chan *xc = &xdev->channels[ch]; + + xc->xdev = xdev; + xc->reg_ch_base = xdev->reg_base + XDMAC_CH_WIDTH * ch; + xc->vc.desc_free = uniphier_xdmac_desc_free; + + vchan_init(&xc->vc, &xdev->ddev); +} + +static struct dma_chan *of_dma_uniphier_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct uniphier_xdmac_device *xdev = ofdma->of_dma_data; + int chan_id = dma_spec->args[0]; + + if (chan_id >= xdev->nr_chans) + return NULL; + + xdev->channels[chan_id].id = chan_id; + xdev->channels[chan_id].req_factor = dma_spec->args[1]; + + return dma_get_slave_channel(&xdev->channels[chan_id].vc.chan); +} + +static int uniphier_xdmac_probe(struct platform_device *pdev) +{ + struct uniphier_xdmac_device *xdev; + struct device *dev = &pdev->dev; + struct dma_device *ddev; + int irq; + int nr_chans; + int i, ret; + + if (of_property_read_u32(dev->of_node, "dma-channels", &nr_chans)) + return -EINVAL; + if (nr_chans > XDMAC_MAX_CHANS) + nr_chans = XDMAC_MAX_CHANS; + + xdev = devm_kzalloc(dev, struct_size(xdev, channels, nr_chans), + GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + xdev->nr_chans = nr_chans; + xdev->reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(xdev->reg_base)) + return PTR_ERR(xdev->reg_base); + + ddev = &xdev->ddev; + ddev->dev = dev; + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_MEMCPY, ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + ddev->src_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; + ddev->dst_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | + BIT(DMA_MEM_TO_MEM); + ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + ddev->max_burst = XDMAC_MAX_WORDS; + ddev->device_free_chan_resources = uniphier_xdmac_free_chan_resources; + ddev->device_prep_dma_memcpy = uniphier_xdmac_prep_dma_memcpy; + ddev->device_prep_slave_sg = uniphier_xdmac_prep_slave_sg; + ddev->device_config = uniphier_xdmac_slave_config; + ddev->device_terminate_all = uniphier_xdmac_terminate_all; + ddev->device_synchronize = uniphier_xdmac_synchronize; + ddev->device_tx_status = dma_cookie_status; + ddev->device_issue_pending = uniphier_xdmac_issue_pending; + INIT_LIST_HEAD(&ddev->channels); + + for (i = 0; i < nr_chans; i++) + uniphier_xdmac_chan_init(xdev, i); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, uniphier_xdmac_irq_handler, + IRQF_SHARED, "xdmac", xdev); + if (ret) { + dev_err(dev, "Failed to request IRQ\n"); + return ret; + } + + ret = dma_async_device_register(ddev); + if (ret) { + dev_err(dev, "Failed to register XDMA device\n"); + return ret; + } + + ret = of_dma_controller_register(dev->of_node, + of_dma_uniphier_xlate, xdev); + if (ret) { + dev_err(dev, "Failed to register XDMA controller\n"); + goto out_unregister_dmac; + } + + platform_set_drvdata(pdev, xdev); + + dev_info(&pdev->dev, "UniPhier XDMAC driver (%d channels)\n", + nr_chans); + + return 0; + +out_unregister_dmac: + dma_async_device_unregister(ddev); + + return ret; +} + +static int uniphier_xdmac_remove(struct platform_device *pdev) +{ + struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); + struct dma_device *ddev = &xdev->ddev; + struct dma_chan *chan; + int ret; + + /* + * Before reaching here, almost all descriptors have been freed by the + * ->device_free_chan_resources() hook. However, each channel might + * be still holding one descriptor that was on-flight at that moment. + * Terminate it to make sure this hardware is no longer running. Then, + * free the channel resources once again to avoid memory leak. + */ + list_for_each_entry(chan, &ddev->channels, device_node) { + ret = dmaengine_terminate_sync(chan); + if (ret) + return ret; + uniphier_xdmac_free_chan_resources(chan); + } + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(ddev); + + return 0; +} + +static const struct of_device_id uniphier_xdmac_match[] = { + { .compatible = "socionext,uniphier-xdmac" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, uniphier_xdmac_match); + +static struct platform_driver uniphier_xdmac_driver = { + .probe = uniphier_xdmac_probe, + .remove = uniphier_xdmac_remove, + .driver = { + .name = "uniphier-xdmac", + .of_match_table = uniphier_xdmac_match, + }, +}; +module_platform_driver(uniphier_xdmac_driver); + +MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>"); +MODULE_DESCRIPTION("UniPhier external DMA controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index a9c5d5cc9f2b..aecd5a35a296 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -125,7 +125,9 @@ #define XILINX_VDMA_ENABLE_VERTICAL_FLIP BIT(0) /* HW specific definitions */ -#define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20 +#define XILINX_MCDMA_MAX_CHANS_PER_DEVICE 0x20 +#define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x2 +#define XILINX_CDMA_MAX_CHANS_PER_DEVICE 0x1 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ @@ -468,6 +470,7 @@ struct xilinx_dma_config { struct clk **tx_clk, struct clk **txs_clk, struct clk **rx_clk, struct clk **rxs_clk); irqreturn_t (*irq_handler)(int irq, void *data); + const int max_channels; }; /** @@ -485,16 +488,15 @@ struct xilinx_dma_config { * @txs_clk: DMA mm2s stream clock * @rx_clk: DMA s2mm clock * @rxs_clk: DMA s2mm stream clock - * @nr_channels: Number of channels DMA device supports - * @chan_id: DMA channel identifier + * @s2mm_chan_id: DMA s2mm channel identifier + * @mm2s_chan_id: DMA mm2s channel identifier * @max_buffer_len: Max buffer length - * @s2mm_index: S2MM channel index */ struct xilinx_dma_device { void __iomem *regs; struct device *dev; struct dma_device common; - struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; + struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE]; u32 flush_on_fsync; bool ext_addr; struct platform_device *pdev; @@ -504,10 +506,9 @@ struct xilinx_dma_device { struct clk *txs_clk; struct clk *rx_clk; struct clk *rxs_clk; - u32 nr_channels; - u32 chan_id; + u32 s2mm_chan_id; + u32 mm2s_chan_id; u32 max_buffer_len; - u32 s2mm_index; }; /* Macros */ @@ -1745,7 +1746,7 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) return IRQ_NONE; if (chan->direction == DMA_DEV_TO_MEM) - chan_offset = chan->xdev->s2mm_index; + chan_offset = chan->xdev->dma_config->max_channels / 2; chan_offset = chan_offset + (chan_id - 1); chan = chan->xdev->chan[chan_offset]; @@ -2404,16 +2405,17 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan) u32 reg; int err; - if (chan->cyclic) - xilinx_dma_chan_reset(chan); - - err = chan->stop_transfer(chan); - if (err) { - dev_err(chan->dev, "Cannot stop channel %p: %x\n", - chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); - chan->err = true; + if (!chan->cyclic) { + err = chan->stop_transfer(chan); + if (err) { + dev_err(chan->dev, "Cannot stop channel %p: %x\n", + chan, dma_ctrl_read(chan, + XILINX_DMA_REG_DMASR)); + chan->err = true; + } } + xilinx_dma_chan_reset(chan); /* Remove and free all of the descriptors in the lists */ xilinx_dma_free_descriptors(chan); chan->idle = true; @@ -2730,12 +2732,11 @@ static void xdma_disable_allclks(struct xilinx_dma_device *xdev) * * @xdev: Driver specific device structure * @node: Device node - * @chan_id: DMA Channel id * * Return: '0' on success and failure value on error */ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, - struct device_node *node, int chan_id) + struct device_node *node) { struct xilinx_dma_chan *chan; bool has_dre = false; @@ -2787,8 +2788,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { chan->direction = DMA_MEM_TO_DEV; - chan->id = chan_id; - chan->tdest = chan_id; + chan->id = xdev->mm2s_chan_id++; + chan->tdest = chan->id; chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { @@ -2804,9 +2805,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, of_device_is_compatible(node, "xlnx,axi-dma-s2mm-channel")) { chan->direction = DMA_DEV_TO_MEM; - chan->id = chan_id; - xdev->s2mm_index = xdev->nr_channels; - chan->tdest = chan_id - xdev->nr_channels; + chan->id = xdev->s2mm_chan_id++; + chan->tdest = chan->id - xdev->dma_config->max_channels / 2; chan->has_vflip = of_property_read_bool(node, "xlnx,enable-vert-flip"); if (chan->has_vflip) { @@ -2908,9 +2908,7 @@ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, dev_warn(xdev->dev, "missing dma-channels property\n"); for (i = 0; i < nr_channels; i++) - xilinx_dma_chan_probe(xdev, node, xdev->chan_id++); - - xdev->nr_channels += nr_channels; + xilinx_dma_chan_probe(xdev, node); return 0; } @@ -2928,7 +2926,7 @@ static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, struct xilinx_dma_device *xdev = ofdma->of_dma_data; int chan_id = dma_spec->args[0]; - if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id]) + if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id]) return NULL; return dma_get_slave_channel(&xdev->chan[chan_id]->common); @@ -2938,23 +2936,27 @@ static const struct xilinx_dma_config axidma_config = { .dmatype = XDMA_TYPE_AXIDMA, .clk_init = axidma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config aximcdma_config = { .dmatype = XDMA_TYPE_AXIMCDMA, .clk_init = axidma_clk_init, .irq_handler = xilinx_mcdma_irq_handler, + .max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config axicdma_config = { .dmatype = XDMA_TYPE_CDMA, .clk_init = axicdma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE, }; static const struct xilinx_dma_config axivdma_config = { .dmatype = XDMA_TYPE_VDMA, .clk_init = axivdma_clk_init, .irq_handler = xilinx_dma_irq_handler, + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, }; static const struct of_device_id xilinx_dma_of_ids[] = { @@ -3011,6 +3013,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Retrieve the DMA engine properties from the device tree */ xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); + xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { @@ -3104,7 +3107,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xdev->chan[i]->num_frms = num_frames; } @@ -3134,7 +3137,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) disable_clks: xdma_disable_allclks(xdev); error: - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); @@ -3156,7 +3159,7 @@ static int xilinx_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&xdev->common); - for (i = 0; i < xdev->nr_channels; i++) + for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); |