diff options
Diffstat (limited to 'drivers/dma/at_xdmac.c')
| -rw-r--r-- | drivers/dma/at_xdmac.c | 220 |
1 files changed, 203 insertions, 17 deletions
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index d6c9781cd46a..1f0fab180f8f 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -21,6 +21,7 @@ #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm.h> +#include <linux/pm_runtime.h> #include "dmaengine.h" @@ -240,6 +241,7 @@ struct at_xdmac_chan { struct at_xdmac { struct dma_device dma; void __iomem *regs; + struct device *dev; int irq; struct clk *clk; u32 save_gim; @@ -361,13 +363,65 @@ MODULE_PARM_DESC(init_nr_desc_per_channel, "initial descriptors per channel (default: 64)"); +static void at_xdmac_runtime_suspend_descriptors(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + struct at_xdmac_desc *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { + if (!desc->active_xfer) + continue; + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + } +} + +static int at_xdmac_runtime_resume_descriptors(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + struct at_xdmac_desc *desc, *_desc; + int ret; + + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { + if (!desc->active_xfer) + continue; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + } + + return 0; +} + static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan) { - return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask; + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return false; + + ret = !!(at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask); + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + + return ret; } static void at_xdmac_off(struct at_xdmac *atxdmac) { + struct dma_chan *chan, *_chan; + struct at_xdmac_chan *atchan; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return; + at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L); /* Wait that all chans are disabled. */ @@ -375,6 +429,18 @@ static void at_xdmac_off(struct at_xdmac *atxdmac) cpu_relax(); at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L); + + /* Decrement runtime PM ref counter for each active descriptor. */ + if (!list_empty(&atxdmac->dma.channels)) { + list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, + device_node) { + atchan = to_at_xdmac_chan(chan); + at_xdmac_runtime_suspend_descriptors(atchan); + } + } + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); } /* Call with lock hold. */ @@ -383,6 +449,11 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, { struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); u32 reg; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return; dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); @@ -462,7 +533,6 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, at_xdmac_chan_read(atchan, AT_XDMAC_CSA), at_xdmac_chan_read(atchan, AT_XDMAC_CDA), at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); - } static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) @@ -1456,14 +1526,14 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, static enum dma_status at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) + struct dma_tx_state *txstate) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); struct at_xdmac_desc *desc, *_desc, *iter; struct list_head *descs_list; enum dma_status ret; - int residue, retry; + int residue, retry, pm_status; u32 cur_nda, check_nda, cur_ubc, mask, value; u8 dwidth = 0; unsigned long flags; @@ -1473,6 +1543,10 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, if (ret == DMA_COMPLETE || !txstate) return ret; + pm_status = pm_runtime_resume_and_get(atxdmac->dev); + if (pm_status < 0) + return DMA_ERROR; + spin_lock_irqsave(&atchan->lock, flags); desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); @@ -1590,6 +1664,8 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, spin_unlock: spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); return ret; } @@ -1636,6 +1712,11 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) { struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); struct at_xdmac_desc *bad_desc; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return; /* * The descriptor currently at the head of the active list is @@ -1665,12 +1746,16 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da, bad_desc->lld.mbr_ubc); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + /* Then continue with usual descriptor management */ } static void at_xdmac_tasklet(struct tasklet_struct *t) { struct at_xdmac_chan *atchan = from_tasklet(atchan, t, tasklet); + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); struct at_xdmac_desc *desc; struct dma_async_tx_descriptor *txd; u32 error_mask; @@ -1720,6 +1805,13 @@ static void at_xdmac_tasklet(struct tasklet_struct *t) list_splice_tail_init(&desc->descs_list, &atchan->free_descs_list); at_xdmac_advance_work(atchan); spin_unlock_irq(&atchan->lock); + + /* + * Decrement runtime PM ref counter incremented in + * at_xdmac_start_xfer(). + */ + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); } static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) @@ -1811,19 +1903,31 @@ static int at_xdmac_device_pause(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status)) return 0; + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask); while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP)) cpu_relax(); + + /* Decrement runtime PM ref counter for each active descriptor. */ + at_xdmac_runtime_suspend_descriptors(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + return 0; } @@ -1832,20 +1936,32 @@ static int at_xdmac_device_resume(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); - if (!at_xdmac_chan_is_paused(atchan)) { - spin_unlock_irqrestore(&atchan->lock, flags); - return 0; - } + if (!at_xdmac_chan_is_paused(atchan)) + goto unlock; + + /* Increment runtime PM ref counter for each active descriptor. */ + ret = at_xdmac_runtime_resume_descriptors(atchan); + if (ret < 0) + goto unlock; at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); + +unlock: spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); - return 0; + return ret; } static int at_xdmac_device_terminate_all(struct dma_chan *chan) @@ -1854,9 +1970,14 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) @@ -1867,12 +1988,24 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan) list_del(&desc->xfer_node); list_splice_tail_init(&desc->descs_list, &atchan->free_descs_list); + /* + * We incremented the runtime PM reference count on + * at_xdmac_start_xfer() for this descriptor. Now it's time + * to release it. + */ + if (desc->active_xfer) { + pm_runtime_put_autosuspend(atxdmac->dev); + pm_runtime_mark_last_busy(atxdmac->dev); + } } clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + return 0; } @@ -1974,6 +2107,11 @@ static int __maybe_unused atmel_xdmac_suspend(struct device *dev) { struct at_xdmac *atxdmac = dev_get_drvdata(dev); struct dma_chan *chan, *_chan; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); @@ -1986,12 +2124,13 @@ static int __maybe_unused atmel_xdmac_suspend(struct device *dev) atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA); atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC); } + + at_xdmac_runtime_suspend_descriptors(atchan); } atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM); at_xdmac_off(atxdmac); - clk_disable_unprepare(atxdmac->clk); - return 0; + return pm_runtime_force_suspend(atxdmac->dev); } static int __maybe_unused atmel_xdmac_resume(struct device *dev) @@ -2003,8 +2142,8 @@ static int __maybe_unused atmel_xdmac_resume(struct device *dev) int i; int ret; - ret = clk_prepare_enable(atxdmac->clk); - if (ret) + ret = pm_runtime_force_resume(atxdmac->dev); + if (ret < 0) return ret; at_xdmac_axi_config(pdev); @@ -2019,6 +2158,11 @@ static int __maybe_unused atmel_xdmac_resume(struct device *dev) at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { atchan = to_at_xdmac_chan(chan); + + ret = at_xdmac_runtime_resume_descriptors(atchan); + if (ret < 0) + return ret; + at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc); if (at_xdmac_chan_is_cyclic(atchan)) { if (at_xdmac_chan_is_paused(atchan)) @@ -2030,9 +2174,29 @@ static int __maybe_unused atmel_xdmac_resume(struct device *dev) at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); } } + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + + return 0; +} + +static int __maybe_unused atmel_xdmac_runtime_suspend(struct device *dev) +{ + struct at_xdmac *atxdmac = dev_get_drvdata(dev); + + clk_disable(atxdmac->clk); + return 0; } +static int __maybe_unused atmel_xdmac_runtime_resume(struct device *dev) +{ + struct at_xdmac *atxdmac = dev_get_drvdata(dev); + + return clk_enable(atxdmac->clk); +} + static int at_xdmac_probe(struct platform_device *pdev) { struct at_xdmac *atxdmac; @@ -2071,6 +2235,7 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->regs = base; atxdmac->irq = irq; + atxdmac->dev = &pdev->dev; atxdmac->layout = of_device_get_match_data(&pdev->dev); if (!atxdmac->layout) @@ -2135,11 +2300,20 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - /* Disable all chans and interrupts. */ - at_xdmac_off(atxdmac); + platform_set_drvdata(pdev, atxdmac); + + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); /* Init channels. */ INIT_LIST_HEAD(&atxdmac->dma.channels); + + /* Disable all chans and interrupts. */ + at_xdmac_off(atxdmac); + for (i = 0; i < nr_channels; i++) { struct at_xdmac_chan *atchan = &atxdmac->chan[i]; @@ -2159,12 +2333,11 @@ static int at_xdmac_probe(struct platform_device *pdev) while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) cpu_relax(); } - platform_set_drvdata(pdev, atxdmac); ret = dma_async_device_register(&atxdmac->dma); if (ret) { dev_err(&pdev->dev, "fail to register DMA engine device\n"); - goto err_clk_disable; + goto err_pm_disable; } ret = of_dma_controller_register(pdev->dev.of_node, @@ -2179,10 +2352,18 @@ static int at_xdmac_probe(struct platform_device *pdev) at_xdmac_axi_config(pdev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + return 0; err_dma_unregister: dma_async_device_unregister(&atxdmac->dma); +err_pm_disable: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); err_clk_disable: clk_disable_unprepare(atxdmac->clk); err_free_irq: @@ -2198,6 +2379,9 @@ static int at_xdmac_remove(struct platform_device *pdev) at_xdmac_off(atxdmac); of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&atxdmac->dma); + pm_runtime_disable(atxdmac->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); clk_disable_unprepare(atxdmac->clk); free_irq(atxdmac->irq, atxdmac); @@ -2215,6 +2399,8 @@ static int at_xdmac_remove(struct platform_device *pdev) static const struct dev_pm_ops __maybe_unused atmel_xdmac_dev_pm_ops = { .prepare = atmel_xdmac_prepare, SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume) + SET_RUNTIME_PM_OPS(atmel_xdmac_runtime_suspend, + atmel_xdmac_runtime_resume, NULL) }; static const struct of_device_id atmel_xdmac_dt_ids[] = { |